You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/08/11 06:44:17 UTC
[1/2] lucene-solr:master: SOLR-9320: A REPLACENODE command to
decommission an existing node with another new node and SOLR-9318 the
DELETENODE command that deletes all replicas in a node
Repository: lucene-solr
Updated Branches:
refs/heads/master bc25a565d -> 92b5a76b5
SOLR-9320: A REPLACENODE command to decommission an existing node with another new node and SOLR-9318 the DELETENODE command that deletes all replicas in a node
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ae60c74f
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ae60c74f
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ae60c74f
Branch: refs/heads/master
Commit: ae60c74f8c6ea2f62e1870802accbcd73bbfdc48
Parents: aba731a
Author: Noble Paul <no...@apache.org>
Authored: Thu Aug 11 12:12:48 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Thu Aug 11 12:12:48 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 5 +
.../org/apache/solr/cloud/DeleteNodeCmd.java | 86 ++++++++++
.../OverseerCollectionConfigSetProcessor.java | 22 ++-
.../cloud/OverseerCollectionMessageHandler.java | 160 ++++++++++++++----
.../solr/cloud/OverseerTaskProcessor.java | 6 +-
.../org/apache/solr/cloud/ReplaceNodeCmd.java | 164 +++++++++++++++++++
.../solr/handler/admin/CollectionsHandler.java | 5 +-
.../org/apache/solr/cloud/DeleteNodeTest.java | 75 +++++++++
.../org/apache/solr/cloud/ReplaceNodeTest.java | 104 ++++++++++++
.../solrj/request/CollectionAdminRequest.java | 52 +++++-
.../apache/solr/common/cloud/ZkNodeProps.java | 12 ++
.../solr/common/params/CollectionParams.java | 3 +
12 files changed, 650 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 4da2d0e..889611f 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -100,6 +100,11 @@ New Features
* SOLR-9275: XML QueryParser support (defType=xmlparser) now extensible via configuration.
(Christine Poerschke)
+* SOLR-9320: A REPLACENODE command to decommission an existing node with another new node
+ (noble, Nitin Sharma, Varun Thacker)
+
+* SOLR-9318: A DELETENODE command to delete all replicas in that node (noble, Nitin Sharma, Varun Thacker)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java
new file mode 100644
index 0000000..cbcfa88
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/DeleteNodeCmd.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+
+import java.lang.invoke.MethodHandles;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DeleteNodeCmd implements OverseerCollectionMessageHandler.Cmd {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final OverseerCollectionMessageHandler ocmh;
+
+ public DeleteNodeCmd(OverseerCollectionMessageHandler ocmh) {
+ this.ocmh = ocmh;
+ }
+
+ @Override
+ public Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
+ ocmh.checkRequired(message, "node");
+ String node = message.getStr("node");
+ if (!state.liveNodesContain(node)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + node + " is not live");
+ }
+ List<ZkNodeProps> sourceReplicas = ReplaceNodeCmd.getReplicasOfNode(node, state);
+ cleanupReplicas(results, state, sourceReplicas, ocmh);
+ return null;
+ }
+
+ static void cleanupReplicas(NamedList results,
+ ClusterState clusterState,
+ List<ZkNodeProps> sourceReplicas,
+ OverseerCollectionMessageHandler ocmh) throws InterruptedException {
+ CountDownLatch cleanupLatch = new CountDownLatch(sourceReplicas.size());
+ for (ZkNodeProps sourceReplica : sourceReplicas) {
+ log.info("deleting replica from from node {} ", Utils.toJSONString(sourceReplica));
+ NamedList deleteResult = new NamedList();
+ try {
+ ocmh.deleteReplica(clusterState, sourceReplica.plus("parallel", "true"), deleteResult, () -> {
+ cleanupLatch.countDown();
+ if (deleteResult.get("failure") != null) {
+ synchronized (results) {
+ results.add("failure", "could not delete because " + deleteResult.get("failure") + " " + Utils.toJSONString(sourceReplica));
+ }
+ }
+ });
+ } catch (KeeperException e) {
+ log.info("Error deleting ", e);
+ cleanupLatch.countDown();
+ } catch (Exception e) {
+ cleanupLatch.countDown();
+ throw e;
+ }
+ }
+ log.info("Waiting for deletes to complete");
+ cleanupLatch.await(5, TimeUnit.MINUTES);
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
index f8f8446..8c7a056 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
@@ -16,6 +16,10 @@
*/
package org.apache.solr.cloud;
+import java.io.IOException;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
@@ -83,12 +87,20 @@ public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor
zkStateReader, myId, shardHandlerFactory, adminPath, stats, overseer, overseerNodePrioritizer);
final OverseerConfigSetMessageHandler configMessageHandler = new OverseerConfigSetMessageHandler(
zkStateReader);
- return message -> {
- String operation = message.getStr(Overseer.QUEUE_OPERATION);
- if (operation != null && operation.startsWith(CONFIGSETS_ACTION_PREFIX)) {
- return configMessageHandler;
+ return new OverseerMessageHandlerSelector() {
+ @Override
+ public void close() throws IOException {
+ IOUtils.closeQuietly(collMessageHandler);
+ }
+
+ @Override
+ public OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message) {
+ String operation = message.getStr(Overseer.QUEUE_OPERATION);
+ if (operation != null && operation.startsWith(CONFIGSETS_ACTION_PREFIX)) {
+ return configMessageHandler;
+ }
+ return collMessageHandler;
}
- return collMessageHandler;
};
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 4e7e429..908d35c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -16,6 +16,7 @@
*/
package org.apache.solr.cloud;
+import java.io.Closeable;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URI;
@@ -35,8 +36,13 @@ import java.util.Optional;
import java.util.Properties;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.SolrServerException;
@@ -75,6 +81,7 @@ import org.apache.solr.common.params.CoreAdminParams;
import org.apache.solr.common.params.CoreAdminParams.CoreAdminAction;
import org.apache.solr.common.params.ModifiableSolrParams;
import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
@@ -88,6 +95,7 @@ import org.apache.solr.handler.component.ShardHandlerFactory;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
import org.apache.solr.update.SolrIndexSplitter;
+import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.TimeOut;
import org.apache.solr.util.stats.Snapshot;
@@ -119,10 +127,12 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.BA
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.CREATESHARD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETENODE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETEREPLICAPROP;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.DELETESHARD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MIGRATESTATEFORMAT;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.REMOVEROLE;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.REPLACENODE;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonParams.NAME;
import static org.apache.solr.common.util.StrUtils.formatString;
@@ -132,7 +142,7 @@ import static org.apache.solr.common.util.Utils.makeMap;
* A {@link OverseerMessageHandler} that handles Collections API related
* overseer messages.
*/
-public class OverseerCollectionMessageHandler implements OverseerMessageHandler {
+public class OverseerCollectionMessageHandler implements OverseerMessageHandler , Closeable {
public static final String NUM_SLICES = "numShards";
@@ -172,7 +182,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
private Overseer overseer;
private ShardHandlerFactory shardHandlerFactory;
private String adminPath;
- private ZkStateReader zkStateReader;
+ ZkStateReader zkStateReader;
private String myId;
private Overseer.Stats stats;
private OverseerNodePrioritizer overseerPrioritizer;
@@ -181,6 +191,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
// This is used for handling mutual exclusion of the tasks.
final private LockTree lockTree = new LockTree();
+ ExecutorService tpe = new ExecutorUtil.MDCAwareThreadPoolExecutor(5, 10, 0L, TimeUnit.MILLISECONDS,
+ new SynchronousQueue<>(),
+ new DefaultSolrThreadFactory("OverseerCollectionMessageHandlerThreadFactory"));
static final Random RANDOM;
static {
@@ -193,6 +206,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
RANDOM = new Random(seed.hashCode());
}
}
+ private final Map<CollectionParams.CollectionAction, Cmd> commandMap;
public OverseerCollectionMessageHandler(ZkStateReader zkStateReader, String myId,
final ShardHandlerFactory shardHandlerFactory,
@@ -207,6 +221,11 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
this.stats = stats;
this.overseer = overseer;
this.overseerPrioritizer = overseerPrioritizer;
+ commandMap = new ImmutableMap.Builder<CollectionParams.CollectionAction, Cmd>()
+ .put(REPLACENODE, new ReplaceNodeCmd(this))
+ .put(DELETENODE, new DeleteNodeCmd(this))
+ .build()
+ ;
}
@Override
@@ -244,7 +263,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
createShard(zkStateReader.getClusterState(), message, results);
break;
case DELETEREPLICA:
- deleteReplica(zkStateReader.getClusterState(), message, results);
+ deleteReplica(zkStateReader.getClusterState(), message, results, null);
break;
case MIGRATE:
migrate(zkStateReader.getClusterState(), message, results);
@@ -256,7 +275,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
processRoleCommand(message, operation);
break;
case ADDREPLICA:
- addReplica(zkStateReader.getClusterState(), message, results);
+ addReplica(zkStateReader.getClusterState(), message, results, null);
break;
case OVERSEERSTATUS:
getOverseerStatus(message, results);
@@ -294,9 +313,15 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
results.add("MOCK_FINISHED", System.currentTimeMillis());
break;
}
- default:
- throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
- + operation);
+ default: {
+ Cmd command = commandMap.get(action);
+ if (command != null) {
+ command.call(zkStateReader.getClusterState(),message, results);
+ } else {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ + operation);
+ }
+ }
}
} catch (Exception e) {
String collName = message.getStr("collection");
@@ -590,12 +615,14 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
@SuppressWarnings("unchecked")
- private void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results)
+ void deleteReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete)
throws KeeperException, InterruptedException {
+ log.info("deleteReplica() : {}", Utils.toJSONString(message));
checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP);
String collectionName = message.getStr(COLLECTION_PROP);
String shard = message.getStr(SHARD_ID_PROP);
String replicaName = message.getStr(REPLICA_PROP);
+ boolean parallel = message.getBool("parallel", false);
DocCollection coll = clusterState.getCollection(collectionName);
Slice slice = coll.getSlice(shard);
@@ -623,9 +650,9 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
String asyncId = message.getStr(ASYNC);
- Map<String, String> requestMap = null;
+ AtomicReference<Map<String, String>> requestMap = new AtomicReference<>(null);
if (asyncId != null) {
- requestMap = new HashMap<>(1, 1.0f);
+ requestMap.set(new HashMap<>(1, 1.0f));
}
ModifiableSolrParams params = new ModifiableSolrParams();
@@ -636,19 +663,42 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
params.set(CoreAdminParams.DELETE_INSTANCE_DIR, message.getBool(CoreAdminParams.DELETE_INSTANCE_DIR, true));
params.set(CoreAdminParams.DELETE_DATA_DIR, message.getBool(CoreAdminParams.DELETE_DATA_DIR, true));
- sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap);
+ sendShardRequest(replica.getNodeName(), params, shardHandler, asyncId, requestMap.get());
+ AtomicReference<Exception> exp = new AtomicReference<>();
- processResponses(results, shardHandler, false, null, asyncId, requestMap);
+ Callable<Boolean> callable = () -> {
+ try {
+ processResponses(results, shardHandler, false, null, asyncId, requestMap.get());
- //check if the core unload removed the corenode zk entry
- if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return;
+ //check if the core unload removed the corenode zk entry
+ if (waitForCoreNodeGone(collectionName, shard, replicaName, 5000)) return Boolean.TRUE;
- // try and ensure core info is removed from cluster state
- deleteCoreNode(collectionName, replicaName, replica, core);
- if (waitForCoreNodeGone(collectionName, shard, replicaName, 30000)) return;
-
- throw new SolrException(ErrorCode.SERVER_ERROR,
- "Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName);
+ // try and ensure core info is removed from cluster state
+ deleteCoreNode(collectionName, replicaName, replica, core);
+ if (waitForCoreNodeGone(collectionName, shard, replicaName, 30000)) return Boolean.TRUE;
+ return Boolean.FALSE;
+ } catch (Exception e) {
+ results.add("failure", "Could not complete delete " + e.getMessage());
+ throw e;
+ } finally {
+ if (onComplete != null) onComplete.run();
+ }
+ };
+
+ if (!parallel) {
+ try {
+ if (!callable.call())
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "Could not remove replica : " + collectionName + "/" + shard + "/" + replicaName);
+ } catch (InterruptedException | KeeperException e) {
+ throw e;
+ } catch (Exception ex) {
+ throw new SolrException(ErrorCode.UNKNOWN, "Error waiting for corenode gone", ex);
+ }
+
+ } else {
+ tpe.submit(callable);
+ }
}
private boolean waitForCoreNodeGone(String collectionName, String shard, String replicaName, int timeoutms) throws InterruptedException {
@@ -679,7 +729,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(m));
}
- private void checkRequired(ZkNodeProps message, String... props) {
+ void checkRequired(ZkNodeProps message, String... props) {
for (String prop : props) {
if(message.get(prop) == null){
throw new SolrException(ErrorCode.BAD_REQUEST, StrUtils.join(Arrays.asList(props),',') +" are required params" );
@@ -1137,7 +1187,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
if (asyncId != null) {
propMap.put(ASYNC, asyncId);
}
- addReplica(clusterState, new ZkNodeProps(propMap), results);
+ addReplica(clusterState, new ZkNodeProps(propMap), results, null);
}
processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard leaders", asyncId, requestMap);
@@ -1307,7 +1357,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
// now actually create replica cores on sub shard nodes
for (Map<String, Object> replica : replicas) {
- addReplica(clusterState, new ZkNodeProps(replica), results);
+ addReplica(clusterState, new ZkNodeProps(replica), results, null);
}
processResponses(results, shardHandler, true, "SPLITSHARD failed to create subshard replicas", asyncId, requestMap);
@@ -1681,7 +1731,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
if(asyncId != null) {
props.put(ASYNC, asyncId);
}
- addReplica(clusterState, new ZkNodeProps(props), results);
+ addReplica(clusterState, new ZkNodeProps(props), results, null);
processResponses(results, shardHandler, true, "MIGRATE failed to create replica of " +
"temporary collection in target leader node.", asyncId, requestMap);
@@ -2110,12 +2160,14 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
}
- private void addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results)
+ ZkNodeProps addReplica(ClusterState clusterState, ZkNodeProps message, NamedList results, Runnable onComplete)
throws KeeperException, InterruptedException {
+ log.info("addReplica() : {}", Utils.toJSONString(message));
String collection = message.getStr(COLLECTION_PROP);
String node = message.getStr(CoreAdminParams.NODE);
String shard = message.getStr(SHARD_ID_PROP);
String coreName = message.getStr(CoreAdminParams.NAME);
+ boolean parallel = message.getBool("parallel", false);
if (StringUtils.isBlank(coreName)) {
coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
}
@@ -2138,7 +2190,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
node = getNodesForNewReplicas(clusterState, collection, shard, 1, node,
overseer.getZkController().getCoreContainer()).get(0).nodeName;
}
- log.info("Node not provided, Identified {} for creating new replica", node);
+ log.info("Node Identified {} for creating new replica", node);
if (!clusterState.liveNodesContain(node)) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
@@ -2161,10 +2213,14 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
if (!Overseer.isLegacy(zkStateReader)) {
if (!skipCreateReplicaInClusterState) {
- ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), ZkStateReader.COLLECTION_PROP,
- collection, ZkStateReader.SHARD_ID_PROP, shard, ZkStateReader.CORE_NAME_PROP, coreName,
- ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), ZkStateReader.BASE_URL_PROP,
- zkStateReader.getBaseUrlForNodeName(node), ZkStateReader.NODE_NAME_PROP, node);
+ ZkNodeProps props = new ZkNodeProps(
+ Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(),
+ ZkStateReader.COLLECTION_PROP, collection,
+ ZkStateReader.SHARD_ID_PROP, shard,
+ ZkStateReader.CORE_NAME_PROP, coreName,
+ ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(),
+ ZkStateReader.BASE_URL_PROP, zkStateReader.getBaseUrlForNodeName(node),
+ ZkStateReader.NODE_NAME_PROP, node);
Overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(props));
}
params.set(CoreAdminParams.CORE_NODE_NAME,
@@ -2204,9 +2260,28 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
Map<String,String> requestMap = new HashMap<>();
sendShardRequest(node, params, shardHandler, asyncId, requestMap);
- processResponses(results, shardHandler, true, "ADDREPLICA failed to create replica", asyncId, requestMap);
+ final String fnode = node;
+ final String fcoreName = coreName;
+
+ Runnable runnable = () -> {
+ processResponses(results, shardHandler, true, "ADDREPLICA failed to create replica", asyncId, requestMap);
+ waitForCoreNodeName(collection, fnode, fcoreName);
+ if (onComplete != null) onComplete.run();
+ };
- waitForCoreNodeName(collection, node, coreName);
+ if (!parallel) {
+ runnable.run();
+ } else {
+ tpe.submit(runnable);
+ }
+
+
+ return new ZkNodeProps(
+ ZkStateReader.COLLECTION_PROP, collection,
+ ZkStateReader.SHARD_ID_PROP, shard,
+ ZkStateReader.CORE_NAME_PROP, coreName,
+ ZkStateReader.NODE_NAME_PROP, node
+ );
}
private void processBackupAction(ZkNodeProps message, NamedList results) throws IOException, KeeperException, InterruptedException {
@@ -2394,7 +2469,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
addPropertyParams(message, propMap);
- addReplica(clusterState, new ZkNodeProps(propMap), new NamedList());
+ addReplica(clusterState, new ZkNodeProps(propMap), new NamedList(), null);
}
//refresh the location copy of collection state
@@ -2443,7 +2518,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
addPropertyParams(message, propMap);
- addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results);
+ addReplica(zkStateReader.getClusterState(), new ZkNodeProps(propMap), results, null);
}
}
}
@@ -2503,7 +2578,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
return configName;
}
-
+
private void validateConfigOrThrowSolrException(String configName) throws KeeperException, InterruptedException {
boolean isValid = zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);
if(!isValid) {
@@ -2723,4 +2798,19 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
);
}
+ @Override
+ public void close() throws IOException {
+ if (tpe != null) {
+ if (!tpe.isShutdown()) {
+ ExecutorUtil.shutdownAndAwaitTermination(tpe);
+ }
+ }
+ }
+
+ interface Cmd {
+
+ Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception;
+
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
index e3bc1f3..074cf16 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit;
import java.util.function.Predicate;
import com.google.common.collect.ImmutableSet;
+import org.apache.commons.io.IOUtils;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.Overseer.LeaderStatus;
@@ -115,7 +116,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
private final Object waitLock = new Object();
- private OverseerMessageHandlerSelector selector;
+ protected OverseerMessageHandlerSelector selector;
private OverseerNodePrioritizer prioritizer;
@@ -328,6 +329,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
ExecutorUtil.shutdownAndAwaitTermination(tpe);
}
}
+ IOUtils.closeQuietly(selector);
}
public static List<String> getSortedOverseerNodeNames(SolrZkClient zk) throws KeeperException, InterruptedException {
@@ -588,7 +590,7 @@ public class OverseerTaskProcessor implements Runnable, Closeable {
* messages only) , or a different handler could be selected based on the
* contents of the message.
*/
- public interface OverseerMessageHandlerSelector {
+ public interface OverseerMessageHandlerSelector extends Closeable {
OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java
new file mode 100644
index 0000000..0cfd089
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/cloud/ReplaceNodeCmd.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CoreAdminParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
+import static org.apache.solr.common.cloud.ZkStateReader.SHARD_ID_PROP;
+
+public class ReplaceNodeCmd implements OverseerCollectionMessageHandler.Cmd {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final OverseerCollectionMessageHandler ocmh;
+
+ public ReplaceNodeCmd(OverseerCollectionMessageHandler ocmh) {
+ this.ocmh = ocmh;
+ }
+
+ @Override
+ public Object call(ClusterState state, ZkNodeProps message, NamedList results) throws Exception {
+ ZkStateReader zkStateReader = ocmh.zkStateReader;
+ ocmh.checkRequired(message, "source", "target");
+ String source = message.getStr("source");
+ String target = message.getStr("target");
+ boolean parallel = message.getBool("parallel", false);
+ ClusterState clusterState = zkStateReader.getClusterState();
+
+ if (!clusterState.liveNodesContain(source)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Source Node: " + source + " is not live");
+ }
+ if (!clusterState.liveNodesContain(target)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Target Node: " + target + " is not live");
+ }
+ List<ZkNodeProps> sourceReplicas = getReplicasOfNode(source, clusterState);
+
+ List<ZkNodeProps> createdReplicas = new ArrayList<>();
+
+ AtomicBoolean anyOneFailed = new AtomicBoolean(false);
+ CountDownLatch countDownLatch = new CountDownLatch(sourceReplicas.size());
+
+ for (ZkNodeProps sourceReplica : sourceReplicas) {
+ NamedList nl = new NamedList();
+ log.info("going to create replica {}", Utils.toJSONString(sourceReplica));
+ ZkNodeProps msg = sourceReplica.plus("parallel", String.valueOf(parallel)).plus(CoreAdminParams.NODE, target);
+ final ZkNodeProps addedReplica = ocmh.addReplica(clusterState,
+ msg, nl, () -> {
+ countDownLatch.countDown();
+ if (nl.get("failure") != null) {
+ log.warn("failed to create : " + Utils.toJSONString(msg));
+ // one replica creation failed. Make the best attempt to
+ // delete all the replicas created so far in the target
+ // and exit
+ synchronized (results) {
+ results.add("failure", "Could not create copy of replica " + Utils.toJSONString(sourceReplica));
+ anyOneFailed.set(true);
+ }
+ } else {
+ log.info("successfully created : " + Utils.toJSONString(msg));
+
+ }
+ });
+
+ if (addedReplica != null) {
+ createdReplicas.add(addedReplica);
+ }
+ }
+
+ log.info("Waiting for creates to complete ");
+ countDownLatch.await(5, TimeUnit.MINUTES);
+ log.info("Waiting over for creates to complete ");
+
+ if (anyOneFailed.get()) {
+ log.info("failed to create some cores delete all " + Utils.toJSONString(createdReplicas));
+ CountDownLatch cleanupLatch = new CountDownLatch(createdReplicas.size());
+ for (ZkNodeProps createdReplica : createdReplicas) {
+ NamedList deleteResult = new NamedList();
+ try {
+ ocmh.deleteReplica(zkStateReader.getClusterState(), createdReplica.plus("parallel", "true"), deleteResult, () -> {
+ cleanupLatch.countDown();
+ if (deleteResult.get("failure") != null) {
+ synchronized (results) {
+ results.add("failure", "could not cleanup, because : " + deleteResult.get("failure") + " " + Utils.toJSONString(createdReplica));
+ }
+ }
+ });
+ } catch (KeeperException e) {
+ cleanupLatch.countDown();
+ log.info("Error deleting ", e);
+ } catch (Exception e) {
+ log.error("Unknown Error deleteing", e);
+ cleanupLatch.countDown();
+ throw e;
+ }
+ }
+ cleanupLatch.await(5, TimeUnit.MINUTES);
+ return null;
+ }
+
+
+ // we have reached this far means all replicas could be recreated
+ //now cleanup the replicas in the source node
+ DeleteNodeCmd.cleanupReplicas(results, state, sourceReplicas, ocmh);
+ results.add("success", "REPLACENODE completed successfully from : " + source + " to : " + target);
+ return null;
+ }
+
+ static List<ZkNodeProps> getReplicasOfNode(String source, ClusterState state) {
+ List<ZkNodeProps> sourceReplicas = new ArrayList<>();
+ for (Map.Entry<String, DocCollection> e : state.getCollectionsMap().entrySet()) {
+ for (Slice slice : e.getValue().getSlices()) {
+ for (Replica replica : slice.getReplicas()) {
+ if (source.equals(replica.getNodeName())) {
+ ZkNodeProps props = new ZkNodeProps(
+ COLLECTION_PROP, e.getKey(),
+ SHARD_ID_PROP, slice.getName(),
+ ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
+ ZkStateReader.REPLICA_PROP, replica.getName(),
+ CoreAdminParams.NODE, source);
+ log.info("src_core : {}", Utils.toJSONString(props));
+ sourceReplicas.add(props
+ );
+ }
+ }
+ }
+ }
+ return sourceReplicas;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index cb72790..2e906ae 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -781,7 +781,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
req.getParams().getAll(params, COLL_CONF, REPLICATION_FACTOR, MAX_SHARDS_PER_NODE, STATE_FORMAT, AUTO_ADD_REPLICAS);
copyPropertiesWithPrefix(req.getParams(), params, COLL_PROP_PREFIX);
return params;
- });
+ }),
+
+ REPLACENODE_OP(REPLACENODE, (req, rsp, h) -> req.getParams().required().getAll(req.getParams().getAll(null, "parallel"), "source", "target")),
+ DELETENODE_OP(DELETENODE, (req, rsp, h) -> req.getParams().required().getAll(null, "node"));
public final CollectionOp fun;
CollectionAction action;
long timeOut;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
new file mode 100644
index 0000000..8d2f6f2
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/DeleteNodeTest.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.common.util.StrUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class DeleteNodeTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(6)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
+ .configure();
+ }
+
+ protected String getSolrXml() {
+ return "solr.xml";
+ }
+
+ @Test
+ public void test() throws Exception {
+ cluster.waitForAllNodes(5000);
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ String coll = "deletenodetest_coll";
+ Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
+ ArrayList<String> l = new ArrayList<>(liveNodes);
+ Collections.shuffle(l, random());
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 5, 2);
+ create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
+ cloudClient.request(create);
+ String node2bdecommissioned = l.get(0);
+ new CollectionAdminRequest.DeleteNode(node2bdecommissioned).processAsync("003", cloudClient);
+ CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("003");
+ boolean success = false;
+ for (int i = 0; i < 200; i++) {
+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
+ if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
+ success = true;
+ break;
+ }
+ assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED);
+ Thread.sleep(50);
+ }
+ assertTrue(success);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
new file mode 100644
index 0000000..1c7575d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud;
+
+
+import java.lang.invoke.MethodHandles;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.CoreAdminRequest;
+import org.apache.solr.client.solrj.response.CoreAdminResponse;
+import org.apache.solr.client.solrj.response.RequestStatusState;
+import org.apache.solr.common.util.StrUtils;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ReplaceNodeTest extends SolrCloudTestCase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(6)
+ .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
+ .configure();
+ }
+
+ protected String getSolrXml() {
+ return "solr.xml";
+ }
+
+ @Test
+ public void test() throws Exception {
+ cluster.waitForAllNodes(5000);
+ String coll = "replacenodetest_coll";
+ log.info("total_jettys: " + cluster.getJettySolrRunners().size());
+
+ CloudSolrClient cloudClient = cluster.getSolrClient();
+ Set<String> liveNodes = cloudClient.getZkStateReader().getClusterState().getLiveNodes();
+ ArrayList<String> l = new ArrayList<>(liveNodes);
+ Collections.shuffle(l, random());
+ String emptyNode = l.remove(0);
+ String node2bdecommissioned = l.get(0);
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 5, 2);
+ create.setCreateNodeSet(StrUtils.join(l, ',')).setMaxShardsPerNode(3);
+ cloudClient.request(create);
+ log.info("excluded_node : {} ", emptyNode);
+ new CollectionAdminRequest.ReplaceNode(node2bdecommissioned, emptyNode).processAsync("000", cloudClient);
+ CollectionAdminRequest.RequestStatus requestStatus = CollectionAdminRequest.requestStatus("000");
+ boolean success = false;
+ for (int i = 0; i < 200; i++) {
+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
+ if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
+ success = true;
+ break;
+ }
+ assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED);
+ Thread.sleep(50);
+ }
+ assertTrue(success);
+ try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(node2bdecommissioned))) {
+ CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
+ assertTrue(status.getCoreStatus().size() == 0);
+ }
+
+ //let's do it back
+ new CollectionAdminRequest.ReplaceNode(emptyNode, node2bdecommissioned).setParallel(Boolean.TRUE).processAsync("001", cloudClient);
+ requestStatus = CollectionAdminRequest.requestStatus("001");
+
+ for (int i = 0; i < 200; i++) {
+ CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient);
+ if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) {
+ success = true;
+ break;
+ }
+ assertFalse(rsp.getRequestStatus() == RequestStatusState.FAILED);
+ Thread.sleep(50);
+ }
+ assertTrue(success);
+ try (HttpSolrClient coreclient = getHttpSolrClient(cloudClient.getZkStateReader().getBaseUrlForNodeName(emptyNode))) {
+ CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient);
+ assertTrue(status.getCoreStatus().size() == 0);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 7bc9e4f..0a0a191 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -112,7 +112,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
* @deprecated Use {@link #processAsync(String, SolrClient)} or {@link #processAsync(SolrClient)}
*/
@Deprecated
- public abstract AsyncCollectionAdminRequest setAsyncId(String id);
+ public AsyncCollectionAdminRequest setAsyncId(String id){return this;};
/**
* Process this request asynchronously, generating and returning a request id
@@ -491,6 +491,56 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
}
}
+ public static class DeleteNode extends AsyncCollectionAdminRequest {
+ String node;
+
+ /**
+ * @param node The node to be deleted
+ */
+ public DeleteNode(String node) {
+ super(CollectionAction.DELETENODE);
+ this.node = node;
+ }
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("node", node);
+ return params;
+ }
+
+
+ }
+
+ public static class ReplaceNode extends AsyncCollectionAdminRequest {
+ String source, target;
+ Boolean parallel;
+
+ /**
+ * @param source node to be cleaned up
+ * @param target node where the new replicas are to be created
+ */
+ public ReplaceNode(String source, String target) {
+ super(CollectionAction.REPLACENODE);
+ this.source = source;
+ this.target = target;
+ }
+
+ public ReplaceNode setParallel(Boolean flag) {
+ this.parallel = flag;
+ return this;
+ }
+
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("source", source);
+ params.set("target", target);
+ if (parallel != null) params.set("parallel", parallel.toString());
+ return params;
+ }
+
+ }
+
/*
* Returns a RebalanceLeaders object to rebalance leaders for a collection
*/
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
index 320ad13..94a673e 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkNodeProps.java
@@ -20,6 +20,7 @@ import org.apache.solr.common.util.Utils;
import org.noggit.JSONUtil;
import org.noggit.JSONWriter;
+import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Set;
@@ -40,6 +41,17 @@ public class ZkNodeProps implements JSONWriter.Writable {
// Always wrapping introduces a memory leak.
}
+ public ZkNodeProps plus(String key , Object val) {
+ return plus(Collections.singletonMap(key,val));
+ }
+
+ public ZkNodeProps plus(Map<String, Object> newVals) {
+ LinkedHashMap<String, Object> copy = new LinkedHashMap<>(propMap);
+ if (newVals == null || newVals.isEmpty()) return new ZkNodeProps(copy);
+ copy.putAll(newVals);
+ return new ZkNodeProps(copy);
+ }
+
/**
* Constructor that populates the from array of Strings in form key1, value1,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ae60c74f/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
index e38ab4f..f10f089 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
@@ -96,6 +96,9 @@ public interface CollectionParams {
// but the overseer is aware of these tasks
MOCK_COLL_TASK(false, LockLevel.COLLECTION),
MOCK_SHARD_TASK(false, LockLevel.SHARD),
+ //TODO when we have a node level lock use it here
+ REPLACENODE(true, LockLevel.NONE),
+ DELETENODE(true, LockLevel.NONE),
MOCK_REPLICA_TASK(false, LockLevel.REPLICA)
;
public final boolean isWrite;
[2/2] lucene-solr:master: Merge remote-tracking branch 'origin/master'
Posted by no...@apache.org.
Merge remote-tracking branch 'origin/master'
Conflicts:
solr/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/92b5a76b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/92b5a76b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/92b5a76b
Branch: refs/heads/master
Commit: 92b5a76b543087a219d3d5dbcd7c93f84edafbf7
Parents: ae60c74 bc25a56
Author: Noble Paul <no...@apache.org>
Authored: Thu Aug 11 12:13:38 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Thu Aug 11 12:13:38 2016 +0530
----------------------------------------------------------------------
lucene/CHANGES.txt | 16 +-
.../lucene/analysis/minhash/MinHashFilter.java | 1 +
.../icu/src/data/uax29/MyanmarSyllable.rbbi | 50 ++
.../segmentation/DefaultICUTokenizerConfig.java | 14 +-
.../analysis/icu/segmentation/ICUTokenizer.java | 2 +-
.../icu/segmentation/ICUTokenizerFactory.java | 6 +-
.../icu/segmentation/MyanmarSyllable.brk | Bin 0 -> 19776 bytes
.../icu/segmentation/TestICUTokenizer.java | 6 +-
.../icu/segmentation/TestICUTokenizerCJK.java | 2 +-
.../icu/segmentation/TestMyanmarSyllable.java | 156 +++++
.../segmentation/TestWithCJKBigramFilter.java | 4 +-
.../classification/utils/DatasetSplitter.java | 14 +-
.../simpletext/SimpleTextPointsWriter.java | 4 +-
.../lucene/codecs/MutablePointsReader.java | 41 ++
.../org/apache/lucene/codecs/PointsWriter.java | 11 +-
.../codecs/lucene60/Lucene60PointsWriter.java | 24 +-
.../java/org/apache/lucene/document/Field.java | 6 +-
.../lucene/index/DocumentsWriterPerThread.java | 2 +-
.../org/apache/lucene/index/IndexWriter.java | 2 +-
.../lucene/index/LiveIndexWriterConfig.java | 19 +-
.../apache/lucene/index/PointValuesWriter.java | 172 +++--
.../org/apache/lucene/search/LRUQueryCache.java | 1 +
.../org/apache/lucene/search/PrefixQuery.java | 5 +-
.../java/org/apache/lucene/util/ArrayUtil.java | 71 +-
.../org/apache/lucene/util/ByteBlockPool.java | 29 +
.../apache/lucene/util/InPlaceMergeSorter.java | 4 +-
.../org/apache/lucene/util/IntroSelector.java | 128 ++++
.../org/apache/lucene/util/IntroSorter.java | 20 +-
.../org/apache/lucene/util/MSBRadixSorter.java | 109 ++-
.../org/apache/lucene/util/RadixSelector.java | 278 ++++++++
.../java/org/apache/lucene/util/Selector.java | 41 ++
.../src/java/org/apache/lucene/util/Sorter.java | 59 +-
.../apache/lucene/util/automaton/Automaton.java | 4 +-
.../org/apache/lucene/util/bkd/BKDWriter.java | 491 ++++++++++----
.../util/bkd/MutablePointsReaderUtils.java | 186 ++++++
.../lucene60/TestLucene60PointsFormat.java | 5 +-
.../apache/lucene/search/TestPointQueries.java | 5 +-
.../apache/lucene/util/TestByteBlockPool.java | 21 +-
.../apache/lucene/util/TestIntroSelector.java | 86 +++
.../apache/lucene/util/TestMSBRadixSorter.java | 68 ++
.../apache/lucene/util/TestRadixSelector.java | 106 +++
.../util/bkd/TestMutablePointsReaderUtils.java | 270 ++++++++
lucene/ivy-versions.properties | 1 +
.../valuesource/ComparisonBoolFunction.java | 105 +++
.../lucene/replicator/nrt/FileMetaData.java | 5 +
.../lucene/replicator/nrt/SimpleCopyJob.java | 2 +
.../apache/lucene/spatial3d/geom/Bounds.java | 7 +
.../lucene/spatial3d/geom/LatLonBounds.java | 5 +
.../org/apache/lucene/spatial3d/geom/Plane.java | 101 +--
.../apache/lucene/spatial3d/geom/XYZBounds.java | 11 +
.../apache/lucene/spatial3d/TestGeo3DPoint.java | 5 +-
.../lucene/spatial3d/geom/GeoBBoxTest.java | 15 +
.../lucene/spatial3d/geom/GeoCircleTest.java | 14 +
.../codecs/asserting/AssertingPointsFormat.java | 5 +-
.../codecs/cranky/CrankyPointsFormat.java | 4 +-
.../apache/lucene/geo/BaseGeoPointTestCase.java | 3 +-
.../org/apache/lucene/index/RandomCodec.java | 9 +-
.../lucene/store/MockDirectoryWrapper.java | 2 +-
.../lucene/store/MockIndexInputWrapper.java | 27 +-
.../store/SlowClosingMockIndexInputWrapper.java | 2 +-
.../store/SlowOpeningMockIndexInputWrapper.java | 2 +-
.../lucene/store/TestMockDirectoryWrapper.java | 36 +
solr/CHANGES.txt | 58 ++
.../accumulator/FacetingAccumulator.java | 2 +-
.../solr/handler/dataimport/JdbcDataSource.java | 6 +-
.../handler/dataimport/TestJdbcDataSource.java | 39 ++
.../apache/solr/hadoop/ZooKeeperInspector.java | 21 +-
solr/core/ivy.xml | 4 +
.../solr/core/IndexDeletionPolicyWrapper.java | 27 +-
.../java/org/apache/solr/core/SolrConfig.java | 29 +-
.../src/java/org/apache/solr/core/SolrCore.java | 36 +-
.../repository/LocalFileSystemRepository.java | 15 +-
.../core/snapshots/SolrSnapshotManager.java | 134 ++++
.../snapshots/SolrSnapshotMetaDataManager.java | 416 ++++++++++++
.../solr/core/snapshots/package-info.java | 22 +
.../org/apache/solr/handler/BlobHandler.java | 2 +-
.../org/apache/solr/handler/CdcrParams.java | 10 +-
.../org/apache/solr/handler/CdcrReplicator.java | 8 +-
.../solr/handler/CdcrReplicatorManager.java | 242 ++++++-
.../solr/handler/CdcrReplicatorScheduler.java | 6 +-
.../solr/handler/CdcrReplicatorState.java | 23 +
.../apache/solr/handler/CdcrRequestHandler.java | 233 ++++++-
.../org/apache/solr/handler/IndexFetcher.java | 26 +-
.../solr/handler/MoreLikeThisHandler.java | 2 +-
.../apache/solr/handler/ReplicationHandler.java | 41 +-
.../apache/solr/handler/RequestHandlerBase.java | 26 +-
.../org/apache/solr/handler/RestoreCore.java | 18 +-
.../org/apache/solr/handler/SchemaHandler.java | 2 +-
.../org/apache/solr/handler/SnapShooter.java | 39 +-
.../org/apache/solr/handler/StreamHandler.java | 2 +
.../solr/handler/admin/CollectionsHandler.java | 8 +-
.../solr/handler/admin/CoreAdminOperation.java | 125 +++-
.../solr/handler/component/ExpandComponent.java | 4 +-
.../solr/handler/component/QueryComponent.java | 2 +-
.../handler/component/RealTimeGetComponent.java | 49 +-
.../handler/component/SpellCheckComponent.java | 3 +-
.../org/apache/solr/request/SimpleFacets.java | 2 +-
.../transform/ChildDocTransformerFactory.java | 4 +-
.../transform/SubQueryAugmenterFactory.java | 17 +
.../org/apache/solr/schema/IndexSchema.java | 6 +-
.../org/apache/solr/search/CacheConfig.java | 24 +-
.../java/org/apache/solr/search/Grouping.java | 2 +-
.../solr/search/IGainTermsQParserPlugin.java | 240 +++++++
.../apache/solr/search/JoinQParserPlugin.java | 2 +-
.../java/org/apache/solr/search/LRUCache.java | 34 +-
.../java/org/apache/solr/search/QParser.java | 11 +
.../org/apache/solr/search/QParserPlugin.java | 12 +-
.../apache/solr/search/ReRankQParserPlugin.java | 108 ++-
.../apache/solr/search/SolrIndexSearcher.java | 69 +-
.../TextLogisticRegressionQParserPlugin.java | 283 ++++++++
.../apache/solr/search/ValueSourceParser.java | 52 ++
.../apache/solr/search/facet/FacetField.java | 10 +-
.../solr/search/facet/FacetProcessor.java | 2 +-
.../apache/solr/search/facet/FacetRequest.java | 2 +-
.../function/SolrComparisonBoolFunction.java | 58 ++
.../distributed/command/QueryCommand.java | 2 +-
.../search/join/ScoreJoinQParserPlugin.java | 2 +-
.../solr/security/AuthenticationPlugin.java | 29 +-
.../apache/solr/security/BasicAuthPlugin.java | 5 +-
.../security/DelegationTokenKerberosFilter.java | 215 ++++++
.../apache/solr/security/KerberosFilter.java | 14 +
.../apache/solr/security/KerberosPlugin.java | 288 ++++++--
.../solr/security/PKIAuthenticationPlugin.java | 13 +-
.../solr/security/PrintWriterWrapper.java | 215 ++++++
.../apache/solr/servlet/SolrDispatchFilter.java | 12 +-
.../org/apache/solr/update/CdcrUpdateLog.java | 7 +-
.../solr/update/DefaultSolrCoreState.java | 6 +-
.../solr/update/DirectUpdateHandler2.java | 117 ++--
.../org/apache/solr/update/SolrCoreState.java | 2 +
.../update/processor/CdcrUpdateProcessor.java | 10 +-
.../processor/DistributedUpdateProcessor.java | 2 +-
.../apache/solr/util/ConcurrentLRUCache.java | 33 +-
.../org/apache/solr/util/SolrPluginUtils.java | 4 +-
.../org/apache/solr/util/TestInjection.java | 95 ++-
.../collection1/conf/schema-psuedo-fields.xml | 3 +
.../configsets/cdcr-source-disabled/schema.xml | 29 +
.../cdcr-source-disabled/solrconfig.xml | 60 ++
.../solr/configsets/cdcr-source/schema.xml | 29 +
.../solr/configsets/cdcr-source/solrconfig.xml | 76 +++
.../solr/configsets/cdcr-target/schema.xml | 29 +
.../solr/configsets/cdcr-target/solrconfig.xml | 63 ++
.../TestReversedWildcardFilterFactory.java | 2 +-
.../solr/cloud/BaseCdcrDistributedZkTest.java | 25 +
.../apache/solr/cloud/CdcrBootstrapTest.java | 396 +++++++++++
.../cloud/CdcrReplicationDistributedZkTest.java | 31 +
.../solr/cloud/ChaosMonkeyShardSplitTest.java | 13 +-
.../apache/solr/cloud/KerberosTestServices.java | 229 +++++++
.../org/apache/solr/cloud/KerberosTestUtil.java | 147 -----
.../apache/solr/cloud/LeaderElectionTest.java | 1 +
...utOfBoxZkACLAndCredentialsProvidersTest.java | 7 +-
...rriddenZkACLAndCredentialsProvidersTest.java | 71 +-
.../org/apache/solr/cloud/OverseerTest.java | 1 +
.../solr/cloud/SaslZkACLProviderTest.java | 39 +-
.../solr/cloud/TestAuthenticationFramework.java | 10 +-
.../solr/cloud/TestCloudPseudoReturnFields.java | 91 +--
.../cloud/TestMiniSolrCloudClusterKerberos.java | 29 +-
.../apache/solr/cloud/TestRandomFlRTGCloud.java | 447 +++++++++++--
.../TestSolrCloudWithDelegationTokens.java | 402 ++++++++++++
.../cloud/TestSolrCloudWithKerberosAlt.java | 37 +-
.../TestSolrCloudWithSecureImpersonation.java | 357 ++++++++++
.../TestStressCloudBlindAtomicUpdates.java | 25 +-
...MParamsZkACLAndCredentialsProvidersTest.java | 25 +-
.../solr/cloud/overseer/ZkStateWriterTest.java | 393 +++++------
.../apache/solr/core/TestSolrConfigHandler.java | 56 +-
.../core/snapshots/TestSolrCoreSnapshots.java | 419 ++++++++++++
.../apache/solr/handler/BackupRestoreUtils.java | 37 ++
.../solr/handler/TestHdfsBackupRestoreCore.java | 46 +-
.../solr/handler/TestReplicationHandler.java | 20 +-
.../apache/solr/search/QueryEqualityTest.java | 45 +-
...OverriddenPrefixQueryForCustomFieldType.java | 2 +-
.../solr/search/TestPseudoReturnFields.java | 25 +-
.../org/apache/solr/search/TestSearchPerf.java | 8 +-
.../apache/solr/search/TestSolrQueryParser.java | 4 +-
.../solr/search/TestStandardQParsers.java | 9 +
.../solr/search/function/TestFunctionQuery.java | 69 +-
.../search/join/TestScoreJoinQPNoScore.java | 4 +-
.../solr/search/join/TestScoreJoinQPScore.java | 2 +-
.../HttpParamDelegationTokenPlugin.java | 272 ++++++++
.../solr/security/MockAuthenticationPlugin.java | 32 +-
.../solr/update/DirectUpdateHandlerTest.java | 48 +-
.../org/apache/solr/util/TestTestInjection.java | 4 +
solr/licenses/curator-recipes-2.8.0.jar.sha1 | 1 +
solr/licenses/curator-recipes-LICENSE-ASL.txt | 202 ++++++
solr/licenses/curator-recipes-NOTICE.txt | 5 +
solr/solrj/ivy.xml | 5 +
.../solr/client/solrj/impl/HttpSolrClient.java | 57 +-
.../solrj/impl/Krb5HttpClientBuilder.java | 18 +-
.../solrj/io/ClassificationEvaluation.java | 85 +++
.../io/stream/FeaturesSelectionStream.java | 436 ++++++++++++
.../client/solrj/io/stream/TextLogitStream.java | 656 +++++++++++++++++++
.../solrj/io/stream/expr/Explanation.java | 1 +
.../client/solrj/request/CoreAdminRequest.java | 57 ++
.../solrj/request/DelegationTokenRequest.java | 152 +++++
.../solrj/response/DelegationTokenResponse.java | 108 +++
.../solr/common/cloud/ConnectionManager.java | 15 +-
.../common/cloud/DefaultConnectionStrategy.java | 4 +-
.../solr/common/cloud/SaslZkACLProvider.java | 21 +-
.../cloud/SecurityAwareZkACLProvider.java | 79 +++
.../apache/solr/common/cloud/SolrZkClient.java | 2 +-
...ParamsAllAndReadonlyDigestZkACLProvider.java | 52 +-
.../cloud/ZkClientConnectionStrategy.java | 4 +-
.../apache/solr/common/cloud/ZkStateReader.java | 29 +-
.../solr/common/params/CoreAdminParams.java | 10 +-
.../solrj/solr/configsets/ml/conf/schema.xml | 77 +++
.../solr/configsets/ml/conf/solrconfig.xml | 51 ++
.../solrj/io/stream/StreamExpressionTest.java | 180 ++++-
.../stream/StreamExpressionToExpessionTest.java | 37 +-
.../StreamExpressionToExplanationTest.java | 1 -
.../request/TestDelegationTokenRequest.java | 70 ++
.../response/TestDelegationTokenResponse.java | 138 ++++
solr/webapp/web/css/angular/cloud.css | 14 +
solr/webapp/web/js/angular/app.js | 2 +-
solr/webapp/web/js/angular/controllers/cloud.js | 5 +
.../web/js/angular/controllers/dataimport.js | 18 +-
solr/webapp/web/js/angular/controllers/query.js | 10 +-
solr/webapp/web/partials/cloud.html | 3 +-
solr/webapp/web/partials/dataimport.html | 8 +-
217 files changed, 11744 insertions(+), 1627 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b5a76b/solr/CHANGES.txt
----------------------------------------------------------------------
diff --cc solr/CHANGES.txt
index 889611f,6755b70..518f63a
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@@ -100,11 -100,25 +100,30 @@@ New Feature
* SOLR-9275: XML QueryParser support (defType=xmlparser) now extensible via configuration.
(Christine Poerschke)
+ * SOLR-9200: Add Delegation Token Support to Solr.
+ (Gregory Chanan)
+
+ * SOLR-9038: Solr core snapshots: The current commit can be snapshotted which retains the commit and associates it with
+ a name. The core admin API can create snapshots, list them, and delete them. Snapshot names can be referenced in
+ doing a core backup, and in replication. Snapshot metadata is stored in a new snapshot_metadata/ dir.
+ (Hrishikesh Gadre via David Smiley)
+
+ * SOLR-9279: New boolean comparison function queries comparing numeric arguments: gt, gte, lt, lte, eq
+ (Doug Turnbull, David Smiley)
+
+ * SOLR-9324: Support Secure Impersonation / Proxy User for solr authentication
+ (Gregory Chanan)
+
+ * SOLR-9252: Feature selection and logistic regression on text (Cao Manh Dat, Joel Bernstein)
+
+ * SOLR-6465: CDCR: fall back to whole-index replication when tlogs are insufficient.
+ (Noble Paul, Renaud Delbru, shalin)
+
+* SOLR-9320: A REPLACENODE command to decommission an existing node with another new node
+ (noble, Nitin Sharma, Varun Thacker)
+
+* SOLR-9318: A DELETENODE command to delete all replicas in that node (noble, Nitin Sharma, Varun Thacker)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/92b5a76b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------