You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by ge...@apache.org on 2022/11/28 13:52:23 UTC
[solr] branch branch_9x updated: SOLR-11028: Created V2 equivalent of REPLACENODE
This is an automated email from the ASF dual-hosted git repository.
gerlowskija pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new 720cd5ddbf1 SOLR-11028: Created V2 equivalent of REPLACENODE
720cd5ddbf1 is described below
commit 720cd5ddbf1cc312b0c97b43ec9448dd1cc2fc78
Author: Joshua Ouma <jo...@gmail.com>
AuthorDate: Thu Nov 10 19:37:51 2022 +0300
SOLR-11028: Created V2 equivalent of REPLACENODE
---
solr/CHANGES.txt | 3 +
.../solr/cloud/api/collections/ReplaceNodeCmd.java | 6 +-
.../solr/handler/admin/CollectionsHandler.java | 27 ++--
.../solr/handler/admin/api/ReplaceNodeAPI.java | 144 +++++++++++++++++++++
.../org/apache/solr/cloud/ReplaceNodeTest.java | 7 +-
.../solr/handler/admin/api/ReplaceNodeAPITest.java | 125 ++++++++++++++++++
.../pages/cluster-node-management.adoc | 11 +-
7 files changed, 307 insertions(+), 16 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 054ceb16d7d..655c6d59343 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -44,6 +44,9 @@ Improvements
* SOLR-16420: Introducing `{!mlt_content}foo bar` to cover existing `/mlt` handler functionality for SolrCloud.
(Mikhail Khludnev)
+* SOLR-11028: A v2 equivalent of the `/admin/collections?action= REPLACE` command is now available at
+ `POST /api/cluster/nodes/nodeName/replace`. (Joshua Ouma via Jason Gerlowski)
+
Optimizations
---------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
index 5f68204c071..52cba5b0fc1 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/ReplaceNodeCmd.java
@@ -63,14 +63,14 @@ public class ReplaceNodeCmd implements CollApiCmds.CollectionApiCommand {
public void call(ClusterState state, ZkNodeProps message, NamedList<Object> results)
throws Exception {
ZkStateReader zkStateReader = ccc.getZkStateReader();
- String source = message.getStr(CollectionParams.SOURCE_NODE, message.getStr("source"));
- String target = message.getStr(CollectionParams.TARGET_NODE, message.getStr("target"));
+ String source = message.getStr(CollectionParams.SOURCE_NODE);
+ String target = message.getStr(CollectionParams.TARGET_NODE);
boolean waitForFinalState = message.getBool(CommonAdminParams.WAIT_FOR_FINAL_STATE, false);
if (source == null) {
throw new SolrException(
SolrException.ErrorCode.BAD_REQUEST, "sourceNode is a required param");
}
- String async = message.getStr("async");
+ String async = message.getStr(ASYNC);
int timeout = message.getInt("timeout", 10 * 60); // 10 minutes
boolean parallel = message.getBool("parallel", false);
ClusterState clusterState = zkStateReader.getClusterState();
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 59d331cd344..79467e08348 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -97,6 +97,8 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.RE
import static org.apache.solr.common.params.CollectionParams.CollectionAction.RESTORE;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.SPLITSHARD;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.SYNCSHARD;
+import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE;
+import static org.apache.solr.common.params.CollectionParams.TARGET_NODE;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;
@@ -219,6 +221,7 @@ import org.apache.solr.handler.admin.api.ModifyCollectionAPI;
import org.apache.solr.handler.admin.api.MoveReplicaAPI;
import org.apache.solr.handler.admin.api.RebalanceLeadersAPI;
import org.apache.solr.handler.admin.api.ReloadCollectionAPI;
+import org.apache.solr.handler.admin.api.ReplaceNodeAPI;
import org.apache.solr.handler.admin.api.SetCollectionPropertyAPI;
import org.apache.solr.handler.admin.api.SplitShardAPI;
import org.apache.solr.handler.admin.api.SyncShardAPI;
@@ -1792,14 +1795,18 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
REPLACENODE_OP(
REPLACENODE,
(req, rsp, h) -> {
- return copy(
- req.getParams(),
- null,
- "source", // legacy
- "target", // legacy
- WAIT_FOR_FINAL_STATE,
- CollectionParams.SOURCE_NODE,
- CollectionParams.TARGET_NODE);
+ final SolrParams params = req.getParams();
+ final RequiredSolrParams requiredParams = req.getParams().required();
+ final ReplaceNodeAPI.ReplaceNodeRequestBody requestBody =
+ new ReplaceNodeAPI.ReplaceNodeRequestBody();
+ requestBody.targetNodeName = params.get(TARGET_NODE);
+ requestBody.waitForFinalState = params.getBool(WAIT_FOR_FINAL_STATE);
+ requestBody.async = params.get(ASYNC);
+ final ReplaceNodeAPI replaceNodeAPI = new ReplaceNodeAPI(h.coreContainer, req, rsp);
+ final SolrJerseyResponse replaceNodeResponse =
+ replaceNodeAPI.replaceNode(requiredParams.get(SOURCE_NODE), requestBody);
+ V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, replaceNodeResponse);
+ return null;
}),
MOVEREPLICA_OP(
MOVEREPLICA,
@@ -1811,7 +1818,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
map,
CollectionParams.FROM_NODE,
CollectionParams.SOURCE_NODE,
- CollectionParams.TARGET_NODE,
+ TARGET_NODE,
WAIT_FOR_FINAL_STATE,
IN_PLACE_MOVE,
"replica",
@@ -2071,7 +2078,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
@Override
public Collection<Class<? extends JerseyResource>> getJerseyResources() {
- return List.of(AddReplicaPropertyAPI.class);
+ return List.of(AddReplicaPropertyAPI.class, ReplaceNodeAPI.class);
}
@Override
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNodeAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNodeAPI.java
new file mode 100644
index 00000000000..dc4fca1707d
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/admin/api/ReplaceNodeAPI.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin.api;
+
+import static org.apache.solr.client.solrj.impl.BinaryResponseParser.BINARY_CONTENT_TYPE_V2;
+import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
+import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE;
+import static org.apache.solr.common.params.CollectionParams.TARGET_NODE;
+import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
+import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
+import static org.apache.solr.handler.admin.CollectionsHandler.DEFAULT_COLLECTION_OP_TIMEOUT;
+import static org.apache.solr.security.PermissionNameProvider.Name.COLL_EDIT_PERM;
+
+import com.fasterxml.jackson.annotation.JsonProperty;
+import io.swagger.v3.oas.annotations.Parameter;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.parameters.RequestBody;
+import java.util.HashMap;
+import java.util.Map;
+import javax.inject.Inject;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.jersey.JacksonReflectMapWriter;
+import org.apache.solr.jersey.PermissionName;
+import org.apache.solr.jersey.SolrJerseyResponse;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+
+/**
+ * V2 API for recreating replicas in one node (the source) on another node(s) (the target).
+ *
+ * <p>This API is analogous to the v1 /admin/collections?action=REPLACENODE command.
+ */
+@Path("cluster/nodes/{sourceNodeName}/replace/")
+public class ReplaceNodeAPI extends AdminAPIBase {
+
+ @Inject
+ public ReplaceNodeAPI(
+ CoreContainer coreContainer,
+ SolrQueryRequest solrQueryRequest,
+ SolrQueryResponse solrQueryResponse) {
+ super(coreContainer, solrQueryRequest, solrQueryResponse);
+ }
+
+ @POST
+ @Produces({"application/json", "application/xml", BINARY_CONTENT_TYPE_V2})
+ @PermissionName(COLL_EDIT_PERM)
+ public SolrJerseyResponse replaceNode(
+ @Parameter(description = "The name of the node to be replaced.", required = true)
+ @PathParam("sourceNodeName")
+ String sourceNodeName,
+ @RequestBody(description = "Contains user provided parameters", required = true)
+ ReplaceNodeRequestBody requestBody)
+ throws Exception {
+ final SolrJerseyResponse response = instantiateJerseyResponse(SolrJerseyResponse.class);
+ final CoreContainer coreContainer = fetchAndValidateZooKeeperAwareCoreContainer();
+ // TODO Record node for log and tracing
+ final ZkNodeProps remoteMessage = createRemoteMessage(sourceNodeName, requestBody);
+ final SolrResponse remoteResponse =
+ CollectionsHandler.submitCollectionApiCommand(
+ coreContainer,
+ coreContainer.getDistributedCollectionCommandRunner(),
+ remoteMessage,
+ CollectionParams.CollectionAction.REPLACENODE,
+ DEFAULT_COLLECTION_OP_TIMEOUT);
+ if (remoteResponse.getException() != null) {
+ throw remoteResponse.getException();
+ }
+
+ disableResponseCaching();
+ return response;
+ }
+
+ public ZkNodeProps createRemoteMessage(String nodeName, ReplaceNodeRequestBody requestBody) {
+ final Map<String, Object> remoteMessage = new HashMap<>();
+ remoteMessage.put(SOURCE_NODE, nodeName);
+ if (requestBody != null) {
+ insertIfValueNotNull(remoteMessage, TARGET_NODE, requestBody.targetNodeName);
+ insertIfValueNotNull(remoteMessage, WAIT_FOR_FINAL_STATE, requestBody.waitForFinalState);
+ insertIfValueNotNull(remoteMessage, ASYNC, requestBody.async);
+ }
+ remoteMessage.put(QUEUE_OPERATION, CollectionAction.REPLACENODE.toLower());
+
+ return new ZkNodeProps(remoteMessage);
+ }
+
+ private void insertIfValueNotNull(Map<String, Object> dest, String key, Object value) {
+ if (value != null) {
+ dest.put(key, value);
+ }
+ }
+
+ public static class ReplaceNodeRequestBody implements JacksonReflectMapWriter {
+
+ public ReplaceNodeRequestBody() {}
+
+ public ReplaceNodeRequestBody(String targetNodeName, Boolean waitForFinalState, String async) {
+ this.targetNodeName = targetNodeName;
+ this.waitForFinalState = waitForFinalState;
+ this.async = async;
+ }
+
+ @Schema(
+ description =
+ "The target node where replicas will be copied. If this parameter is not provided, Solr "
+ + "will identify nodes automatically based on policies or number of cores in each node.")
+ @JsonProperty("targetNodeName")
+ public String targetNodeName;
+
+ @Schema(
+ description =
+ "If true, the request will complete only when all affected replicas become active. "
+ + "If false, the API will return the status of the single action, which may be "
+ + "before the new replica is online and active.")
+ @JsonProperty("waitForFinalState")
+ public Boolean waitForFinalState = false;
+
+ @Schema(description = "Request ID to track this action which will be processed asynchronously.")
+ @JsonProperty("async")
+ public String async;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
index 42f0f936756..6690e762ec9 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ReplaceNodeTest.java
@@ -17,6 +17,9 @@
package org.apache.solr.cloud;
+import static org.apache.solr.common.params.CollectionParams.SOURCE_NODE;
+import static org.apache.solr.common.params.CollectionParams.TARGET_NODE;
+
import com.codahale.metrics.Metric;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
@@ -322,9 +325,9 @@ public class ReplaceNodeTest extends SolrCloudTestCase {
@Override
public SolrParams getParams() {
ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
- params.set("source", sourceNode);
+ params.set(SOURCE_NODE, sourceNode);
if (!StringUtils.isEmpty(targetNode)) {
- params.setNonNull("target", targetNode);
+ params.setNonNull(TARGET_NODE, targetNode);
}
if (parallel != null) params.set("parallel", parallel.toString());
return params;
diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java
new file mode 100644
index 00000000000..3a460245775
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/admin/api/ReplaceNodeAPITest.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.handler.admin.api;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Map;
+import java.util.Optional;
+import org.apache.solr.SolrTestCaseJ4;
+import org.apache.solr.cloud.OverseerSolrResponse;
+import org.apache.solr.cloud.api.collections.DistributedCollectionConfigSetCommandRunner;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+
+/** Unit tests for {@link ReplaceNodeAPI} */
+public class ReplaceNodeAPITest extends SolrTestCaseJ4 {
+
+ private CoreContainer mockCoreContainer;
+ private SolrQueryRequest mockQueryRequest;
+ private SolrQueryResponse queryResponse;
+ private ReplaceNodeAPI replaceNodeApi;
+ private DistributedCollectionConfigSetCommandRunner mockCommandRunner;
+ private ArgumentCaptor<ZkNodeProps> messageCapturer;
+
+ @BeforeClass
+ public static void ensureWorkingMockito() {
+ assumeWorkingMockito();
+ }
+
+ @Override
+ @Before
+ public void setUp() throws Exception {
+ super.setUp();
+
+ mockCoreContainer = mock(CoreContainer.class);
+ mockCommandRunner = mock(DistributedCollectionConfigSetCommandRunner.class);
+ when(mockCoreContainer.getDistributedCollectionCommandRunner())
+ .thenReturn(Optional.of(mockCommandRunner));
+ when(mockCommandRunner.runCollectionCommand(any(), any(), anyLong()))
+ .thenReturn(new OverseerSolrResponse(new NamedList<>()));
+ mockQueryRequest = mock(SolrQueryRequest.class);
+ queryResponse = new SolrQueryResponse();
+ replaceNodeApi = new ReplaceNodeAPI(mockCoreContainer, mockQueryRequest, queryResponse);
+ messageCapturer = ArgumentCaptor.forClass(ZkNodeProps.class);
+
+ when(mockCoreContainer.isZooKeeperAware()).thenReturn(true);
+ }
+
+ @Test
+ public void testCreatesValidOverseerMessage() throws Exception {
+ ReplaceNodeAPI.ReplaceNodeRequestBody requestBody =
+ new ReplaceNodeAPI.ReplaceNodeRequestBody("demoTargetNode", false, "async");
+ replaceNodeApi.replaceNode("demoSourceNode", requestBody);
+ verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong());
+
+ final ZkNodeProps createdMessage = messageCapturer.getValue();
+ final Map<String, Object> createdMessageProps = createdMessage.getProperties();
+ assertEquals(5, createdMessageProps.size());
+ assertEquals("demoSourceNode", createdMessageProps.get("sourceNode"));
+ assertEquals("demoTargetNode", createdMessageProps.get("targetNode"));
+ assertEquals(false, createdMessageProps.get("waitForFinalState"));
+ assertEquals("async", createdMessageProps.get("async"));
+ assertEquals("replacenode", createdMessageProps.get("operation"));
+ }
+
+ @Test
+ public void testRequestBodyCanBeOmittedAltogether() throws Exception {
+ replaceNodeApi.replaceNode("demoSourceNode", null);
+ verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong());
+
+ final ZkNodeProps createdMessage = messageCapturer.getValue();
+ final Map<String, Object> createdMessageProps = createdMessage.getProperties();
+ assertEquals(2, createdMessageProps.size());
+ assertEquals("demoSourceNode", createdMessageProps.get("sourceNode"));
+ assertEquals("replacenode", createdMessageProps.get("operation"));
+ }
+
+ @Test
+ public void testOptionalValuesNotAddedToRemoteMessageIfNotProvided() throws Exception {
+ ReplaceNodeAPI.ReplaceNodeRequestBody requestBody =
+ new ReplaceNodeAPI.ReplaceNodeRequestBody("demoTargetNode", null, null);
+ replaceNodeApi.replaceNode("demoSourceNode", requestBody);
+ verify(mockCommandRunner).runCollectionCommand(messageCapturer.capture(), any(), anyLong());
+
+ final ZkNodeProps createdMessage = messageCapturer.getValue();
+ final Map<String, Object> createdMessageProps = createdMessage.getProperties();
+
+ assertEquals(3, createdMessageProps.size());
+ assertEquals("demoSourceNode", createdMessageProps.get("sourceNode"));
+ assertEquals("demoTargetNode", createdMessageProps.get("targetNode"));
+ assertEquals("replacenode", createdMessageProps.get("operation"));
+ assertFalse(
+ "Expected message to not contain value for waitForFinalState: "
+ + createdMessageProps.get("waitForFinalState"),
+ createdMessageProps.containsKey("waitForFinalState"));
+ assertFalse(
+ "Expected message to not contain value for async: " + createdMessageProps.get("async"),
+ createdMessageProps.containsKey("async"));
+ }
+}
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
index 48f592b6009..217c7bdbdce 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/cluster-node-management.adoc
@@ -499,7 +499,16 @@ http://localhost:8983/solr/admin/collections?action=REPLACENODE&sourceNode=sourc
====
[.tab-label]*V2 API*
-We do not currently have a V2 equivalent.
+[source,bash]
+----
+curl -X POST "http://localhost:8983/api/cluster/nodes/localhost:7574_solr/commands/replace/" -H 'Content-Type: application/json' -d '
+ {
+ "targetNodeName": "localhost:8983_solr",
+ "waitForFinalState": "false",
+ "async": "async"
+ }
+'
+----
====
--