You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/06/30 07:43:23 UTC
lucene-solr:branch_6x: SOLR-9216: Support collection.configName in
MODIFYCOLLECTION request
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x d1a9daec8 -> ce3ea7678
SOLR-9216: Support collection.configName in MODIFYCOLLECTION request
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/ce3ea767
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/ce3ea767
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/ce3ea767
Branch: refs/heads/branch_6x
Commit: ce3ea76781c79b700a09a4c5fa36621af14016de
Parents: d1a9dae
Author: Noble Paul <no...@apache.org>
Authored: Thu Jun 30 12:10:36 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Thu Jun 30 13:13:01 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +
.../cloud/OverseerCollectionMessageHandler.java | 63 ++++++-------
.../solr/cloud/overseer/CollectionMutator.java | 11 ++-
.../solr/handler/admin/CollectionsHandler.java | 25 +++--
.../cloud/OverseerModifyCollectionTest.java | 96 ++++++++++++++++++++
.../apache/solr/common/cloud/DocCollection.java | 2 +-
.../apache/solr/common/cloud/ZkStateReader.java | 6 +-
.../apache/solr/common/params/SolrParams.java | 24 +++--
8 files changed, 177 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 6ccff2a..80ad0d7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -40,6 +40,9 @@ New Features
* SOLR-7374: Core level Backup/Restore now supports specifying the directory implementation to use
via the "repository" parameter. (Hrishikesh Gadre, Varun Thacker, Mark Miller)
+* SOLR-9216: Support collection.configName in MODIFYCOLLECTION request (Keith Laban, noble)
+
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index d7c7ad2..a0ac732 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -136,7 +136,7 @@ import static org.apache.solr.common.util.Utils.makeMap;
public class OverseerCollectionMessageHandler implements OverseerMessageHandler {
public static final String NUM_SLICES = "numShards";
-
+
static final boolean CREATE_NODE_SET_SHUFFLE_DEFAULT = true;
public static final String CREATE_NODE_SET_SHUFFLE = "createNodeSet.shuffle";
public static final String CREATE_NODE_SET_EMPTY = "EMPTY";
@@ -275,7 +275,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
processRebalanceLeaders(message);
break;
case MODIFYCOLLECTION:
- overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
+ modifyCollection(message, results);
break;
case MIGRATESTATEFORMAT:
migrateStateFormat(message, results);
@@ -1824,6 +1824,25 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
return nodeList;
}
+
+
+ private void modifyCollection(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
+
+ final String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
+ //the rest of the processing is based on writing cluster state properties
+ //remove the property here to avoid any errors down the pipeline due to this property appearing
+ String configName = (String) message.getProperties().remove(COLL_CONF);
+
+ if(configName != null) {
+ validateConfigOrThrowSolrException(configName);
+
+ boolean isLegacyCloud = Overseer.isLegacy(zkStateReader);
+ createConfNode(configName, collectionName, isLegacyCloud);
+ reloadCollection(new ZkNodeProps(NAME, collectionName), results);
+ }
+
+ overseer.getStateUpdateQueue(zkStateReader.getZkClient()).offer(Utils.toJSON(message));
+ }
private void createCollection(ClusterState clusterState, ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
final String collectionName = message.getStr(NAME);
@@ -1835,9 +1854,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
String configName = getConfigName(collectionName, message);
if (configName == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "No config set found to associate with the collection.");
- } else if (!validateConfig(configName)) {
- throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName);
}
+
+ validateConfigOrThrowSolrException(configName);
+
try {
// look at the replication factor and see if it matches reality
@@ -2487,8 +2507,11 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
return configName;
}
- private boolean validateConfig(String configName) throws KeeperException, InterruptedException {
- return zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);
+ private void validateConfigOrThrowSolrException(String configName) throws KeeperException, InterruptedException {
+ boolean isValid = zkStateReader.getZkClient().exists(ZkConfigManager.CONFIGS_ZKNODE + "/" + configName, true);
+ if(!isValid) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Can not find the specified config set: " + configName);
+ }
}
/**
@@ -2680,34 +2703,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
- /* @Override
- public void markExclusiveTask(String collectionName, ZkNodeProps message) {
- if (collectionName != null) {
- synchronized (collectionWip) {
- collectionWip.add(collectionName);
- }
- }
- }
-
- @Override
- public void unmarkExclusiveTask(String collectionName, String operation, ZkNodeProps message) {
- if(collectionName != null) {
- synchronized (collectionWip) {
- collectionWip.remove(collectionName);
- }
- }
- }*/
-/*
- @Override
- public ExclusiveMarking checkExclusiveMarking(String collectionName, ZkNodeProps message) {
- synchronized (collectionWip) {
- if(collectionWip.contains(collectionName))
- return ExclusiveMarking.NONEXCLUSIVE;
- }
-
- return ExclusiveMarking.NOTDETERMINED;
- }*/
-
private long sessionId = -1;
private LockTree.Session lockSession;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
index 3d950fe..4d1ddcd 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
@@ -92,9 +92,18 @@ public class CollectionMutator {
if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
DocCollection coll = clusterState.getCollection(message.getStr(COLLECTION_PROP));
Map<String, Object> m = coll.shallowCopy();
+ boolean hasAnyOps = false;
for (String prop : CollectionsHandler.MODIFIABLE_COLL_PROPS) {
- if(message.get(prop)!= null) m.put(prop,message.get(prop));
+ if(message.get(prop)!= null) {
+ hasAnyOps = true;
+ m.put(prop,message.get(prop));
+ }
+ }
+
+ if(!hasAnyOps) {
+ return ZkStateWriter.NO_OP;
}
+
return new ZkWriteCommand(coll.getName(),
new DocCollection(coll.getName(),coll.getSlicesMap(),m,coll.getRouter(),coll.getZNodeVersion(),coll.getZNode()));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index 6acd86a..85c98c1 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -29,7 +29,6 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -51,8 +50,18 @@ import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
-import org.apache.solr.common.cloud.*;
+import org.apache.solr.common.cloud.ClusterProperties;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ImplicitDocRouter;
+import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Replica.State;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkCmdExecutor;
+import org.apache.solr.common.cloud.ZkCoreNodeProps;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionAdminParams;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
@@ -765,13 +774,13 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
return null;
}
},
- MODIFYCOLLECTION_OP(MODIFYCOLLECTION, DEFAULT_COLLECTION_OP_TIMEOUT, false) {
+ MODIFYCOLLECTION_OP(MODIFYCOLLECTION) {
@Override
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
- Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS.toArray(new String[0]));
+ Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS);
if (m.isEmpty()) throw new SolrException(ErrorCode.BAD_REQUEST,
- formatString("no supported values provided rule, snitch, masShardsPerNode, replicationFactor"));
+ formatString("no supported values provided rule, snitch, maxShardsPerNode, replicationFactor, collection.configName"));
req.getParams().required().getAll(m, COLLECTION_PROP);
addMapObject(m, RULE);
addMapObject(m, SNITCH);
@@ -1045,11 +1054,11 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
}
}
- public static final List<String> MODIFIABLE_COLL_PROPS = ImmutableList.of(
+ public static final List<String> MODIFIABLE_COLL_PROPS = Arrays.asList(
RULE,
SNITCH,
REPLICATION_FACTOR,
MAX_SHARDS_PER_NODE,
- AUTO_ADD_REPLICAS);
-
+ AUTO_ADD_REPLICAS,
+ COLL_CONF);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/core/src/test/org/apache/solr/cloud/OverseerModifyCollectionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerModifyCollectionTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerModifyCollectionTest.java
new file mode 100644
index 0000000..6809b6d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerModifyCollectionTest.java
@@ -0,0 +1,96 @@
+package org.apache.solr.cloud;
+
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
+import static org.apache.solr.common.params.CommonParams.COLLECTIONS_HANDLER_PATH;
+
+import java.lang.invoke.MethodHandles;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.impl.HttpSolrClient.RemoteSolrException;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.ConfigSetAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
+import org.apache.solr.client.solrj.response.ConfigSetAdminResponse;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.util.Utils;
+import org.apache.zookeeper.KeeperException;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class OverseerModifyCollectionTest extends AbstractFullDistribZkTestBase {
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ @Test
+ public void testModifyColl() throws Exception {
+ String collName = "modifyColl";
+ String newConfName = "conf" + random().nextInt();
+ String oldConfName = "conf1";
+ try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
+ CollectionAdminResponse rsp;
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collName, oldConfName, 1, 2);
+ rsp = create.process(client);
+ assertEquals(0, rsp.getStatus());
+ assertTrue(rsp.isSuccess());
+
+ ConfigSetAdminRequest.Create createConfig = new ConfigSetAdminRequest.Create()
+ .setBaseConfigSetName(oldConfName)
+ .setConfigSetName(newConfName);
+
+ ConfigSetAdminResponse configRsp = createConfig.process(client);
+
+ assertEquals(0, configRsp.getStatus());
+
+ ModifiableSolrParams p = new ModifiableSolrParams();
+ p.add("collection", collName);
+ p.add("action", "MODIFYCOLLECTION");
+ p.add("collection.configName", newConfName);
+ client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
+ }
+
+ assertEquals(newConfName, getConfigNameFromZk(collName));
+
+ //Try an invalid config name
+ try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
+ ModifiableSolrParams p = new ModifiableSolrParams();
+ p.add("collection", collName);
+ p.add("action", "MODIFYCOLLECTION");
+ p.add("collection.configName", "notARealConfigName");
+ try{
+ client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
+ fail("Exception should be thrown");
+ } catch(RemoteSolrException e) {
+ assertTrue(e.getMessage(), e.getMessage().contains("Can not find the specified config set"));
+ }
+ }
+
+ }
+
+ private String getConfigNameFromZk(String collName) throws KeeperException, InterruptedException {
+ byte[] b = cloudClient.getZkStateReader().getZkClient().getData(ZkStateReader.getCollectionPathRoot(collName), null, null, false);
+ Map confData = (Map) Utils.fromJSON(b);
+ return (String) confData.get(ZkController.CONFIGNAME_PROP);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
index 5504a8b..978bdc0 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
@@ -107,7 +107,7 @@ public class DocCollection extends ZkNodeProps implements Iterable<Slice> {
case "rule":
return (List) o;
default:
- throw new SolrException(ErrorCode.SERVER_ERROR, "Unknown property " + propName);
+ return o;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
index aff5bba..fe3922c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
@@ -1075,9 +1075,13 @@ public class ZkStateReader implements Closeable {
}
}
}
+
+ public static String getCollectionPathRoot(String coll) {
+ return COLLECTIONS_ZKNODE+"/"+coll;
+ }
public static String getCollectionPath(String coll) {
- return COLLECTIONS_ZKNODE+"/"+coll + "/state.json";
+ return getCollectionPathRoot(coll) + "/state.json";
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ce3ea767/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
index 090fdb8..aac7598 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/SolrParams.java
@@ -26,6 +26,9 @@ import java.io.Serializable;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -371,23 +374,28 @@ public abstract class SolrParams implements Serializable {
return result;
}
- /**Copy all params to the given map or if the given map is null
- * create a new one
- */
- public Map<String, Object> getAll(Map<String, Object> sink, String... params){
- if(sink == null) sink = new LinkedHashMap<>();
+ public Map<String, Object> getAll(Map<String, Object> sink, Collection<String> params) {
+ if (sink == null) sink = new LinkedHashMap<>();
for (String param : params) {
String[] v = getParams(param);
- if(v != null && v.length>0 ) {
- if(v.length == 1) {
+ if (v != null && v.length > 0) {
+ if (v.length == 1) {
sink.put(param, v[0]);
} else {
- sink.put(param,v);
+ sink.put(param, v);
}
}
}
return sink;
}
+
+
+ /**Copy all params to the given map or if the given map is null
+ * create a new one
+ */
+ public Map<String, Object> getAll(Map<String, Object> sink, String... params){
+ return getAll(sink, params == null ? Collections.emptyList() : Arrays.asList(params));
+ }
/** Returns this SolrParams as a properly URL encoded string, starting with {@code "?"}, if not empty. */
public String toQueryString() {