You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2015/05/18 18:23:20 UTC
svn commit: r1680052 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/cloud/
core/src/java/org/apache/solr/cloud/overseer/
core/src/java/org/apache/solr/cloud/rule/
core/src/java/org/apache/solr/handler/admin/ core/src/test/org/apache/solr...
Author: noble
Date: Mon May 18 16:23:19 2015
New Revision: 1680052
URL: http://svn.apache.org/r1680052
Log:
SOLR-5132: Added a new collection action MODIFYCOLLECTION
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Mon May 18 16:23:19 2015
@@ -569,12 +569,14 @@ New Features
* SOLR-7226: Make /query/* jmx/* , requestDispatcher/*, <listener> <initParams>
properties in solrconfig.xml editable (Noble Paul)
-* SOLR-7240: '/' redirects to '/solr/' for convinience (Martijn Koster, hossman)
+* SOLR-7240: '/' redirects to '/solr/' for convenience (Martijn Koster, hossman)
* SOLR-5911: Added payload support for term vectors. New "termPayloads" option for fields
/ types in the schema, and "tv.payloads" param for the term vector component.
(Mike McCandless, David Smiley)
+* SOLR-5132: Added a new collection action MODIFYCOLLECTION (Noble Paul)
+
Bug Fixes
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/Overseer.java Mon May 18 16:23:19 2015
@@ -53,6 +53,7 @@ import org.apache.solr.common.cloud.ZkSt
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.IOUtils;
import org.apache.solr.core.CloudConfig;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.util.stats.Clock;
@@ -353,6 +354,9 @@ public class Overseer implements Closeab
return new ZkWriteCommand(collName, dProp.getDocCollection());
}
break;
+ case MODIFYCOLLECTION:
+ CollectionsHandler.verifyRuleParams(zkController.getCoreContainer() ,message.getProperties());
+ return new CollectionMutator(reader).modifyCollection(clusterState,message);
default:
throw new RuntimeException("unknown operation:" + operation
+ " contents:" + message.getProperties());
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Mon May 18 16:23:19 2015
@@ -18,6 +18,7 @@ package org.apache.solr.cloud;
*/
import static org.apache.solr.cloud.Assign.*;
+import static org.apache.solr.common.cloud.ZkNodeProps.makeMap;
import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonParams.*;
@@ -135,13 +136,13 @@ public class OverseerCollectionProcessor
public int maxParallelThreads = 10;
- public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
+ public static final Map<String, Object> COLL_PROPS = Collections.unmodifiableMap(makeMap(
ROUTER, DocRouter.DEFAULT_NAME,
ZkStateReader.REPLICATION_FACTOR, "1",
ZkStateReader.MAX_SHARDS_PER_NODE, "1",
ZkStateReader.AUTO_ADD_REPLICAS, "false",
- "rule", null,
- "snitch",null);
+ DocCollection.RULE, null,
+ DocCollection.SNITCH, null));
static final Random RANDOM;
static {
@@ -615,6 +616,9 @@ public class OverseerCollectionProcessor
case REBALANCELEADERS:
processRebalanceLeaders(message);
break;
+ case MODIFYCOLLECTION:
+ overseer.getInQueue(zkStateReader.getZkClient()).offer(ZkStateReader.toJSON(message));
+ break;
default:
throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ operation);
@@ -1045,7 +1049,7 @@ public class OverseerCollectionProcessor
String core = replica.getStr(ZkStateReader.CORE_NAME_PROP);
// assume the core exists and try to unload it
- Map m = ZkNodeProps.makeMap("qt", adminPath, CoreAdminParams.ACTION,
+ Map m = makeMap("qt", adminPath, CoreAdminParams.ACTION,
CoreAdminAction.UNLOAD.toString(), CoreAdminParams.CORE, core,
CoreAdminParams.DELETE_INSTANCE_DIR, "true",
CoreAdminParams.DELETE_DATA_DIR, "true");
@@ -1292,8 +1296,8 @@ public class OverseerCollectionProcessor
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
DocCollection collection = clusterState.getCollection(collectionName);
- int maxShardsPerNode = collection.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1);
- int repFactor = message.getInt(ZkStateReader.REPLICATION_FACTOR, collection.getInt(ZkStateReader.REPLICATION_FACTOR, 1));
+ int maxShardsPerNode = collection.getInt(MAX_SHARDS_PER_NODE, 1);
+ int repFactor = message.getInt(REPLICATION_FACTOR, collection.getInt(REPLICATION_FACTOR, 1));
String createNodeSetStr = message.getStr(CREATE_NODE_SET);
ArrayList<Node> sortedNodeList = getNodesForNewShard(clusterState, collectionName, numSlices, maxShardsPerNode, repFactor, createNodeSetStr);
@@ -1981,7 +1985,7 @@ public class OverseerCollectionProcessor
String tempSourceCollectionName = "split_" + sourceSlice.getName() + "_temp_" + targetSlice.getName();
if (clusterState.hasCollection(tempSourceCollectionName)) {
log.info("Deleting temporary collection: " + tempSourceCollectionName);
- Map<String, Object> props = ZkNodeProps.makeMap(
+ Map<String, Object> props = makeMap(
Overseer.QUEUE_OPERATION, DELETE.toLower(),
NAME, tempSourceCollectionName);
@@ -2065,10 +2069,10 @@ public class OverseerCollectionProcessor
// create a temporary collection with just one node on the shard leader
String configName = zkStateReader.readConfigName(sourceCollection.getName());
- Map<String, Object> props = ZkNodeProps.makeMap(
+ Map<String, Object> props = makeMap(
Overseer.QUEUE_OPERATION, CREATE.toLower(),
NAME, tempSourceCollectionName,
- ZkStateReader.REPLICATION_FACTOR, 1,
+ REPLICATION_FACTOR, 1,
NUM_SLICES, 1,
COLL_CONF, configName,
CREATE_NODE_SET, sourceLeader.getNodeName());
@@ -2194,7 +2198,7 @@ public class OverseerCollectionProcessor
try {
log.info("Deleting temporary collection: " + tempSourceCollectionName);
- props = ZkNodeProps.makeMap(
+ props = makeMap(
Overseer.QUEUE_OPERATION, DELETE.toLower(),
NAME, tempSourceCollectionName);
deleteCollection(new ZkNodeProps(props), results);
@@ -2293,7 +2297,7 @@ public class OverseerCollectionProcessor
// look at the replication factor and see if it matches reality
// if it does not, find best nodes to create more cores
- int repFactor = message.getInt(ZkStateReader.REPLICATION_FACTOR, 1);
+ int repFactor = message.getInt(REPLICATION_FACTOR, 1);
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
String async = null;
@@ -2312,10 +2316,10 @@ public class OverseerCollectionProcessor
ClusterStateMutator.getShardNames(numSlices, shardNames);
}
- int maxShardsPerNode = message.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1);
+ int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
if (repFactor <= 0) {
- throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.REPLICATION_FACTOR + " must be greater than 0");
+ throw new SolrException(ErrorCode.BAD_REQUEST, REPLICATION_FACTOR + " must be greater than 0");
}
if (numSlices <= 0) {
@@ -2330,7 +2334,7 @@ public class OverseerCollectionProcessor
if (repFactor > nodeList.size()) {
log.warn("Specified "
- + ZkStateReader.REPLICATION_FACTOR
+ + REPLICATION_FACTOR
+ " of "
+ repFactor
+ " on collection "
@@ -2344,11 +2348,11 @@ public class OverseerCollectionProcessor
int requestedShardsToCreate = numSlices * repFactor;
if (maxShardsAllowedToCreate < requestedShardsToCreate) {
throw new SolrException(ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
- + ZkStateReader.MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
+ + MAX_SHARDS_PER_NODE + " is " + maxShardsPerNode
+ ", and the number of nodes currently live or live and part of your "+CREATE_NODE_SET+" is " + nodeList.size()
+ ". This allows a maximum of " + maxShardsAllowedToCreate
+ " to be created. Value of " + NUM_SLICES + " is " + numSlices
- + " and value of " + ZkStateReader.REPLICATION_FACTOR + " is " + repFactor
+ + " and value of " + REPLICATION_FACTOR + " is " + repFactor
+ ". This requires " + requestedShardsToCreate
+ " shards to be created (higher than the allowed number)");
}
@@ -2556,7 +2560,7 @@ public class OverseerCollectionProcessor
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
if (node == null) {
- node = getNodesForNewShard(clusterState, collection, coll.getSlices().size(), coll.getInt(ZkStateReader.MAX_SHARDS_PER_NODE, 1), coll.getInt(ZkStateReader.REPLICATION_FACTOR, 1), null).get(0).nodeName;
+ node = getNodesForNewShard(clusterState, collection, coll.getSlices().size(), coll.getInt(MAX_SHARDS_PER_NODE, 1), coll.getInt(REPLICATION_FACTOR, 1), null).get(0).nodeName;
log.info("Node not provided, Identified {} for creating new replica", node);
}
@@ -2684,7 +2688,7 @@ public class OverseerCollectionProcessor
if (configName != null) {
String collDir = ZkStateReader.COLLECTIONS_ZKNODE + "/" + coll;
log.info("creating collections conf node {} ", collDir);
- byte[] data = ZkStateReader.toJSON(ZkNodeProps.makeMap(ZkController.CONFIGNAME_PROP, configName));
+ byte[] data = ZkStateReader.toJSON(makeMap(ZkController.CONFIGNAME_PROP, configName));
if (zkStateReader.getZkClient().exists(collDir, true)) {
zkStateReader.getZkClient().setData(collDir, data, true);
} else {
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/overseer/CollectionMutator.java Mon May 18 16:23:19 2015
@@ -29,9 +29,11 @@ import org.apache.solr.common.cloud.Repl
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.handler.admin.CollectionsHandler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
import static org.apache.solr.common.params.CommonParams.NAME;
public class CollectionMutator {
@@ -84,6 +86,17 @@ public class CollectionMutator {
return new ZkWriteCommand(collection, newCollection);
}
+ public ZkWriteCommand modifyCollection(final ClusterState clusterState, ZkNodeProps message){
+ if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
+ DocCollection coll = clusterState.getCollection(message.getStr(COLLECTION_PROP));
+ Map<String, Object> m = coll.shallowCopy();
+ for (String prop : CollectionsHandler.MODIFIABLE_COLL_PROPS) {
+ if(message.get(prop)!= null) m.put(prop,message.get(prop));
+ }
+ return new ZkWriteCommand(coll.getName(),
+ new DocCollection(coll.getName(),coll.getSlicesMap(),m,coll.getRouter(),coll.getZNodeVersion(),coll.getZNode()));
+ }
+
public static DocCollection updateSlice(String collectionName, DocCollection collection, Slice slice) {
DocCollection newCollection = null;
Map<String, Slice> slices;
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/cloud/rule/ReplicaAssigner.java Mon May 18 16:23:19 2015
@@ -326,62 +326,39 @@ public class ReplicaAssigner {
public Map<String, SnitchContext> failedNodes = new HashMap<>();
+ static class SnitchInfoImpl extends SnitchContext.SnitchInfo {
+ final Snitch snitch;
+ final Set<String> myTags = new HashSet<>();
+ final Map<String, SnitchContext> nodeVsContext = new HashMap<>();
+ private final CoreContainer cc;
+
+ SnitchInfoImpl(Map<String, Object> conf, Snitch snitch, CoreContainer cc) {
+ super(conf);
+ this.snitch = snitch;
+ this.cc = cc;
+ }
+
+ @Override
+ public Set<String> getTagNames() {
+ return myTags;
+ }
+
+ @Override
+ public CoreContainer getCoreContainer() {
+ return cc;
+ }
+ }
+
/**
* This method uses the snitches and get the tags for all the nodes
*/
private Map<String, Map<String, Object>> getTagsForNodes(final CoreContainer cc, List snitchConf) {
- class Info extends SnitchContext.SnitchInfo {
- final Snitch snitch;
- final Set<String> myTags = new HashSet<>();
- final Map<String, SnitchContext> nodeVsContext = new HashMap<>();
-
- Info(Map<String, Object> conf, Snitch snitch) {
- super(conf);
- this.snitch = snitch;
- }
-
- @Override
- public Set<String> getTagNames() {
- return myTags;
- }
-
- @Override
- public CoreContainer getCoreContainer() {
- return cc;
- }
- }
-
- Map<Class, Info> snitches = new LinkedHashMap<>();
- for (Object o : snitchConf) {
- //instantiating explicitly specified snitches
- String klas = null;
- Map map = Collections.emptyMap();
- if (o instanceof Map) {//it can be a Map
- map = (Map) o;
- klas = (String) map.get("class");
- if (klas == null) {
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "snitch must have a class attribute");
- }
- } else { //or just the snitch name
- klas = o.toString();
- }
- try {
- if (klas.indexOf('.') == -1) klas = Snitch.class.getPackage().getName() + "." + klas;
- Snitch inst = cc == null ?
- (Snitch) Snitch.class.getClassLoader().loadClass(klas).newInstance() :
- cc.getResourceLoader().newInstance(klas, Snitch.class);
- snitches.put(inst.getClass(), new Info(map, inst));
- } catch (Exception e) {
- throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
-
- }
-
- }
+ Map<Class, SnitchInfoImpl> snitches = getSnitchInfos(cc, snitchConf);
for (Class c : Snitch.WELL_KNOWN_SNITCHES) {
if (snitches.containsKey(c)) continue;// it is already specified explicitly , ignore
try {
- snitches.put(c, new Info(Collections.EMPTY_MAP, (Snitch) c.newInstance()));
+ snitches.put(c, new SnitchInfoImpl(Collections.EMPTY_MAP, (Snitch) c.newInstance(), cc));
} catch (Exception e) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error instantiating Snitch " + c.getName());
}
@@ -389,7 +366,7 @@ public class ReplicaAssigner {
for (String tagName : tagNames) {
//identify which snitch is going to provide values for a given tag
boolean foundProvider = false;
- for (Info info : snitches.values()) {
+ for (SnitchInfoImpl info : snitches.values()) {
if (info.snitch.isKnownTag(tagName)) {
foundProvider = true;
info.myTags.add(tagName);
@@ -403,7 +380,7 @@ public class ReplicaAssigner {
for (String node : liveNodes) {
//now use the Snitch to get the tags
- for (Info info : snitches.values()) {
+ for (SnitchInfoImpl info : snitches.values()) {
if (!info.myTags.isEmpty()) {
SnitchContext context = new SnitchContext(info, node);
info.nodeVsContext.put(node, context);
@@ -417,7 +394,7 @@ public class ReplicaAssigner {
}
Map<String, Map<String, Object>> result = new HashMap<>();
- for (Info info : snitches.values()) {
+ for (SnitchInfoImpl info : snitches.values()) {
for (Map.Entry<String, SnitchContext> e : info.nodeVsContext.entrySet()) {
SnitchContext context = e.getValue();
String node = e.getKey();
@@ -448,4 +425,40 @@ public class ReplicaAssigner {
}
+ public static void verifySnitchConf(CoreContainer cc, List snitchConf) {
+ getSnitchInfos(cc, snitchConf);
+ }
+
+
+ static Map<Class, SnitchInfoImpl> getSnitchInfos(CoreContainer cc, List snitchConf) {
+ Map<Class, SnitchInfoImpl> snitches = new LinkedHashMap<>();
+ if (snitchConf == null) return snitches;
+ for (Object o : snitchConf) {
+ //instantiating explicitly specified snitches
+ String klas = null;
+ Map map = Collections.emptyMap();
+ if (o instanceof Map) {//it can be a Map
+ map = (Map) o;
+ klas = (String) map.get("class");
+ if (klas == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "snitch must have a class attribute");
+ }
+ } else { //or just the snitch name
+ klas = o.toString();
+ }
+ try {
+ if (klas.indexOf('.') == -1) klas = Snitch.class.getPackage().getName() + "." + klas;
+ Snitch inst = cc == null ?
+ (Snitch) Snitch.class.getClassLoader().loadClass(klas).newInstance() :
+ cc.getResourceLoader().newInstance(klas, Snitch.class);
+ snitches.put(inst.getClass(), new SnitchInfoImpl(map, inst, cc));
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, e);
+
+ }
+
+ }
+ return snitches;
+ }
+
}
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Mon May 18 16:23:19 2015
@@ -19,6 +19,7 @@ package org.apache.solr.handler.admin;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
@@ -27,6 +28,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.SolrResponse;
@@ -35,12 +37,15 @@ import org.apache.solr.client.solrj.requ
import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard;
import org.apache.solr.cloud.DistributedQueue;
import org.apache.solr.cloud.DistributedQueue.QueueEvent;
+import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.cloud.overseer.SliceMutator;
+import org.apache.solr.cloud.rule.ReplicaAssigner;
import org.apache.solr.cloud.rule.Rule;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.ImplicitDocRouter;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkCmdExecutor;
@@ -76,6 +81,8 @@ import static org.apache.solr.cloud.Over
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARDS_PROP;
import static org.apache.solr.cloud.OverseerCollectionProcessor.SHARD_UNIQUE;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
+import static org.apache.solr.common.cloud.DocCollection.RULE;
+import static org.apache.solr.common.cloud.DocCollection.SNITCH;
import static org.apache.solr.common.cloud.DocCollection.STATE_FORMAT;
import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
import static org.apache.solr.common.cloud.ZkStateReader.COLLECTION_PROP;
@@ -91,6 +98,7 @@ import static org.apache.solr.common.par
import static org.apache.solr.common.params.CoreAdminParams.DATA_DIR;
import static org.apache.solr.common.params.CoreAdminParams.INSTANCE_DIR;
import static org.apache.solr.common.params.ShardParams._ROUTE_;
+import static org.apache.solr.common.util.StrUtils.formatString;
public class CollectionsHandler extends RequestHandlerBase {
protected static Logger log = LoggerFactory.getLogger(CollectionsHandler.class);
@@ -157,7 +165,9 @@ public class CollectionsHandler extends
if (result != null) {
result.put(QUEUE_OPERATION, operation.action.toLower());
ZkNodeProps props = new ZkNodeProps(result);
- handleResponse(operation.action.toLower(), props, rsp, operation.timeOut);
+ if (operation.sendToOCPQueue) handleResponse(operation.action.toLower(), props, rsp, operation.timeOut);
+ else Overseer.getInQueue(coreContainer.getZkController().getZkClient()).offer(ZkStateReader.toJSON(props));
+
}
} else {
throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
@@ -287,7 +297,6 @@ public class CollectionsHandler extends
Map<String, Object> props = req.getParams().required().getAll(null, NAME);
props.put("fromApi", "true");
req.getParams().getAll(props,
- NAME,
REPLICATION_FACTOR,
COLL_CONF,
NUM_SLICES,
@@ -296,16 +305,18 @@ public class CollectionsHandler extends
SHARDS_PROP,
ASYNC,
STATE_FORMAT,
- AUTO_ADD_REPLICAS);
+ AUTO_ADD_REPLICAS,
+ RULE,
+ SNITCH);
if (props.get(STATE_FORMAT) == null) {
props.put(STATE_FORMAT, "2");
}
- addRuleMap(req.getParams(), props, "rule");
- addRuleMap(req.getParams(), props, "snitch");
-
+ addMapObject(props, RULE);
+ addMapObject(props, SNITCH);
+ verifyRuleParams(h.coreContainer, props);
if (SYSTEM_COLL.equals(props.get(NAME))) {
- //We must always create asystem collection with only a single shard
+ //We must always create a .system collection with only a single shard
props.put(NUM_SLICES, 1);
props.remove(SHARDS_PROP);
createSysConfigSet(h.coreContainer);
@@ -316,15 +327,6 @@ public class CollectionsHandler extends
}
- private void addRuleMap(SolrParams params, Map<String, Object> props, String key) {
- String[] rules = params.getParams(key);
- if (rules != null && rules.length > 0) {
- ArrayList<Map> l = new ArrayList<>();
- for (String rule : rules) l.add(Rule.parseRule(rule));
- props.put(key, l);
- }
- }
-
private void createSysConfigSet(CoreContainer coreContainer) throws KeeperException, InterruptedException {
SolrZkClient zk = coreContainer.getZkController().getZkStateReader().getZkClient();
ZkCmdExecutor cmdExecutor = new ZkCmdExecutor(zk.getZkClientTimeout());
@@ -396,7 +398,7 @@ public class CollectionsHandler extends
}
},
- SPLITSHARD_OP(SPLITSHARD, DEFAULT_ZK_TIMEOUT * 5) {
+ SPLITSHARD_OP(SPLITSHARD, DEFAULT_ZK_TIMEOUT * 5, true) {
@Override
Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h)
throws Exception {
@@ -670,17 +672,35 @@ public class CollectionsHandler extends
new RebalanceLeaders(req,rsp,h).execute();
return null;
}
+ },
+ MODIFYCOLLECTION_OP(MODIFYCOLLECTION, DEFAULT_ZK_TIMEOUT, false) {
+ @Override
+ Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, CollectionsHandler h) throws Exception {
+
+ Map<String, Object> m = req.getParams().getAll(null, MODIFIABLE_COLL_PROPS.toArray(new String[0]));
+ if (m.isEmpty()) throw new SolrException(ErrorCode.BAD_REQUEST,
+ formatString("no supported values provided rule, snitch, masShardsPerNode, replicationFactor"));
+ req.getParams().required().getAll(m, COLLECTION_PROP);
+ addMapObject(m, RULE);
+ addMapObject(m, SNITCH);
+ for (String prop : MODIFIABLE_COLL_PROPS) DocCollection.verifyProp(m, prop);
+ verifyRuleParams(h.coreContainer, m);
+ return m;
+ }
};
CollectionAction action;
long timeOut;
+ boolean sendToOCPQueue;
CollectionOperation(CollectionAction action) {
- this(action, DEFAULT_ZK_TIMEOUT);
+ this(action, DEFAULT_ZK_TIMEOUT, true);
}
- CollectionOperation(CollectionAction action, long timeOut) {
+ CollectionOperation(CollectionAction action, long timeOut, boolean sendToOCPQueue) {
this.action = action;
this.timeOut = timeOut;
+ this.sendToOCPQueue = sendToOCPQueue;
+
}
/**
@@ -697,4 +717,46 @@ public class CollectionsHandler extends
}
}
+ public static void verifyRuleParams(CoreContainer cc, Map<String, Object> m) {
+ List l = (List) m.get(RULE);
+ if (l != null) {
+ for (Object o : l) {
+ Map map = (Map) o;
+ try {
+ new Rule(map);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Error in rule " + m, e);
+ }
+ }
+ }
+ ReplicaAssigner.verifySnitchConf(cc, (List) m.get(SNITCH));
+ }
+
+ /**
+ * Converts a String of the form a:b,c:d to a Map
+ */
+ private static Map<String, Object> addMapObject(Map<String, Object> props, String key) {
+ Object v = props.get(key);
+ if (v == null) return props;
+ List<String> val = new ArrayList<>();
+ if (v instanceof String[]) {
+ val.addAll(Arrays.asList((String[]) v));
+ } else {
+ val.add(v.toString());
+ }
+ if (val.size() > 0) {
+ ArrayList<Map> l = new ArrayList<>();
+ for (String rule : val) l.add(Rule.parseRule(rule));
+ props.put(key, l);
+ }
+ return props;
+ }
+
+ public static final List<String> MODIFIABLE_COLL_PROPS = ImmutableList.of(
+ RULE,
+ SNITCH,
+ REPLICATION_FACTOR,
+ MAX_SHARDS_PER_NODE,
+ AUTO_ADD_REPLICAS);
+
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java Mon May 18 16:23:19 2015
@@ -21,15 +21,24 @@ import java.util.List;
import java.util.Map;
import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
+import org.apache.solr.cloud.OverseerCollectionProcessor;
import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.core.CoreContainer;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
+import static org.apache.solr.core.CoreContainer.COLLECTIONS_HANDLER_PATH;
+
public class RulesTest extends AbstractFullDistribZkTestBase {
static final Logger log = LoggerFactory.getLogger(RulesTest.class);
@@ -62,6 +71,53 @@ public class RulesTest extends AbstractF
}
+ @Test
+ public void testModifyColl() throws Exception {
+ String rulesColl = "modifyColl";
+ try (SolrClient client = createNewSolrClient("", getBaseUrl((HttpSolrClient) clients.get(0)))) {
+ CollectionAdminResponse rsp;
+ CollectionAdminRequest.Create create = new CollectionAdminRequest.Create();
+ create.setCollectionName(rulesColl);
+ create.setNumShards(1);
+ create.setReplicationFactor(2);
+ create.setRule("cores:<4", "node:*,replica:1", "freedisk:>1");
+ create.setSnitch("class:ImplicitSnitch");
+ rsp = create.process(client);
+ assertEquals(0, rsp.getStatus());
+ assertTrue(rsp.isSuccess());
+ ModifiableSolrParams p = new ModifiableSolrParams();
+ p.add("collection", rulesColl);
+ p.add("action", "MODIFYCOLLECTION");
+ p.add("rule", "cores:<5");
+ p.add("rule", "node:*,replica:1");
+ p.add("rule", "freedisk:>5");
+ p.add("autoAddReplicas", "true");
+ client.request(new GenericSolrRequest(POST, COLLECTIONS_HANDLER_PATH, p));
+ }
+
+
+ for (int i = 0; i < 20; i++) {
+ DocCollection rulesCollection = ZkStateReader.getCollectionLive(cloudClient.getZkStateReader(), rulesColl);
+ log.info("version_of_coll {} ", rulesCollection.getZNodeVersion());
+ List list = (List) rulesCollection.get("rule");
+ assertEquals(3, list.size());
+ if (!"<5".equals(((Map) list.get(0)).get("cores"))) {
+ if (i < 19) {
+ Thread.sleep(100);
+ continue;
+ }
+
+ }
+ assertEquals("<5", ((Map) list.get(0)).get("cores"));
+ assertEquals("1", ((Map) list.get(1)).get("replica"));
+ assertEquals(">5", ((Map) list.get(2)).get("freedisk"));
+ assertEquals("true", String.valueOf(rulesCollection.getProperties().get("autoAddReplicas")));
+ list = (List) rulesCollection.get("snitch");
+ assertEquals(1, list.size());
+ assertEquals("ImplicitSnitch", ((Map) list.get(0)).get("class"));
+ }
+ }
+
}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/cloud/DocCollection.java Mon May 18 16:23:19 2015
@@ -21,6 +21,7 @@ import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import org.apache.solr.common.SolrException;
@@ -28,6 +29,10 @@ import org.apache.solr.common.SolrExcept
import org.noggit.JSONUtil;
import org.noggit.JSONWriter;
+import static org.apache.solr.common.cloud.ZkStateReader.AUTO_ADD_REPLICAS;
+import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
+import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR;
+
/**
* Models a Collection in zookeeper (but that Java name is obviously taken, hence "DocCollection")
*/
@@ -35,6 +40,8 @@ public class DocCollection extends ZkNod
public static final String DOC_ROUTER = "router";
public static final String SHARDS = "shards";
public static final String STATE_FORMAT = "stateFormat";
+ public static final String RULE = "rule";
+ public static final String SNITCH = "snitch";
private int znodeVersion = -1; // sentinel
private final String name;
@@ -45,7 +52,7 @@ public class DocCollection extends ZkNod
private final Integer replicationFactor;
private final Integer maxShardsPerNode;
- private final boolean autoAddReplicas;
+ private final Boolean autoAddReplicas;
public DocCollection(String name, Map<String, Slice> slices, Map<String, Object> props, DocRouter router) {
@@ -64,25 +71,12 @@ public class DocCollection extends ZkNod
this.slices = slices;
this.activeSlices = new HashMap<>();
- Object replicationFactorObject = (Object) props.get(ZkStateReader.REPLICATION_FACTOR);
- if (replicationFactorObject != null) {
- this.replicationFactor = Integer.parseInt(replicationFactorObject.toString());
- } else {
- this.replicationFactor = null;
- }
- Object maxShardsPerNodeObject = (Object) props.get(ZkStateReader.MAX_SHARDS_PER_NODE);
- if (maxShardsPerNodeObject != null) {
- this.maxShardsPerNode = Integer.parseInt(maxShardsPerNodeObject.toString());
- } else {
- this.maxShardsPerNode = null;
- }
- Object autoAddReplicasObject = (Object) props.get(ZkStateReader.AUTO_ADD_REPLICAS);
- if (autoAddReplicasObject != null) {
- this.autoAddReplicas = Boolean.parseBoolean(autoAddReplicasObject.toString());
- } else {
- this.autoAddReplicas = false;
- }
-
+ this.replicationFactor = (Integer) verifyProp(props, REPLICATION_FACTOR);
+ this.maxShardsPerNode = (Integer) verifyProp(props, MAX_SHARDS_PER_NODE);
+ Boolean autoAddReplicas = (Boolean) verifyProp(props, AUTO_ADD_REPLICAS);
+ this.autoAddReplicas = autoAddReplicas == null ? false : autoAddReplicas;
+ verifyProp(props, RULE);
+ verifyProp(props, SNITCH);
Iterator<Map.Entry<String, Slice>> iter = slices.entrySet().iterator();
while (iter.hasNext()) {
@@ -95,6 +89,24 @@ public class DocCollection extends ZkNod
assert name != null && slices != null;
}
+ public static Object verifyProp(Map<String, Object> props, String propName) {
+ Object o = props.get(propName);
+ if (o == null) return null;
+ switch (propName) {
+ case MAX_SHARDS_PER_NODE:
+ case REPLICATION_FACTOR:
+ return Integer.parseInt(o.toString());
+ case AUTO_ADD_REPLICAS:
+ return Boolean.parseBoolean(o.toString());
+ case "snitch":
+ case "rule":
+ return (List) o;
+ default:
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Unknown property " + propName);
+ }
+
+ }
+
/**Use this to make an exact copy of DocCollection with a new set of Slices and every other property as is
* @param slices the new set of Slices
* @return the resulting DocCollection
@@ -164,7 +176,7 @@ public class DocCollection extends ZkNod
public int getMaxShardsPerNode() {
if (maxShardsPerNode == null) {
- throw new SolrException(ErrorCode.BAD_REQUEST, ZkStateReader.MAX_SHARDS_PER_NODE + " is not in the cluster state.");
+ throw new SolrException(ErrorCode.BAD_REQUEST, MAX_SHARDS_PER_NODE + " is not in the cluster state.");
}
return maxShardsPerNode;
}
Modified: lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java?rev=1680052&r1=1680051&r2=1680052&view=diff
==============================================================================
--- lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java (original)
+++ lucene/dev/trunk/solr/solrj/src/java/org/apache/solr/common/params/CollectionParams.java Mon May 18 16:23:19 2015
@@ -19,6 +19,8 @@ package org.apache.solr.common.params;
import java.util.Locale;
+import org.apache.solr.common.SolrException;
+
public interface CollectionParams
{
/** What action **/
@@ -50,17 +52,17 @@ public interface CollectionParams
ADDREPLICAPROP,
DELETEREPLICAPROP,
BALANCESHARDUNIQUE,
- REBALANCELEADERS;
-
- public static CollectionAction get( String p )
- {
+ REBALANCELEADERS,
+ MODIFYCOLLECTION;
+
+ public static CollectionAction get(String p) {
if( p != null ) {
try {
return CollectionAction.valueOf( p.toUpperCase(Locale.ROOT) );
}
catch( Exception ex ) {}
}
- return null;
+ return null;
}
public boolean isEqual(String s){
if(s == null) return false;