You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gc...@apache.org on 2015/08/27 07:16:09 UTC
svn commit: r1698079 [1/2] - in /lucene/dev/branches/branch_5x/solr: ./
core/src/java/org/apache/solr/cloud/ core/src/java/org/apache/solr/core/
core/src/java/org/apache/solr/handler/
core/src/java/org/apache/solr/handler/admin/ core/src/test-files/sol...
Author: gchanan
Date: Thu Aug 27 05:16:07 2015
New Revision: 1698079
URL: http://svn.apache.org/r1698079
Log:
SOLR-7789: Introduce a ConfigSet management API
Added:
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java
- copied, changed from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java (with props)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java
- copied, changed from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java
- copied, changed from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionQueue.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java (with props)
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java
- copied, changed from r1698058, lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java
- copied, changed from r1698058, lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionQueueTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java (with props)
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIExclusivity.java (with props)
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPIZkFailure.java (with props)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/request/ConfigSetAdminRequest.java (with props)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/client/solrj/response/ConfigSetAdminResponse.java (with props)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/ConfigSetParams.java (with props)
lucene/dev/branches/branch_5x/solr/solrj/src/test/org/apache/solr/client/solrj/request/TestConfigSetAdminRequest.java (with props)
Removed:
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionQueue.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionQueueTest.java
Modified:
lucene/dev/branches/branch_5x/solr/CHANGES.txt
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreContainer.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/NodeConfig.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java
lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/solr-50-all.xml
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/RollingRestartTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/SimpleCollectionCreateDeleteTest.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestLeaderElectionZkExpiry.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestCoreContainer.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/core/TestSolrXml.java
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkConfigManager.java
lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/ZkTestServer.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Thu Aug 27 05:16:07 2015
@@ -52,6 +52,8 @@ New Features
* SOLR-7961: Print Solr's version with command bin/solr version (janhoy)
+* SOLR-7789: Introduce a ConfigSet management API (Gregory Chanan)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/LeaderElector.java Thu Aug 27 05:16:07 2015
@@ -280,7 +280,7 @@ public class LeaderElector {
try {
if(joinAtHead){
log.info("Node {} trying to join election at the head", id);
- List<String> nodes = OverseerProcessor.getSortedElectionNodes(zkClient, shardsElectZkPath);
+ List<String> nodes = OverseerTaskProcessor.getSortedElectionNodes(zkClient, shardsElectZkPath);
if(nodes.size() <2){
leaderSeqPath = zkClient.create(shardsElectZkPath + "/" + id + "-n_", null,
CreateMode.EPHEMERAL_SEQUENTIAL, false);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Thu Aug 27 05:16:07 2015
@@ -309,7 +309,7 @@ public class Overseer implements Closeab
try {
Map m = (Map) Utils.fromJSON(data);
String id = (String) m.get("id");
- if(overseerCollectionProcessor.getId().equals(id)){
+ if(overseerCollectionConfigSetProcessor.getId().equals(id)){
try {
log.info("I'm exiting , but I'm still the leader");
zkClient.delete(path,stat.getVersion(),true);
@@ -390,7 +390,7 @@ public class Overseer implements Closeab
case QUIT:
if (myId.equals(message.get("id"))) {
log.info("Quit command received {}", LeaderElector.getNodeName(myId));
- overseerCollectionProcessor.close();
+ overseerCollectionConfigSetProcessor.close();
close();
} else {
log.warn("Overseer received wrong QUIT message {}", message);
@@ -786,7 +786,7 @@ public class Overseer implements Closeab
private final String adminPath;
- private OverseerCollectionProcessor overseerCollectionProcessor;
+ private OverseerCollectionConfigSetProcessor overseerCollectionConfigSetProcessor;
private ZkController zkController;
@@ -824,8 +824,8 @@ public class Overseer implements Closeab
ThreadGroup ccTg = new ThreadGroup("Overseer collection creation process.");
OverseerNodePrioritizer overseerPrioritizer = new OverseerNodePrioritizer(reader, adminPath, shardHandler.getShardHandlerFactory());
- overseerCollectionProcessor = new OverseerCollectionProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer);
- ccThread = new OverseerThread(ccTg, overseerCollectionProcessor, "OverseerCollectionProcessor-" + id);
+ overseerCollectionConfigSetProcessor = new OverseerCollectionConfigSetProcessor(reader, id, shardHandler, adminPath, stats, Overseer.this, overseerPrioritizer);
+ ccThread = new OverseerThread(ccTg, overseerCollectionConfigSetProcessor, "OverseerCollectionConfigSetProcessor-" + id);
ccThread.setDaemon(true);
ThreadGroup ohcfTg = new ThreadGroup("Overseer Hdfs SolrCore Failover Thread.");
@@ -922,15 +922,27 @@ public class Overseer implements Closeab
}
/* Collection creation queue */
- static OverseerCollectionQueue getCollectionQueue(final SolrZkClient zkClient) {
+ static OverseerTaskQueue getCollectionQueue(final SolrZkClient zkClient) {
return getCollectionQueue(zkClient, new Stats());
}
- static OverseerCollectionQueue getCollectionQueue(final SolrZkClient zkClient, Stats zkStats) {
+ static OverseerTaskQueue getCollectionQueue(final SolrZkClient zkClient, Stats zkStats) {
createOverseerNode(zkClient);
- return new OverseerCollectionQueue(zkClient, "/overseer/collection-queue-work", zkStats);
+ return new OverseerTaskQueue(zkClient, "/overseer/collection-queue-work", zkStats);
}
-
+
+ /* The queue for ConfigSet related operations */
+ static OverseerTaskQueue getConfigSetQueue(final SolrZkClient zkClient) {
+ return getConfigSetQueue(zkClient, new Stats());
+ }
+
+ static OverseerTaskQueue getConfigSetQueue(final SolrZkClient zkClient, Stats zkStats) {
+ // For now, we use the same queue as the collection queue, but ensure
+ // that the actions are prefixed with a unique string.
+ createOverseerNode(zkClient);
+ return getCollectionQueue(zkClient, zkStats);
+ }
+
private static void createOverseerNode(final SolrZkClient zkClient) {
try {
zkClient.create("/overseer", new byte[0], CreateMode.PERSISTENT, true);
Copied: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java (from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java?p2=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java&p1=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java&r1=1698058&r2=1698079&rev=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionConfigSetProcessor.java Thu Aug 27 05:16:07 2015
@@ -21,14 +21,16 @@ import org.apache.solr.common.cloud.ZkNo
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.handler.component.ShardHandler;
import org.apache.solr.handler.component.ShardHandlerFactory;
+import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.CONFIGSETS_ACTION_PREFIX;
/**
- * An {@link OverseerProcessor} that handles collection-related Overseer
- * messages only.
+ * An {@link OverseerTaskProcessor} that handles:
+ * 1) collection-related Overseer messages
+ * 2) configset-related Overseer messages
*/
-public class OverseerCollectionProcessor extends OverseerProcessor {
+public class OverseerCollectionConfigSetProcessor extends OverseerTaskProcessor {
- public OverseerCollectionProcessor(ZkStateReader zkStateReader, String myId,
+ public OverseerCollectionConfigSetProcessor(ZkStateReader zkStateReader, String myId,
final ShardHandler shardHandler,
String adminPath, Overseer.Stats stats, Overseer overseer,
OverseerNodePrioritizer overseerNodePrioritizer) {
@@ -47,13 +49,13 @@ public class OverseerCollectionProcessor
);
}
- protected OverseerCollectionProcessor(ZkStateReader zkStateReader, String myId,
+ protected OverseerCollectionConfigSetProcessor(ZkStateReader zkStateReader, String myId,
final ShardHandlerFactory shardHandlerFactory,
String adminPath,
Overseer.Stats stats,
Overseer overseer,
OverseerNodePrioritizer overseerNodePrioritizer,
- OverseerCollectionQueue workQueue,
+ OverseerTaskQueue workQueue,
DistributedMap runningMap,
DistributedMap completedMap,
DistributedMap failureMap) {
@@ -80,12 +82,18 @@ public class OverseerCollectionProcessor
Overseer.Stats stats,
Overseer overseer,
OverseerNodePrioritizer overseerNodePrioritizer) {
- final OverseerCollectionMessageHandler messageHandler = new OverseerCollectionMessageHandler(
+ final OverseerCollectionMessageHandler collMessageHandler = new OverseerCollectionMessageHandler(
zkStateReader, myId, shardHandlerFactory, adminPath, stats, overseer, overseerNodePrioritizer);
+ final OverseerConfigSetMessageHandler configMessageHandler = new OverseerConfigSetMessageHandler(
+ zkStateReader);
return new OverseerMessageHandlerSelector() {
@Override
public OverseerMessageHandler selectOverseerMessageHandler(ZkNodeProps message) {
- return messageHandler;
+ String operation = message.getStr(Overseer.QUEUE_OPERATION);
+ if (operation != null && operation.startsWith(CONFIGSETS_ACTION_PREFIX)) {
+ return configMessageHandler;
+ }
+ return collMessageHandler;
}
};
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java Thu Aug 27 05:16:07 2015
@@ -121,7 +121,10 @@ import static org.apache.solr.common.par
import static org.apache.solr.common.util.StrUtils.formatString;
import static org.apache.solr.common.util.Utils.makeMap;
-
+/**
+ * A {@link OverseerMessageHandler} that handles Collections API related
+ * overseer messages.
+ */
public class OverseerCollectionMessageHandler implements OverseerMessageHandler {
public static final String NUM_SLICES = "numShards";
@@ -203,7 +206,7 @@ public class OverseerCollectionMessageHa
@Override
@SuppressWarnings("unchecked")
public SolrResponse processMessage(ZkNodeProps message, String operation) {
- log.warn("OverseerCollectionProcessor.processMessage : "+ operation + " , "+ message.toString());
+ log.warn("OverseerCollectionMessageHandler.processMessage : "+ operation + " , "+ message.toString());
NamedList results = new NamedList();
try {
@@ -371,7 +374,7 @@ public class OverseerCollectionMessageHa
@SuppressWarnings("unchecked")
private void getOverseerStatus(ZkNodeProps message, NamedList results) throws KeeperException, InterruptedException {
- String leaderNode = OverseerProcessor.getLeaderNode(zkStateReader.getZkClient());
+ String leaderNode = OverseerTaskProcessor.getLeaderNode(zkStateReader.getZkClient());
results.add("leader", leaderNode);
Stat stat = new Stat();
zkStateReader.getZkClient().getData("/overseer/queue",null, stat, true);
@@ -2470,7 +2473,7 @@ public class OverseerCollectionMessageHa
@Override
public String getName() {
- return "Overseer Collection Processor";
+ return "Overseer Collection Message Handler";
}
@Override
@@ -2495,7 +2498,7 @@ public class OverseerCollectionMessageHa
}
@Override
- public void unmarkExclusiveTask(String collectionName, String operation) {
+ public void unmarkExclusiveTask(String collectionName, String operation, ZkNodeProps message) {
if(!CLUSTERSTATUS.isEqual(operation) && collectionName != null) {
synchronized (collectionWip) {
collectionWip.remove(collectionName);
@@ -2510,8 +2513,10 @@ public class OverseerCollectionMessageHa
if(CLUSTERSTATUS.isEqual(message.getStr(Overseer.QUEUE_OPERATION)))
return ExclusiveMarking.EXCLUSIVE;
- if(collectionWip.contains(collectionName))
- return ExclusiveMarking.NONEXCLUSIVE;
+ synchronized (collectionWip) {
+ if(collectionWip.contains(collectionName))
+ return ExclusiveMarking.NONEXCLUSIVE;
+ }
return ExclusiveMarking.NOTDETERMINED;
}
Added: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java?rev=1698079&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java (added)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerConfigSetMessageHandler.java Thu Aug 27 05:16:07 2015
@@ -0,0 +1,365 @@
+package org.apache.solr.cloud;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.ConfigSetParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.ConfigSetProperties;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.noggit.JSONUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.cloud.OverseerMessageHandler.ExclusiveMarking.NONEXCLUSIVE;
+import static org.apache.solr.cloud.OverseerMessageHandler.ExclusiveMarking.NOTDETERMINED;
+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.CREATE;
+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.DELETE;
+import static org.apache.solr.common.params.CommonParams.NAME;
+
+/**
+ * A {@link OverseerMessageHandler} that handles ConfigSets API related
+ * overseer messages.
+ */
+public class OverseerConfigSetMessageHandler implements OverseerMessageHandler {
+
+ /**
+ * Prefix to specify an action should be handled by this handler.
+ */
+ public static final String CONFIGSETS_ACTION_PREFIX = "configsets:";
+
+ /**
+ * Name of the ConfigSet to copy from for CREATE
+ */
+ public static final String BASE_CONFIGSET = "baseConfigSet";
+
+ /**
+ * Prefix for properties that should be applied to the ConfigSet for CREATE
+ */
+ public static final String PROPERTY_PREFIX = "configSetProp";
+
+ private ZkStateReader zkStateReader;
+
+ // we essentially implement a read/write lock for the ConfigSet exclusivity as follows:
+ // WRITE: CREATE/DELETE on the ConfigSet under operation
+ // READ: for the Base ConfigSet being copied in CREATE.
+ // in this way, we prevent a Base ConfigSet from being deleted while it is being copied
+ // but don't prevent different ConfigSets from being created with the same Base ConfigSet
+ // at the same time.
+ final private Set configSetWriteWip;
+ final private Set configSetReadWip;
+
+ private static Logger log = LoggerFactory
+ .getLogger(OverseerConfigSetMessageHandler.class);
+
+ public OverseerConfigSetMessageHandler(ZkStateReader zkStateReader) {
+ this.zkStateReader = zkStateReader;
+ this.configSetWriteWip = new HashSet();
+ this.configSetReadWip = new HashSet();
+ }
+
+ @Override
+ public SolrResponse processMessage(ZkNodeProps message, String operation) {
+ NamedList results = new NamedList();
+ try {
+ if (!operation.startsWith(CONFIGSETS_ACTION_PREFIX)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Operation does not contain proper prefix: " + operation
+ + " expected: " + CONFIGSETS_ACTION_PREFIX);
+ }
+ operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());
+ log.info("OverseerConfigSetMessageHandler.processMessage : "+ operation + " , "+ message.toString());
+
+ ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
+ if (action == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:" + operation);
+ }
+ switch (action) {
+ case CREATE:
+ createConfigSet(message);
+ break;
+ case DELETE:
+ deleteConfigSet(message);
+ break;
+ default:
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Unknown operation:"
+ + operation);
+ }
+ } catch (Exception e) {
+ String configSetName = message.getStr(NAME);
+
+ if (configSetName == null) {
+ SolrException.log(log, "Operation " + operation + " failed", e);
+ } else {
+ SolrException.log(log, "ConfigSet: " + configSetName + " operation: " + operation
+ + " failed", e);
+ }
+
+ results.add("Operation " + operation + " caused exception:", e);
+ SimpleOrderedMap nl = new SimpleOrderedMap();
+ nl.add("msg", e.getMessage());
+ nl.add("rspCode", e instanceof SolrException ? ((SolrException)e).code() : -1);
+ results.add("exception", nl);
+ }
+ return new OverseerSolrResponse(results);
+ }
+
+ @Override
+ public String getName() {
+ return "Overseer ConfigSet Message Handler";
+ }
+
+ @Override
+ public String getTimerName(String operation) {
+ return "configset_" + operation;
+ }
+
+ @Override
+ public String getTaskKey(ZkNodeProps message) {
+ return message.getStr(NAME);
+ }
+
+ @Override
+ public void markExclusiveTask(String configSetName, ZkNodeProps message) {
+ String baseConfigSet = getBaseConfigSetIfCreate(message);
+ markExclusive(configSetName, baseConfigSet);
+ }
+
+ private void markExclusive(String configSetName, String baseConfigSetName) {
+ synchronized (configSetWriteWip) {
+ configSetWriteWip.add(configSetName);
+ if (baseConfigSetName != null) configSetReadWip.add(baseConfigSetName);
+ }
+ }
+
+ @Override
+ public void unmarkExclusiveTask(String configSetName, String operation, ZkNodeProps message) {
+ String baseConfigSet = getBaseConfigSetIfCreate(message);
+ unmarkExclusiveConfigSet(configSetName, baseConfigSet);
+ }
+
+ private void unmarkExclusiveConfigSet(String configSetName, String baseConfigSetName) {
+ synchronized (configSetWriteWip) {
+ configSetWriteWip.remove(configSetName);
+ if (baseConfigSetName != null) configSetReadWip.remove(baseConfigSetName);
+ }
+ }
+
+ @Override
+ public ExclusiveMarking checkExclusiveMarking(String configSetName, ZkNodeProps message) {
+ String baseConfigSet = getBaseConfigSetIfCreate(message);
+ return checkExclusiveMarking(configSetName, baseConfigSet);
+ }
+
+ private ExclusiveMarking checkExclusiveMarking(String configSetName, String baseConfigSetName) {
+ synchronized (configSetWriteWip) {
+ // need to acquire:
+ // 1) write lock on ConfigSet
+ // 2) read lock on Base ConfigSet
+ if (configSetWriteWip.contains(configSetName) || configSetReadWip.contains(configSetName)) {
+ return NONEXCLUSIVE;
+ }
+ if (baseConfigSetName != null && configSetWriteWip.contains(baseConfigSetName)) {
+ return NONEXCLUSIVE;
+ }
+ }
+
+ return NOTDETERMINED;
+ }
+
+ private String getBaseConfigSetIfCreate(ZkNodeProps message) {
+ String operation = message.getStr(Overseer.QUEUE_OPERATION);
+ if (operation != null) {
+ operation = operation.substring(CONFIGSETS_ACTION_PREFIX.length());
+ ConfigSetParams.ConfigSetAction action = ConfigSetParams.ConfigSetAction.get(operation);
+ if (action == CREATE) {
+ return message.getStr(BASE_CONFIGSET);
+ }
+ }
+ return null;
+ }
+
+ private NamedList getConfigSetProperties(String path) throws IOException {
+ byte [] oldPropsData = null;
+ try {
+ oldPropsData = zkStateReader.getZkClient().getData(path, null, null, true);
+ } catch (KeeperException.NoNodeException e) {
+ log.info("no existing ConfigSet properties found");
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error reading old properties",
+ SolrZkClient.checkInterrupted(e));
+ }
+
+ if (oldPropsData != null) {
+ InputStreamReader reader = new InputStreamReader(new ByteArrayInputStream(oldPropsData), StandardCharsets.UTF_8);
+ try {
+ return ConfigSetProperties.readFromInputStream(reader);
+ } finally {
+ reader.close();
+ }
+ }
+ return null;
+ }
+
+ private Map<String, Object> getNewProperties(ZkNodeProps message) {
+ Map<String, Object> properties = null;
+ for (Map.Entry<String, Object> entry : message.getProperties().entrySet()) {
+ if (entry.getKey().startsWith(PROPERTY_PREFIX + ".")) {
+ if (properties == null) {
+ properties = new HashMap<String, Object>();
+ }
+ properties.put(entry.getKey().substring((PROPERTY_PREFIX + ".").length()),
+ entry.getValue());
+ }
+ }
+ return properties;
+ }
+
+ private void mergeOldProperties(Map<String, Object> newProps, NamedList oldProps) {
+ Iterator<Map.Entry<String, Object>> it = oldProps.iterator();
+ while (it.hasNext()) {
+ Map.Entry<String, Object> oldEntry = it.next();
+ if (!newProps.containsKey(oldEntry.getKey())) {
+ newProps.put(oldEntry.getKey(), oldEntry.getValue());
+ }
+ }
+ }
+
+ private byte[] getPropertyData(Map<String, Object> newProps) {
+ if (newProps != null) {
+ String propertyDataStr = JSONUtil.toJSON(newProps);
+ if (propertyDataStr == null) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Invalid property specification");
+ }
+ return propertyDataStr.getBytes(StandardCharsets.UTF_8);
+ }
+ return null;
+ }
+
+ private String getPropertyPath(String configName, String propertyPath) {
+ return ZkConfigManager.CONFIGS_ZKNODE + "/" + configName + "/" + propertyPath;
+ }
+
+ private void createConfigSet(ZkNodeProps message) throws IOException {
+ String configSetName = getTaskKey(message);
+ if (configSetName == null || configSetName.length() == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
+ }
+
+ String baseConfigSetName = message.getStr(BASE_CONFIGSET);
+ if (baseConfigSetName == null || baseConfigSetName.length() == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Base ConfigSet name not specified");
+ }
+
+ ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());
+ if (configManager.configExists(configSetName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet already exists: " + configSetName);
+ }
+
+ // is there a base config that already exists
+ if (!configManager.configExists(baseConfigSetName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Base ConfigSet does not exist: " + baseConfigSetName);
+ }
+
+ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
+ Map<String, Object> props = getNewProperties(message);
+ if (props != null) {
+ // read the old config properties and do a merge, if necessary
+ NamedList oldProps = getConfigSetProperties(getPropertyPath(baseConfigSetName,propertyPath));
+ if (oldProps != null) {
+ mergeOldProperties(props, oldProps);
+ }
+ }
+ byte[] propertyData = getPropertyData(props);
+
+ Set<String> copiedToZkPaths = new HashSet<String>();
+ try {
+ configManager.copyConfigDir(baseConfigSetName, configSetName, copiedToZkPaths);
+ if (propertyData != null) {
+ try {
+ zkStateReader.getZkClient().makePath(
+ getPropertyPath(configSetName, propertyPath),
+ propertyData, CreateMode.PERSISTENT, null, false, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error writing new properties",
+ SolrZkClient.checkInterrupted(e));
+ }
+ }
+ } catch (Exception e) {
+ // copying the config dir or writing the properties file may have failed.
+ // we should delete the ConfigSet because it may be invalid,
+ // assuming we actually wrote something. E.g. could be
+ // the entire baseConfig set with the old properties, including immutable,
+ // that would make it impossible for the user to delete.
+ try {
+ if (configManager.configExists(configSetName) && copiedToZkPaths.size() > 0) {
+ deleteConfigSet(configSetName, true);
+ }
+ } catch (IOException ioe) {
+ log.error("Error while trying to delete partially created ConfigSet", ioe);
+ }
+ throw e;
+ }
+ }
+
+ private void deleteConfigSet(ZkNodeProps message) throws IOException {
+ String configSetName = getTaskKey(message);
+ if (configSetName == null || configSetName.length() == 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet name not specified");
+ }
+
+ deleteConfigSet(configSetName, false);
+ }
+
+ private void deleteConfigSet(String configSetName, boolean force) throws IOException {
+ ZkConfigManager configManager = new ZkConfigManager(zkStateReader.getZkClient());
+ if (!configManager.configExists(configSetName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "ConfigSet does not exist to delete: " + configSetName);
+ }
+
+ String propertyPath = ConfigSetProperties.DEFAULT_FILENAME;
+ NamedList properties = getConfigSetProperties(getPropertyPath(configSetName, propertyPath));
+ if (properties != null) {
+ Object immutable = properties.get(ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG);
+ boolean isImmutableConfigSet = immutable != null ? Boolean.parseBoolean(immutable.toString()) : false;
+ if (!force && isImmutableConfigSet) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Requested delete of immutable ConfigSet: " + configSetName);
+ }
+ }
+ configManager.deleteConfigDir(configSetName);
+ }
+}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerMessageHandler.java Thu Aug 27 05:16:07 2015
@@ -21,7 +21,7 @@ import org.apache.solr.client.solrj.Solr
import org.apache.solr.common.cloud.ZkNodeProps;
/**
- * Interface for processing messages received by an {@link OverseerProcessor}
+ * Interface for processing messages received by an {@link OverseerTaskProcessor}
*/
public interface OverseerMessageHandler {
@@ -61,8 +61,9 @@ public interface OverseerMessageHandler
/**
* @param taskKey the key associated with the task
* @param operation the operation being processed
+ * @param message the message being processed
*/
- void unmarkExclusiveTask(String taskKey, String operation);
+ void unmarkExclusiveTask(String taskKey, String operation, ZkNodeProps message);
/**
* @param taskKey the key associated with the task
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerNodePrioritizer.java Thu Aug 27 05:16:07 2015
@@ -61,10 +61,10 @@ public class OverseerNodePrioritizer {
List overseerDesignates = (List) m.get("overseer");
if(overseerDesignates==null || overseerDesignates.isEmpty()) return;
- String ldr = OverseerProcessor.getLeaderNode(zk);
+ String ldr = OverseerTaskProcessor.getLeaderNode(zk);
if(overseerDesignates.contains(ldr)) return;
log.info("prioritizing overseer nodes at {} overseer designates are {}", overseerId, overseerDesignates);
- List<String> electionNodes = OverseerProcessor.getSortedElectionNodes(zk, OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE);
+ List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zk, OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE);
if(electionNodes.size()<2) return;
log.info("sorted nodes {}", electionNodes);
@@ -89,7 +89,7 @@ public class OverseerNodePrioritizer {
//now ask the current leader to QUIT , so that the designate can takeover
Overseer.getInQueue(zkStateReader.getZkClient()).offer(
Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(),
- "id", OverseerProcessor.getLeaderId(zkStateReader.getZkClient()))));
+ "id", OverseerTaskProcessor.getLeaderId(zkStateReader.getZkClient()))));
}
Copied: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java (from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerProcessor.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java?p2=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java&p1=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerProcessor.java&r1=1698058&r2=1698079&rev=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskProcessor.java Thu Aug 27 05:16:07 2015
@@ -29,7 +29,7 @@ import java.util.concurrent.SynchronousQ
import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrResponse;
-import org.apache.solr.cloud.OverseerCollectionQueue.QueueEvent;
+import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.Overseer.LeaderStatus;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -56,16 +56,16 @@ import static org.apache.solr.common.par
* {@link OverseerMessageHandler} handles specific messages in the
* queue.
*/
-public class OverseerProcessor implements Runnable, Closeable {
+public class OverseerTaskProcessor implements Runnable, Closeable {
public int maxParallelThreads = 10;
public ExecutorService tpe ;
private static Logger log = LoggerFactory
- .getLogger(OverseerProcessor.class);
+ .getLogger(OverseerTaskProcessor.class);
- private OverseerCollectionQueue workQueue;
+ private OverseerTaskQueue workQueue;
private DistributedMap runningMap;
private DistributedMap completedMap;
private DistributedMap failureMap;
@@ -98,13 +98,13 @@ public class OverseerProcessor implement
private OverseerNodePrioritizer prioritizer;
- public OverseerProcessor(ZkStateReader zkStateReader, String myId,
+ public OverseerTaskProcessor(ZkStateReader zkStateReader, String myId,
final ShardHandlerFactory shardHandlerFactory,
String adminPath,
Overseer.Stats stats,
OverseerMessageHandlerSelector selector,
OverseerNodePrioritizer prioritizer,
- OverseerCollectionQueue workQueue,
+ OverseerTaskQueue workQueue,
DistributedMap runningMap,
DistributedMap completedMap,
DistributedMap failureMap) {
@@ -460,7 +460,7 @@ public class OverseerProcessor implement
log.debug("Completed task:[{}]", head.getId());
}
- markTaskComplete(messageHandler, head.getId(), asyncId, taskKey);
+ markTaskComplete(messageHandler, head.getId(), asyncId, taskKey, message);
log.debug("Marked task [{}] as completed.", head.getId());
printTrackingMaps();
@@ -471,13 +471,13 @@ public class OverseerProcessor implement
SolrException.log(log, "", e);
} catch (InterruptedException e) {
// Reset task from tracking data structures so that it can be retried.
- resetTaskWithException(messageHandler, head.getId(), asyncId, taskKey);
+ resetTaskWithException(messageHandler, head.getId(), asyncId, taskKey, message);
log.warn("Resetting task {} as the thread was interrupted.", head.getId());
Thread.currentThread().interrupt();
} finally {
if(!success) {
// Reset task from tracking data structures so that it can be retried.
- resetTaskWithException(messageHandler, head.getId(), asyncId, taskKey);
+ resetTaskWithException(messageHandler, head.getId(), asyncId, taskKey, message);
}
synchronized (waitLock){
waitLock.notifyAll();
@@ -485,7 +485,7 @@ public class OverseerProcessor implement
}
}
- private void markTaskComplete(OverseerMessageHandler messageHandler, String id, String asyncId, String taskKey)
+ private void markTaskComplete(OverseerMessageHandler messageHandler, String id, String asyncId, String taskKey, ZkNodeProps message)
throws KeeperException, InterruptedException {
synchronized (completedTasks) {
completedTasks.put(id, head);
@@ -498,10 +498,10 @@ public class OverseerProcessor implement
if(asyncId != null)
runningMap.remove(asyncId);
- messageHandler.unmarkExclusiveTask(taskKey, operation);
+ messageHandler.unmarkExclusiveTask(taskKey, operation, message);
}
- private void resetTaskWithException(OverseerMessageHandler messageHandler, String id, String asyncId, String taskKey) {
+ private void resetTaskWithException(OverseerMessageHandler messageHandler, String id, String asyncId, String taskKey, ZkNodeProps message) {
log.warn("Resetting task: {}, requestid: {}, taskKey: {}", id, asyncId, taskKey);
try {
if (asyncId != null)
@@ -511,7 +511,7 @@ public class OverseerProcessor implement
runningTasks.remove(id);
}
- messageHandler.unmarkExclusiveTask(taskKey, operation);
+ messageHandler.unmarkExclusiveTask(taskKey, operation, message);
} catch (KeeperException e) {
SolrException.log(log, "", e);
} catch (InterruptedException e) {
Copied: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java (from r1698058, lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionQueue.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java?p2=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java&p1=lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionQueue.java&r1=1698058&r2=1698079&rev=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionQueue.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerTaskQueue.java Thu Aug 27 05:16:07 2015
@@ -33,20 +33,20 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * A {@link DistributedQueue} augmented with helper methods specific to the collection queue.
+ * A {@link DistributedQueue} augmented with helper methods specific to the overseer task queues.
* Methods specific to this subclass ignore superclass internal state and hit ZK directly.
* This is inefficient! But the API on this class is kind of muddy..
*/
-public class OverseerCollectionQueue extends DistributedQueue {
- private static final Logger LOG = LoggerFactory.getLogger(OverseerCollectionQueue.class);
+public class OverseerTaskQueue extends DistributedQueue {
+ private static final Logger LOG = LoggerFactory.getLogger(OverseerTaskQueue.class);
private final String response_prefix = "qnr-" ;
- public OverseerCollectionQueue(SolrZkClient zookeeper, String dir) {
+ public OverseerTaskQueue(SolrZkClient zookeeper, String dir) {
this(zookeeper, dir, new Overseer.Stats());
}
- public OverseerCollectionQueue(SolrZkClient zookeeper, String dir, Overseer.Stats stats) {
+ public OverseerTaskQueue(SolrZkClient zookeeper, String dir, Overseer.Stats stats) {
super(zookeeper, dir, stats);
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkController.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkController.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkController.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkController.java Thu Aug 27 05:16:07 2015
@@ -122,7 +122,8 @@ public final class ZkController {
private final boolean SKIP_AUTO_RECOVERY = Boolean.getBoolean("solrcloud.skip.autorecovery");
private final DistributedQueue overseerJobQueue;
- private final OverseerCollectionQueue overseerCollectionQueue;
+ private final OverseerTaskQueue overseerCollectionQueue;
+ private final OverseerTaskQueue overseerConfigSetQueue;
private final DistributedMap overseerRunningMap;
private final DistributedMap overseerCompletedMap;
@@ -382,6 +383,7 @@ public final class ZkController {
this.overseerJobQueue = Overseer.getInQueue(zkClient);
this.overseerCollectionQueue = Overseer.getCollectionQueue(zkClient);
+ this.overseerConfigSetQueue = Overseer.getConfigSetQueue(zkClient);
this.overseerRunningMap = Overseer.getRunningMap(zkClient);
this.overseerCompletedMap = Overseer.getCompletedMap(zkClient);
this.overseerFailureMap = Overseer.getFailureMap(zkClient);
@@ -1777,10 +1779,14 @@ public final class ZkController {
return overseerJobQueue;
}
- public OverseerCollectionQueue getOverseerCollectionQueue() {
+ public OverseerTaskQueue getOverseerCollectionQueue() {
return overseerCollectionQueue;
}
+ public OverseerTaskQueue getOverseerConfigSetQueue() {
+ return overseerConfigSetQueue;
+ }
+
public DistributedMap getOverseerRunningMap() {
return overseerRunningMap;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/ConfigSetProperties.java Thu Aug 27 05:16:07 2015
@@ -36,6 +36,9 @@ public class ConfigSetProperties {
private static final Logger log = LoggerFactory.getLogger(ConfigSetProperties.class);
+ public static final String DEFAULT_FILENAME = "configsetprops.json";
+ public static final String IMMUTABLE_CONFIGSET_ARG = "immutable";
+
/**
* Return the properties associated with the ConfigSet (e.g. immutable)
*
@@ -55,9 +58,18 @@ public class ConfigSetProperties {
}
try {
+ return readFromInputStream(reader);
+ } finally {
+ IOUtils.closeQuietly(reader);
+ }
+ }
+
+ public static NamedList readFromInputStream(InputStreamReader reader) {
+ try {
JSONParser jsonParser = new JSONParser(reader);
Object object = ObjectBuilder.getVal(jsonParser);
if (!(object instanceof Map)) {
+ final String objectClass = object == null ? "null" : object.getClass().getName();
throw new SolrException(ErrorCode.SERVER_ERROR, "Invalid JSON type " + object.getClass().getName() + ", expected Map");
}
return new NamedList((Map)object);
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreContainer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreContainer.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreContainer.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreContainer.java Thu Aug 27 05:16:07 2015
@@ -46,6 +46,7 @@ import org.apache.solr.common.util.IOUti
import org.apache.solr.common.util.Utils;
import org.apache.solr.handler.RequestHandlerBase;
import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.handler.admin.ConfigSetsHandler;
import org.apache.solr.handler.admin.CoreAdminHandler;
import org.apache.solr.handler.admin.InfoHandler;
import org.apache.solr.handler.admin.SecurityConfHandler;
@@ -97,6 +98,7 @@ public class CoreContainer {
protected CoreAdminHandler coreAdminHandler = null;
protected CollectionsHandler collectionsHandler = null;
private InfoHandler infoHandler;
+ protected ConfigSetsHandler configSetsHandler = null;
private PKIAuthenticationPlugin pkiAuthenticationPlugin;
@@ -129,6 +131,7 @@ public class CoreContainer {
public static final String CORES_HANDLER_PATH = "/admin/cores";
public static final String COLLECTIONS_HANDLER_PATH = "/admin/collections";
public static final String INFO_HANDLER_PATH = "/admin/info";
+ public static final String CONFIGSETS_HANDLER_PATH = "/admin/configs";
private PluginBag<SolrRequestHandler> containerHandlers = new PluginBag<>(SolrRequestHandler.class, null);
@@ -407,6 +410,8 @@ public class CoreContainer {
containerHandlers.put(INFO_HANDLER_PATH, infoHandler);
coreAdminHandler = createHandler(cfg.getCoreAdminHandlerClass(), CoreAdminHandler.class);
containerHandlers.put(CORES_HANDLER_PATH, coreAdminHandler);
+ configSetsHandler = createHandler(cfg.getConfigSetsHandlerClass(), ConfigSetsHandler.class);
+ containerHandlers.put(CONFIGSETS_HANDLER_PATH, configSetsHandler);
containerHandlers.put("/admin/authorization", securityConfHandler);
containerHandlers.put("/admin/authentication", securityConfHandler);
if(pkiAuthenticationPlugin != null)
@@ -1040,6 +1045,10 @@ public class CoreContainer {
return infoHandler;
}
+ public ConfigSetsHandler getConfigSetsHandler() {
+ return configSetsHandler;
+ }
+
public String getHostName() {
return this.hostName;
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/CoreDescriptor.java Thu Aug 27 05:16:07 2015
@@ -84,7 +84,7 @@ public class CoreDescriptor {
private static ImmutableMap<String, String> defaultProperties = new ImmutableMap.Builder<String, String>()
.put(CORE_CONFIG, "solrconfig.xml")
.put(CORE_SCHEMA, "schema.xml")
- .put(CORE_CONFIGSET_PROPERTIES, "configsetprops.json")
+ .put(CORE_CONFIGSET_PROPERTIES, ConfigSetProperties.DEFAULT_FILENAME)
.put(CORE_DATADIR, "data" + File.separator)
.put(CORE_TRANSIENT, "false")
.put(CORE_LOADONSTARTUP, "true")
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/NodeConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/NodeConfig.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/NodeConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/NodeConfig.java Thu Aug 27 05:16:07 2015
@@ -44,6 +44,8 @@ public class NodeConfig {
private final String infoHandlerClass;
+ private final String configSetsHandlerClass;
+
private final LogWatcherConfig logWatcherConfig;
private final CloudConfig cloudConfig;
@@ -58,7 +60,8 @@ public class NodeConfig {
private NodeConfig(String nodeName, String coreRootDirectory, String configSetBaseDirectory, String sharedLibDirectory,
PluginInfo shardHandlerFactoryConfig, UpdateShardHandlerConfig updateShardHandlerConfig,
- String coreAdminHandlerClass, String collectionsAdminHandlerClass, String infoHandlerClass,
+ String coreAdminHandlerClass, String collectionsAdminHandlerClass,
+ String infoHandlerClass, String configSetsHandlerClass,
LogWatcherConfig logWatcherConfig, CloudConfig cloudConfig, int coreLoadThreads,
int transientCacheSize, boolean useSchemaCache, String managementPath,
SolrResourceLoader loader, Properties solrProperties) {
@@ -71,6 +74,7 @@ public class NodeConfig {
this.coreAdminHandlerClass = coreAdminHandlerClass;
this.collectionsAdminHandlerClass = collectionsAdminHandlerClass;
this.infoHandlerClass = infoHandlerClass;
+ this.configSetsHandlerClass = configSetsHandlerClass;
this.logWatcherConfig = logWatcherConfig;
this.cloudConfig = cloudConfig;
this.coreLoadThreads = coreLoadThreads;
@@ -142,6 +146,10 @@ public class NodeConfig {
return infoHandlerClass;
}
+ public String getConfigSetsHandlerClass() {
+ return configSetsHandlerClass;
+ }
+
public boolean hasSchemaCache() {
return useSchemaCache;
}
@@ -187,6 +195,7 @@ public class NodeConfig {
private String coreAdminHandlerClass = DEFAULT_ADMINHANDLERCLASS;
private String collectionsAdminHandlerClass = DEFAULT_COLLECTIONSHANDLERCLASS;
private String infoHandlerClass = DEFAULT_INFOHANDLERCLASS;
+ private String configSetsHandlerClass = DEFAULT_CONFIGSETSHANDLERCLASS;
private LogWatcherConfig logWatcherConfig = new LogWatcherConfig(true, null, null, 50);
private CloudConfig cloudConfig;
private int coreLoadThreads = DEFAULT_CORE_LOAD_THREADS;
@@ -205,6 +214,7 @@ public class NodeConfig {
private static final String DEFAULT_ADMINHANDLERCLASS = "org.apache.solr.handler.admin.CoreAdminHandler";
private static final String DEFAULT_INFOHANDLERCLASS = "org.apache.solr.handler.admin.InfoHandler";
private static final String DEFAULT_COLLECTIONSHANDLERCLASS = "org.apache.solr.handler.admin.CollectionsHandler";
+ private static final String DEFAULT_CONFIGSETSHANDLERCLASS = "org.apache.solr.handler.admin.ConfigSetsHandler";
public NodeConfigBuilder(String nodeName, SolrResourceLoader loader) {
this.nodeName = nodeName;
@@ -252,6 +262,11 @@ public class NodeConfig {
return this;
}
+ public NodeConfigBuilder setConfigSetsHandlerClass(String configSetsHandlerClass) {
+ this.configSetsHandlerClass = configSetsHandlerClass;
+ return this;
+ }
+
public NodeConfigBuilder setLogWatcherConfig(LogWatcherConfig logWatcherConfig) {
this.logWatcherConfig = logWatcherConfig;
return this;
@@ -289,7 +304,7 @@ public class NodeConfig {
public NodeConfig build() {
return new NodeConfig(nodeName, coreRootDirectory, configSetBaseDirectory, sharedLibDirectory, shardHandlerFactoryConfig,
- updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, infoHandlerClass,
+ updateShardHandlerConfig, coreAdminHandlerClass, collectionsAdminHandlerClass, infoHandlerClass, configSetsHandlerClass,
logWatcherConfig, cloudConfig, coreLoadThreads, transientCacheSize, useSchemaCache, managementPath, loader, solrProperties);
}
}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/core/SolrXmlConfig.java Thu Aug 27 05:16:07 2015
@@ -234,6 +234,9 @@ public class SolrXmlConfig {
case "infoHandler":
builder.setInfoHandlerClass(value);
break;
+ case "configSetsHandler":
+ builder.setConfigSetsHandlerClass(value);
+ break;
case "coreRootDirectory":
builder.setCoreRootDirectory(value);
break;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SchemaHandler.java Thu Aug 27 05:16:07 2015
@@ -41,11 +41,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.solr.common.params.CommonParams.JSON;
+import static org.apache.solr.core.ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG;
public class SchemaHandler extends RequestHandlerBase {
private static final Logger log = LoggerFactory.getLogger(SchemaHandler.class);
-
- public static final String IMMUTABLE_CONFIGSET_ARG = "immutable";
private boolean isImmutableConfigSet = false;
@Override
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/SolrConfigHandler.java Thu Aug 27 05:16:07 2015
@@ -82,6 +82,7 @@ import static org.apache.solr.common.par
import static org.apache.solr.common.util.StrUtils.formatString;
import static org.apache.solr.core.ConfigOverlay.NOT_EDITABLE;
import static org.apache.solr.core.ConfigOverlay.ZNODEVER;
+import static org.apache.solr.core.ConfigSetProperties.IMMUTABLE_CONFIGSET_ARG;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_CLASS;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME;
import static org.apache.solr.core.SolrConfig.PluginOpts.REQUIRE_NAME_IN_OVERLAY;
@@ -91,7 +92,6 @@ public class SolrConfigHandler extends R
public static final Logger log = LoggerFactory.getLogger(SolrConfigHandler.class);
public static final String CONFIGSET_EDITING_DISABLED_ARG = "disable.configEdit";
public static final boolean configEditing_disabled = Boolean.getBoolean(CONFIGSET_EDITING_DISABLED_ARG);
- public static final String IMMUTABLE_CONFIGSET_ARG = "immutable";
private static final Map<String, SolrConfig.SolrPluginInfo> namedPlugins;
private Lock reloadLock = new ReentrantLock(true);
private boolean isImmutableConfigSet = false;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Thu Aug 27 05:16:07 2015
@@ -36,8 +36,8 @@ import org.apache.solr.client.solrj.impl
import org.apache.solr.client.solrj.request.CoreAdminRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.RequestSyncShard;
import org.apache.solr.cloud.DistributedMap;
-import org.apache.solr.cloud.OverseerCollectionQueue;
-import org.apache.solr.cloud.OverseerCollectionQueue.QueueEvent;
+import org.apache.solr.cloud.OverseerTaskQueue;
+import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.Overseer;
import org.apache.solr.cloud.OverseerSolrResponse;
import org.apache.solr.cloud.overseer.SliceMutator;
@@ -246,13 +246,13 @@ public class CollectionsHandler extends
+ event.getWatchedEvent().getType() + "]");
} else {
throw new SolrException(ErrorCode.SERVER_ERROR, operation
- + " the collection unkown case");
+ + " the collection unknown case");
}
}
}
private boolean overseerCollectionQueueContains(String asyncId) throws KeeperException, InterruptedException {
- OverseerCollectionQueue collectionQueue = coreContainer.getZkController().getOverseerCollectionQueue();
+ OverseerTaskQueue collectionQueue = coreContainer.getZkController().getOverseerCollectionQueue();
return collectionQueue.containsTaskWithRequestId(ASYNC, asyncId);
}
Added: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java?rev=1698079&view=auto
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java (added)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java Thu Aug 27 05:16:07 2015
@@ -0,0 +1,190 @@
+package org.apache.solr.handler.admin;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Iterator;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.cloud.Overseer;
+import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.ZkNodeProps;
+import org.apache.solr.common.params.ConfigSetParams;
+import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.zookeeper.KeeperException;
+import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.BASE_CONFIGSET;
+import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.CONFIGSETS_ACTION_PREFIX;
+import static org.apache.solr.cloud.OverseerConfigSetMessageHandler.PROPERTY_PREFIX;
+import static org.apache.solr.common.params.CommonParams.NAME;
+import static org.apache.solr.common.params.ConfigSetParams.ConfigSetAction.*;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+import static org.apache.solr.cloud.Overseer.QUEUE_OPERATION;
+
+/**
+ * A {@link org.apache.solr.request.SolrRequestHandler} for ConfigSets API requests.
+ */
+public class ConfigSetsHandler extends RequestHandlerBase {
+ protected static Logger log = LoggerFactory.getLogger(ConfigSetsHandler.class);
+ protected final CoreContainer coreContainer;
+ public static long DEFAULT_ZK_TIMEOUT = 180*1000;
+
+ /**
+ * Overloaded ctor to inject CoreContainer into the handler.
+ *
+ * @param coreContainer Core Container of the solr webapp installed.
+ */
+ public ConfigSetsHandler(final CoreContainer coreContainer) {
+ this.coreContainer = coreContainer;
+ }
+
+ @Override
+ final public void init(NamedList args) {
+
+ }
+
+ @Override
+ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception {
+ if (coreContainer == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Core container instance missing");
+ }
+
+ // Make sure that the core is ZKAware
+ if(!coreContainer.isZooKeeperAware()) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "Solr instance is not running in SolrCloud mode.");
+ }
+
+ // Pick the action
+ SolrParams params = req.getParams();
+ String a = params.get(ConfigSetParams.ACTION);
+ if (a != null) {
+ ConfigSetAction action = ConfigSetAction.get(a);
+ if (action == null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown action: " + a);
+ ConfigSetOperation operation = ConfigSetOperation.get(action);
+ log.info("Invoked ConfigSet Action :{} with params {} ", action.toLower(), req.getParamString());
+ Map<String, Object> result = operation.call(req, rsp, this);
+ if (result != null) {
+ // We need to differentiate between collection and configsets actions since they currently
+ // use the same underlying queue.
+ result.put(QUEUE_OPERATION, CONFIGSETS_ACTION_PREFIX + operation.action.toLower());
+ ZkNodeProps props = new ZkNodeProps(result);
+ handleResponse(operation.action.toLower(), props, rsp, DEFAULT_ZK_TIMEOUT);
+ }
+ } else {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "action is a required param");
+ }
+
+ rsp.setHttpCaching(false);
+ }
+
+ private void handleResponse(String operation, ZkNodeProps m,
+ SolrQueryResponse rsp, long timeout) throws KeeperException, InterruptedException {
+ long time = System.nanoTime();
+
+ QueueEvent event = coreContainer.getZkController()
+ .getOverseerConfigSetQueue()
+ .offer(Utils.toJSON(m), timeout);
+ if (event.getBytes() != null) {
+ SolrResponse response = SolrResponse.deserialize(event.getBytes());
+ rsp.getValues().addAll(response.getResponse());
+ SimpleOrderedMap exp = (SimpleOrderedMap) response.getResponse().get("exception");
+ if (exp != null) {
+ Integer code = (Integer) exp.get("rspCode");
+ rsp.setException(new SolrException(code != null && code != -1 ? ErrorCode.getErrorCode(code) : ErrorCode.SERVER_ERROR, (String)exp.get("msg")));
+ }
+ } else {
+ if (System.nanoTime() - time >= TimeUnit.NANOSECONDS.convert(timeout, TimeUnit.MILLISECONDS)) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the configset time out:" + timeout / 1000 + "s");
+ } else if (event.getWatchedEvent() != null) {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the configset error [Watcher fired on path: "
+ + event.getWatchedEvent().getPath() + " state: "
+ + event.getWatchedEvent().getState() + " type "
+ + event.getWatchedEvent().getType() + "]");
+ } else {
+ throw new SolrException(ErrorCode.SERVER_ERROR, operation
+ + " the configset unknown case");
+ }
+ }
+ }
+
+ private static Map<String, Object> copyPropertiesWithPrefix(SolrParams params, Map<String, Object> props, String prefix) {
+ Iterator<String> iter = params.getParameterNamesIterator();
+ while (iter.hasNext()) {
+ String param = iter.next();
+ if (param.startsWith(prefix)) {
+ props.put(param, params.get(param));
+ }
+ }
+ return props;
+ }
+
+ @Override
+ public String getDescription() {
+ return "Manage SolrCloud ConfigSets";
+ }
+
+ enum ConfigSetOperation {
+ CREATE_OP(CREATE) {
+ @Override
+ Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {
+ Map<String, Object> props = req.getParams().required().getAll(null, NAME, BASE_CONFIGSET);
+ return copyPropertiesWithPrefix(req.getParams(), props, PROPERTY_PREFIX + ".");
+ }
+ },
+ DELETE_OP(DELETE) {
+ @Override
+ Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception {
+ return req.getParams().required().getAll(null, NAME);
+ }
+ };
+
+ ConfigSetAction action;
+
+ ConfigSetOperation(ConfigSetAction action) {
+ this.action = action;
+ }
+
+ abstract Map<String, Object> call(SolrQueryRequest req, SolrQueryResponse rsp, ConfigSetsHandler h) throws Exception;
+
+ public static ConfigSetOperation get(ConfigSetAction action) {
+ for (ConfigSetOperation op : values()) {
+ if (op.action == action) return op;
+ }
+ throw new SolrException(ErrorCode.SERVER_ERROR, "No such action" + action);
+ }
+ }
+}
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java Thu Aug 27 05:16:07 2015
@@ -25,7 +25,7 @@ import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.cloud.LeaderElector;
-import org.apache.solr.cloud.OverseerProcessor;
+import org.apache.solr.cloud.OverseerTaskProcessor;
import org.apache.solr.cloud.overseer.SliceMutator;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
@@ -160,7 +160,7 @@ class RebalanceLeaders {
ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();
- List<String> electionNodes = OverseerProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
+ List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
ZkStateReader.getShardLeadersElectPath(collectionName, slice.getName()));
if (electionNodes.size() < 2) { // if there's only one node in the queue, should already be leader and we shouldn't be here anyway.
@@ -193,7 +193,7 @@ class RebalanceLeaders {
throws KeeperException, InterruptedException {
ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();
- List<String> electionNodes = OverseerProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
+ List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
ZkStateReader.getShardLeadersElectPath(collectionName, slice.getName()));
// First, queue up the preferred leader at the head of the queue.
@@ -210,12 +210,12 @@ class RebalanceLeaders {
return; // let's not continue if we didn't get what we expect. Possibly we're offline etc..
}
- List<String> electionNodesTmp = OverseerProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
+ List<String> electionNodesTmp = OverseerTaskProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
ZkStateReader.getShardLeadersElectPath(collectionName, slice.getName()));
// Now find other nodes that have the same sequence number as this node and re-queue them at the end of the queue.
- electionNodes = OverseerProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
+ electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
ZkStateReader.getShardLeadersElectPath(collectionName, slice.getName()));
for (String thisNode : electionNodes) {
@@ -238,7 +238,7 @@ class RebalanceLeaders {
int oldSeq = LeaderElector.getSeq(electionNode);
for (int idx = 0; idx < 600; ++idx) {
ZkStateReader zkStateReader = coreContainer.getZkController().getZkStateReader();
- List<String> electionNodes = OverseerProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
+ List<String> electionNodes = OverseerTaskProcessor.getSortedElectionNodes(zkStateReader.getZkClient(),
ZkStateReader.getShardLeadersElectPath(collectionName, slice.getName()));
for (String testNode : electionNodes) {
if (LeaderElector.getNodeName(testNode).equals(nodeName) && oldSeq != LeaderElector.getSeq(testNode)) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/solr-50-all.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/solr-50-all.xml?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/solr-50-all.xml (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test-files/solr/solr-50-all.xml Thu Aug 27 05:16:07 2015
@@ -21,6 +21,7 @@
<int name="coreLoadThreads">11</int>
<str name="coreRootDirectory">${coreRootDirectory:testCoreRootDirectory}</str>
<str name="infoHandler">testInfoHandler</str>
+ <str name="configSetsHandler">testConfigSetsHandler</str>
<str name="managementPath">testManagementPath</str>
<str name="sharedLib">testSharedLib</str>
<str name="shareSchema">${shareSchema:true}</str>
Copied: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java (from r1698058, lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java?p2=lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java&p1=lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java&r1=1698058&r2=1698079&rev=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionProcessorTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionConfigSetProcessorTest.java Thu Aug 27 05:16:07 2015
@@ -20,7 +20,7 @@ package org.apache.solr.cloud;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.OverseerCollectionQueue.QueueEvent;
+import org.apache.solr.cloud.OverseerTaskQueue.QueueEvent;
import org.apache.solr.cloud.Overseer.LeaderStatus;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.SolrZkClient;
@@ -73,13 +73,13 @@ import static org.easymock.EasyMock.repl
import static org.easymock.EasyMock.reset;
import static org.easymock.EasyMock.verify;
-public class OverseerCollectionProcessorTest extends SolrTestCaseJ4 {
+public class OverseerCollectionConfigSetProcessorTest extends SolrTestCaseJ4 {
private static final String ADMIN_PATH = "/admin/cores";
private static final String COLLECTION_NAME = "mycollection";
private static final String CONFIG_NAME = "myconfig";
- private static OverseerCollectionQueue workQueueMock;
+ private static OverseerTaskQueue workQueueMock;
private static DistributedMap runningMapMock;
private static DistributedMap completedMapMock;
private static DistributedMap failureMapMock;
@@ -93,19 +93,19 @@ public class OverseerCollectionProcessor
private SolrResponse lastProcessMessageResult;
- private OverseerCollectionProcessorToBeTested underTest;
+ private OverseerCollectionConfigSetProcessorToBeTested underTest;
private Thread thread;
private Queue<QueueEvent> queue = new ArrayBlockingQueue<>(10);
- private class OverseerCollectionProcessorToBeTested extends
- OverseerCollectionProcessor {
+ private class OverseerCollectionConfigSetProcessorToBeTested extends
+ OverseerCollectionConfigSetProcessor {
- public OverseerCollectionProcessorToBeTested(ZkStateReader zkStateReader,
+ public OverseerCollectionConfigSetProcessorToBeTested(ZkStateReader zkStateReader,
String myId, ShardHandlerFactory shardHandlerFactory,
String adminPath,
- OverseerCollectionQueue workQueue, DistributedMap runningMap,
+ OverseerTaskQueue workQueue, DistributedMap runningMap,
DistributedMap completedMap,
DistributedMap failureMap) {
super(zkStateReader, myId, shardHandlerFactory, adminPath, new Overseer.Stats(), null, new OverseerNodePrioritizer(zkStateReader, adminPath, shardHandlerFactory), workQueue, runningMap, completedMap, failureMap);
@@ -120,7 +120,7 @@ public class OverseerCollectionProcessor
@BeforeClass
public static void setUpOnce() throws Exception {
- workQueueMock = createMock(OverseerCollectionQueue.class);
+ workQueueMock = createMock(OverseerTaskQueue.class);
runningMapMock = createMock(DistributedMap.class);
completedMapMock = createMock(DistributedMap.class);
failureMapMock = createMock(DistributedMap.class);
@@ -158,7 +158,7 @@ public class OverseerCollectionProcessor
reset(zkStateReaderMock);
reset(clusterStateMock);
reset(solrZkClientMock);
- underTest = new OverseerCollectionProcessorToBeTested(zkStateReaderMock,
+ underTest = new OverseerCollectionConfigSetProcessorToBeTested(zkStateReaderMock,
"1234", shardHandlerFactoryMock, ADMIN_PATH, workQueueMock, runningMapMock,
completedMapMock, failureMapMock);
zkMap.clear();
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java?rev=1698079&r1=1698078&r2=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerRolesTest.java Thu Aug 27 05:16:07 2015
@@ -43,8 +43,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.concurrent.TimeUnit;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.getLeaderNode;
-import static org.apache.solr.cloud.OverseerCollectionProcessor.getSortedOverseerNodeNames;
+import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getLeaderNode;
+import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames;
import static org.apache.solr.cloud.OverseerCollectionMessageHandler.NUM_SLICES;
import static org.apache.solr.common.util.Utils.makeMap;
import static org.apache.solr.common.cloud.ZkStateReader.MAX_SHARDS_PER_NODE;
@@ -88,7 +88,7 @@ public class OverseerRolesTest extends
final TimeOut timeout = new TimeOut(10, TimeUnit.SECONDS);
String newLeader=null;
for(;! timeout.hasTimedOut();){
- newLeader = OverseerCollectionProcessor.getLeaderNode(zk);
+ newLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(zk);
if(newLeader!=null && !newLeader.equals(leader)) break;
Thread.sleep(100);
}
@@ -96,7 +96,7 @@ public class OverseerRolesTest extends
- assertTrue("The old leader should have rejoined election ", OverseerCollectionProcessor.getSortedOverseerNodeNames(zk).contains(leader));
+ assertTrue("The old leader should have rejoined election ", OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zk).contains(leader));
}
@@ -108,10 +108,10 @@ public class OverseerRolesTest extends
createCollection(collectionName, client);
waitForRecoveriesToFinish(collectionName, false);
- List<String> l = OverseerCollectionProcessor.getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) ;
+ List<String> l = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) ;
log.info("All nodes {}", l);
- String currentLeader = OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient());
+ String currentLeader = OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient());
log.info("Current leader {} ", currentLeader);
l.remove(currentLeader);
@@ -124,7 +124,7 @@ public class OverseerRolesTest extends
boolean leaderchanged = false;
for(;!timeout.hasTimedOut();){
- if(overseerDesignate.equals(OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient()))){
+ if(overseerDesignate.equals(OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient()))){
log.info("overseer designate is the new overseer");
leaderchanged =true;
break;
@@ -134,7 +134,7 @@ public class OverseerRolesTest extends
assertTrue("could not set the new overseer . expected "+
overseerDesignate + " current order : " +
getSortedOverseerNodeNames(client.getZkStateReader().getZkClient()) +
- " ldr :"+ OverseerCollectionProcessor.getLeaderNode(client.getZkStateReader().getZkClient()) ,leaderchanged);
+ " ldr :"+ OverseerCollectionConfigSetProcessor.getLeaderNode(client.getZkStateReader().getZkClient()) ,leaderchanged);
@@ -176,7 +176,7 @@ public class OverseerRolesTest extends
log.info("leader node {}", leaderJetty.getBaseUrl());
log.info ("current election Queue",
- OverseerCollectionProcessor.getSortedElectionNodes(client.getZkStateReader().getZkClient(),
+ OverseerCollectionConfigSetProcessor.getSortedElectionNodes(client.getZkStateReader().getZkClient(),
OverseerElectionContext.PATH + LeaderElector.ELECTION_NODE));
ChaosMonkey.stop(leaderJetty);
timeout = new TimeOut(10, TimeUnit.SECONDS);
Copied: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java (from r1698058, lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionQueueTest.java)
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java?p2=lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java&p1=lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionQueueTest.java&r1=1698058&r2=1698079&rev=1698079&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerCollectionQueueTest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/OverseerTaskQueueTest.java Thu Aug 27 05:16:07 2015
@@ -16,13 +16,13 @@ package org.apache.solr.cloud;
* the License.
*/
-public class OverseerCollectionQueueTest extends DistributedQueueTest {
+public class OverseerTaskQueueTest extends DistributedQueueTest {
- // TODO: OverseerCollectionQueue specific tests.
+ // TODO: OverseerTaskQueue specific tests.
@Override
- protected OverseerCollectionQueue makeDistributedQueue(String dqZNode) throws Exception {
- return new OverseerCollectionQueue(zkClient, setupNewDistributedQueueZNode(dqZNode));
+ protected OverseerTaskQueue makeDistributedQueue(String dqZNode) throws Exception {
+ return new OverseerTaskQueue(zkClient, setupNewDistributedQueueZNode(dqZNode));
}
}