You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2015/04/22 13:19:00 UTC
svn commit: r1675335 - in /lucene/dev/branches/branch_5x: ./ solr/
solr/core/ solr/core/src/java/org/apache/solr/cloud/
solr/core/src/java/org/apache/solr/handler/admin/
solr/core/src/test/org/apache/solr/cloud/ solr/solrj/
solr/solrj/src/java/org/apac...
Author: noble
Date: Wed Apr 22 11:18:59 2015
New Revision: 1675335
URL: http://svn.apache.org/r1675335
Log:
SOLR-7176: zkcli script can perfrom the CLUSTERPROP command without a running Solr cluster
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/solr/core/ (props changed)
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
lucene/dev/branches/branch_5x/solr/solrj/ (props changed)
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
Modified: lucene/dev/branches/branch_5x/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/CHANGES.txt?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/solr/CHANGES.txt Wed Apr 22 11:18:59 2015
@@ -53,6 +53,9 @@ New Features
top_authors : { terms : { field:author, limit:5 } }
(yonik)
+* SOLR-7176: zkcli script can perfrom the CLUSTERPROP command without a running Solr cluster
+ (Hrishikesh Gadre, Per Steffensen, Noble Paul)
+
Bug Fixes
----------------------
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/Overseer.java Wed Apr 22 11:18:59 2015
@@ -100,7 +100,6 @@ public class Overseer implements Closeab
private final Stats zkStats;
- private Map clusterProps;
private boolean isClosed = false;
public ClusterStateUpdater(final ZkStateReader reader, final String myId, Stats zkStats) {
@@ -113,7 +112,6 @@ public class Overseer implements Closeab
this.completedMap = getCompletedMap(zkClient);
this.myId = myId;
this.reader = reader;
- clusterProps = reader.getClusterProps();
}
public Stats getStateUpdateQueueStats() {
@@ -344,9 +342,6 @@ public class Overseer implements Closeab
return new CollectionMutator(getZkStateReader()).deleteShard(clusterState, message);
case ADDREPLICA:
return new SliceMutator(getZkStateReader()).addReplica(clusterState, message);
- case CLUSTERPROP:
- handleProp(message);
- break;
case ADDREPLICAPROP:
return new ReplicaMutator(getZkStateReader()).addReplicaProperty(clusterState, message);
case DELETEREPLICAPROP:
@@ -397,25 +392,6 @@ public class Overseer implements Closeab
return ZkStateWriter.NO_OP;
}
- private void handleProp(ZkNodeProps message) {
- String name = message.getStr(NAME);
- String val = message.getStr("val");
- Map m = reader.getClusterProps();
- if(val ==null) m.remove(name);
- else m.put(name,val);
-
- try {
- if (reader.getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true))
- reader.getZkClient().setData(ZkStateReader.CLUSTER_PROPS, ZkStateReader.toJSON(m), true);
- else
- reader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, ZkStateReader.toJSON(m),CreateMode.PERSISTENT, true);
- clusterProps = reader.getClusterProps();
- } catch (Exception e) {
- log.error("Unable to set cluster property", e);
-
- }
- }
-
private LeaderStatus amILeader() {
TimerContext timerContext = stats.time("am_i_leader");
boolean success = true;
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionProcessor.java Wed Apr 22 11:18:59 2015
@@ -130,9 +130,6 @@ public class OverseerCollectionProcessor
public int maxParallelThreads = 10;
- public static final Set<String> KNOWN_CLUSTER_PROPS = ImmutableSet.of(ZkStateReader.LEGACY_CLOUD, ZkStateReader.URL_SCHEME,
- ZkStateReader.AUTO_ADD_REPLICAS);
-
public static final Map<String,Object> COLL_PROPS = ZkNodeProps.makeMap(
ROUTER, DocRouter.DEFAULT_NAME,
ZkStateReader.REPLICATION_FACTOR, "1",
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/cloud/ZkCLI.java Wed Apr 22 11:18:59 2015
@@ -1,5 +1,8 @@
package org.apache.solr.cloud;
+import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
+import static org.apache.solr.common.params.CommonParams.*;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -10,15 +13,19 @@ import org.apache.commons.cli.ParseExcep
import org.apache.commons.cli.PosixParser;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.OnReconnect;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkConfigManager;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.core.CoreContainer;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.xml.sax.SAXException;
import javax.xml.parsers.ParserConfigurationException;
+
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
@@ -123,7 +130,9 @@ public class ZkCLI {
"run zk internally by passing the solr run port - only for clusters on one machine (tests, dev)");
options.addOption("h", HELP, false, "bring up this help page");
-
+ options.addOption(NAME, true, "name of the cluster property to set");
+ options.addOption(VALUE_LONG, true, "value of the cluster to set");
+
try {
// parse the command line arguments
CommandLine line = parser.parse(options, args);
@@ -145,6 +154,7 @@ public class ZkCLI {
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + GET_FILE + " /solr.xml solr.xml.file");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLEAR + " /solr");
System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + LIST);
+ System.out.println("zkcli.sh -zkhost localhost:9983 -cmd " + CLUSTERPROP + " -" + NAME + " urlScheme -" + VALUE_LONG + " https" );
return;
}
@@ -294,6 +304,37 @@ public class ZkCLI {
}
byte [] data = zkClient.getData(arglist.get(0).toString(), null, null, true);
FileUtils.writeByteArrayToFile(new File(arglist.get(1).toString()), data);
+ } else if (CollectionAction.get(line.getOptionValue(CMD)) == CLUSTERPROP) {
+ if(!line.hasOption(NAME)) {
+ System.out.println("-" + NAME + " is required for " + CLUSTERPROP);
+ }
+ String propertyName = line.getOptionValue(NAME);
+ //If -val option is missing, we will use the null value. This is required to maintain
+ //compatibility with Collections API.
+ String propertyValue = line.getOptionValue(VALUE_LONG);
+ ZkStateReader reader = new ZkStateReader(zkClient);
+ try {
+ reader.setClusterProperty(propertyName, propertyValue);
+ } catch (SolrException ex) {
+ //This can happen if two concurrent invocations of this command collide
+ //with each other. Here we are just adding a defensive check to see if
+ //the value is already set to expected value. If yes, then we don't
+ //fail the command.
+ Throwable cause = ex.getCause();
+ if(cause instanceof KeeperException.NodeExistsException
+ || cause instanceof KeeperException.BadVersionException) {
+ String currentValue = (String)reader.getClusterProps().get(propertyName);
+ if((currentValue == propertyValue) || (currentValue != null && currentValue.equals(propertyValue))) {
+ return;
+ }
+ }
+ System.out.println("Unable to set the cluster property due to following error : " +
+ ex.getLocalizedMessage() +
+ ((cause instanceof KeeperException.BadVersionException)?". Try again":""));
+ System.exit(1);
+ } finally {
+ reader.close();
+ }
}
} finally {
if (solrPort != null) {
Modified: lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java Wed Apr 22 11:18:59 2015
@@ -20,7 +20,6 @@ package org.apache.solr.handler.admin;
import static org.apache.solr.cloud.Overseer.*;
import static org.apache.solr.cloud.OverseerCollectionProcessor.*;
import static org.apache.solr.common.cloud.DocCollection.*;
-import static org.apache.solr.common.cloud.ZkNodeProps.*;
import static org.apache.solr.common.cloud.ZkStateReader.*;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonParams.*;
@@ -569,17 +568,8 @@ public class CollectionsHandler extends
private void handleProp(SolrQueryRequest req, SolrQueryResponse rsp) throws KeeperException, InterruptedException {
req.getParams().required().check(NAME);
String name = req.getParams().get(NAME);
- if(!OverseerCollectionProcessor.KNOWN_CLUSTER_PROPS.contains(name)){
- throw new SolrException(ErrorCode.BAD_REQUEST, "Not a known cluster property "+ name);
- }
-
- Map<String,Object> props = ZkNodeProps.makeMap(
- Overseer.QUEUE_OPERATION, CLUSTERPROP.toLower() );
- copyIfNotNull(req.getParams(),props,
- NAME,
- "val");
-
- Overseer.getInQueue(coreContainer.getZkController().getZkClient()).offer(ZkStateReader.toJSON(props)) ;
+ String val = req.getParams().get(VALUE_LONG);
+ coreContainer.getZkController().getZkStateReader().setClusterProperty(name, val);
}
static Set<String> KNOWN_ROLES = ImmutableSet.of("overseer");
Modified: lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java (original)
+++ lucene/dev/branches/branch_5x/solr/core/src/test/org/apache/solr/cloud/ZkCLITest.java Wed Apr 22 11:18:59 2015
@@ -298,6 +298,26 @@ public class ZkCLITest extends SolrTestC
zkClient.close();
}
+ @Test
+ public void testSetClusterProperty() throws Exception {
+ ZkStateReader reader = new ZkStateReader(zkClient);
+ try {
+ // add property urlScheme=http
+ String[] args = new String[] {"-zkhost", zkServer.getZkAddress(),
+ "-cmd", "CLUSTERPROP", "-name", "urlScheme", "-val", "http"};
+ ZkCLI.main(args);
+ assertEquals("http", reader.getClusterProps().get("urlScheme"));
+
+ // remove it again
+ args = new String[] {"-zkhost", zkServer.getZkAddress(),
+ "-cmd", "CLUSTERPROP", "-name", "urlScheme"};
+ ZkCLI.main(args);
+ assertNull(reader.getClusterProps().get("urlScheme"));
+ } finally {
+ reader.close();
+ }
+ }
+
@Override
public void tearDown() throws Exception {
if (VERBOSE) {
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/cloud/ZkStateReader.java Wed Apr 22 11:18:59 2015
@@ -20,6 +20,7 @@ package org.apache.solr.common.cloud;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.ByteUtils;
+import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.WatchedEvent;
import org.apache.zookeeper.Watcher;
@@ -51,6 +52,9 @@ import java.util.concurrent.ScheduledExe
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
+import static java.util.Arrays.asList;
+import static java.util.Collections.unmodifiableSet;
+
public class ZkStateReader implements Closeable {
private static Logger log = LoggerFactory.getLogger(ZkStateReader.class);
@@ -113,6 +117,10 @@ public class ZkStateReader implements Cl
private final ZkConfigManager configManager;
+ public static final Set<String> KNOWN_CLUSTER_PROPS = unmodifiableSet(new HashSet<>(asList(
+ LEGACY_CLOUD,
+ URL_SCHEME,
+ AUTO_ADD_REPLICAS)));
//
// convenience methods... should these go somewhere else?
@@ -765,7 +773,58 @@ public class ZkStateReader implements Cl
throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading cluster properties",e) ;
}
}
-
+
+ /**
+ * This method sets a cluster property.
+ *
+ * @param propertyName The property name to be set.
+ * @param propertyValue The value of the property.
+ */
+ public void setClusterProperty(String propertyName, String propertyValue) {
+ if (!KNOWN_CLUSTER_PROPS.contains(propertyName)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "Not a known cluster property " + propertyName);
+ }
+
+ for (; ; ) {
+ Stat s = new Stat();
+ try {
+ if (getZkClient().exists(CLUSTER_PROPS, true)) {
+ int v = 0;
+ Map properties = (Map) fromJSON(getZkClient().getData(CLUSTER_PROPS, null, s, true));
+ if (propertyValue == null) {
+ //Don't update ZK unless absolutely necessary.
+ if (properties.get(propertyName) != null) {
+ properties.remove(propertyName);
+ getZkClient().setData(CLUSTER_PROPS, toJSON(properties), s.getVersion(), true);
+ }
+ } else {
+ //Don't update ZK unless absolutely necessary.
+ if (!propertyValue.equals(properties.get(propertyName))) {
+ properties.put(propertyName, propertyValue);
+ getZkClient().setData(CLUSTER_PROPS, toJSON(properties), s.getVersion(), true);
+ }
+ }
+ } else {
+ Map properties = new LinkedHashMap();
+ properties.put(propertyName, propertyValue);
+ getZkClient().create(CLUSTER_PROPS, toJSON(properties), CreateMode.PERSISTENT, true);
+ }
+ } catch (KeeperException.BadVersionException bve) {
+ log.warn("Race condition while trying to set a new cluster prop on current version " + s.getVersion());
+ //race condition
+ continue;
+ } catch (KeeperException.NodeExistsException nee) {
+ log.warn("Race condition while trying to set a new cluster prop on current version " + s.getVersion());
+ //race condition
+ continue;
+ } catch (Exception ex) {
+ log.error("Error updating path " + CLUSTER_PROPS, ex);
+ throw new SolrException(ErrorCode.SERVER_ERROR, "Error updating cluster property " + propertyName, ex);
+ }
+ break;
+ }
+ }
+
/**
* Returns the baseURL corresponding to a given node's nodeName --
* NOTE: does not (currently) imply that the nodeName (or resulting
Modified: lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java?rev=1675335&r1=1675334&r2=1675335&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java (original)
+++ lucene/dev/branches/branch_5x/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java Wed Apr 22 11:18:59 2015
@@ -232,5 +232,6 @@ public interface CommonParams {
public static final String PATH = "path";
public static final String NAME = "name";
+ public static final String VALUE_LONG = "val";
}