You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/08/16 11:56:39 UTC
lucene-solr:master: SOLR-11239: A special value of -1 can be
specified for 'maxShardsPerNode' to denote that there is no limit. The
bin/solr script send maxShardsPerNode=-1 when creating collections. The use
of maxShardsPerNode is not supported when a cl
Repository: lucene-solr
Updated Branches:
refs/heads/master 755cffdec -> 7a576ffa1
SOLR-11239: A special value of -1 can be specified for 'maxShardsPerNode' to denote that there is no limit. The bin/solr script send maxShardsPerNode=-1 when creating collections. The use of maxShardsPerNode is not supported when a cluster policy is in effect or when a collection specific policy is specified during collection creation
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/7a576ffa
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/7a576ffa
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/7a576ffa
Branch: refs/heads/master
Commit: 7a576ffa1b1f4b9632ff2767686fe203949c2aaf
Parents: 755cffd
Author: Shalin Shekhar Mangar <sh...@apache.org>
Authored: Wed Aug 16 17:26:32 2017 +0530
Committer: Shalin Shekhar Mangar <sh...@apache.org>
Committed: Wed Aug 16 17:26:32 2017 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 9 ++
.../apache/solr/cloud/CreateCollectionCmd.java | 22 +++-
.../src/java/org/apache/solr/util/SolrCLI.java | 4 -
.../autoscaling/AutoScalingHandlerTest.java | 22 +++-
.../solr/cloud/autoscaling/TestPolicyCloud.java | 1 -
.../apache/solr/util/TestSolrCLIRunExample.java | 119 ++++++++++++++++++-
6 files changed, 168 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 5b5a489..a90f078 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -292,6 +292,9 @@ Upgrading from Solr 6.x
* SOLR-11023: EnumField has been deprecated in favor of new EnumFieldType.
+* SOLR-11239: The use of maxShardsPerNode is not supported when a cluster policy is in effect or
+ when a collection specific policy is specified during collection creation.
+
New Features
----------------------
* SOLR-9857, SOLR-9858: Collect aggregated metrics from nodes and shard leaders in overseer. (ab)
@@ -668,6 +671,12 @@ Other Changes
* SOLR-10821: Ref guide documentation for Autoscaling (Noble Paul, Cassandra Targett, shalin)
+* SOLR-11239: A special value of -1 can be specified for 'maxShardsPerNode' to denote that
+ there is no limit. The bin/solr script send maxShardsPerNode=-1 when creating collections. The
+ use of maxShardsPerNode is not supported when a cluster policy is in effect or when a
+ collection specific policy is specified during collection creation.
+ (Noble Paul, shalin)
+
================== 6.7.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
index fc96a61..970327f 100644
--- a/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/CreateCollectionCmd.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Properties;
import java.util.concurrent.TimeUnit;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.cloud.OverseerCollectionMessageHandler.Cmd;
import org.apache.solr.cloud.overseer.ClusterStateMutator;
import org.apache.solr.common.cloud.ReplicaPosition;
@@ -85,6 +86,7 @@ public class CreateCollectionCmd implements Cmd {
if (clusterState.hasCollection(collectionName)) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "collection already exists: " + collectionName);
}
+ boolean usePolicyFramework = usePolicyFramework(ocmh.zkStateReader, message);
String configName = getConfigName(collectionName, message);
if (configName == null) {
@@ -118,7 +120,10 @@ public class CreateCollectionCmd implements Cmd {
}
int maxShardsPerNode = message.getInt(MAX_SHARDS_PER_NODE, 1);
-
+ if (usePolicyFramework && message.getStr(MAX_SHARDS_PER_NODE) != null && maxShardsPerNode > 0) {
+ throw new SolrException(ErrorCode.BAD_REQUEST, "'maxShardsPerNode>0' is not supported when autoScaling policies are used");
+ }
+ if (maxShardsPerNode == -1 || usePolicyFramework) maxShardsPerNode = Integer.MAX_VALUE;
if (numNrtReplicas + numTlogReplicas <= 0) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, NRT_REPLICAS + " + " + TLOG_REPLICAS + " must be greater than 0");
}
@@ -149,7 +154,9 @@ public class CreateCollectionCmd implements Cmd {
+ "). It's unusual to run two replica of the same slice on the same Solr-instance.");
}
- int maxShardsAllowedToCreate = maxShardsPerNode * nodeList.size();
+ int maxShardsAllowedToCreate = maxShardsPerNode == Integer.MAX_VALUE ?
+ Integer.MAX_VALUE :
+ maxShardsPerNode * nodeList.size();
int requestedShardsToCreate = numSlices * totalNumReplicas;
if (maxShardsAllowedToCreate < requestedShardsToCreate) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Cannot create collection " + collectionName + ". Value of "
@@ -484,4 +491,15 @@ public class CreateCollectionCmd implements Cmd {
"Could not find configName for collection " + collection + " found:" + configNames);
}
}
+
+ public static boolean usePolicyFramework(ZkStateReader zkStateReader, ZkNodeProps message) {
+ Map autoScalingJson = Collections.emptyMap();
+ try {
+ autoScalingJson = Utils.getJson(zkStateReader.getZkClient(), SOLR_AUTOSCALING_CONF_PATH, true);
+ } catch (Exception e) {
+ return false;
+ }
+ return autoScalingJson.get(Policy.CLUSTER_POLICY) != null || message.getStr(Policy.POLICY) != null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/core/src/java/org/apache/solr/util/SolrCLI.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/util/SolrCLI.java b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
index da53aff..e696019 100644
--- a/solr/core/src/java/org/apache/solr/util/SolrCLI.java
+++ b/solr/core/src/java/org/apache/solr/util/SolrCLI.java
@@ -1501,10 +1501,6 @@ public class SolrCLI {
if (cli.hasOption("maxShardsPerNode")) {
maxShardsPerNode = Integer.parseInt(cli.getOptionValue("maxShardsPerNode"));
- } else {
- // need number of live nodes to determine maxShardsPerNode if it is not set
- int numNodes = liveNodes.size();
- maxShardsPerNode = ((numShards*replicationFactor)+numNodes-1)/numNodes;
}
String confname = cli.getOptionValue("confname");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index c752253..719b0ac 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -291,9 +291,24 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, "{set-cluster-policy : []}");
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ // lets create a collection which violates the rule replicas < 2
+ try {
+ CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
+ create.setMaxShardsPerNode(10);
+ create.process(solrClient);
+ fail();
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains("'maxShardsPerNode>0' is not supported when autoScaling policies are used"));
+
+ }
+
+
// lets create a collection which violates the rule replicas < 2
CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
- create.setMaxShardsPerNode(10);
CollectionAdminResponse adminResponse = create.process(solrClient);
assertTrue(adminResponse.isSuccess());
@@ -302,6 +317,11 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+
// get the diagnostics output again
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null);
response = solrClient.request(req);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
index cb519f2..f657ad4 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
@@ -104,7 +104,6 @@ public class TestPolicyCloud extends SolrCloudTestCase {
String collectionName = "testCreateCollectionSplitShard";
CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2)
.setPolicy("c1")
- .setMaxShardsPerNode(10)
.process(cluster.getSolrClient());
DocCollection docCollection = getCollectionState(collectionName);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/7a576ffa/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
index 0c2421f..8900851 100644
--- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
@@ -40,6 +40,7 @@ import org.apache.commons.exec.ExecuteResultHandler;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -47,6 +48,7 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -54,6 +56,8 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+
/**
* Tests the SolrCLI.RunExampleTool implementation that supports bin/solr -e [example]
*/
@@ -481,6 +485,115 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
File node1SolrHome = new File(solrExampleDir, "cloud/node1/solr");
if (!node1SolrHome.isDirectory()) {
+ fail(node1SolrHome.getAbsolutePath() + " not found! run cloud example failed; tool output: " + toolOutput);
+ }
+
+ // delete the collection
+ SolrCLI.DeleteTool deleteTool = new SolrCLI.DeleteTool(stdoutSim);
+ String[] deleteArgs = new String[]{"-name", collectionName, "-solrUrl", solrUrl};
+ deleteTool.runTool(
+ SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs));
+
+ // dump all the output written by the SolrCLI commands to stdout
+ //System.out.println(toolOutput);
+
+ // stop the test instance
+ executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p " + bindPort));
+ }
+
+ @Test
+ public void testInteractiveSolrCloudExampleWithAutoScalingPolicy() throws Exception {
+ File solrHomeDir = new File(ExternalPaths.SERVER_HOME);
+ if (!solrHomeDir.isDirectory())
+ fail(solrHomeDir.getAbsolutePath() + " not found and is required to run this test!");
+
+ Path tmpDir = createTempDir();
+ File solrExampleDir = tmpDir.toFile();
+
+ File solrServerDir = solrHomeDir.getParentFile();
+
+ String[] toolArgs = new String[]{
+ "-example", "cloud",
+ "-serverDir", solrServerDir.getAbsolutePath(),
+ "-exampleDir", solrExampleDir.getAbsolutePath()
+ };
+
+ int bindPort = -1;
+ try (ServerSocket socket = new ServerSocket(0)) {
+ bindPort = socket.getLocalPort();
+ }
+
+ String collectionName = "testCloudExamplePrompt1";
+
+ // this test only support launching one SolrCloud node due to how MiniSolrCloudCluster works
+ // and the need for setting the host and port system properties ...
+ String userInput = "1\n" + bindPort + "\n" + collectionName + "\n2\n2\n_default\n";
+
+ // simulate user input from stdin
+ InputStream userInputSim = new ByteArrayInputStream(userInput.getBytes(StandardCharsets.UTF_8));
+
+ // capture tool output to stdout
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name());
+
+ RunExampleExecutor executor = new RunExampleExecutor(stdoutSim);
+ closeables.add(executor);
+
+ SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, userInputSim, stdoutSim);
+ try {
+ tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs));
+ } catch (Exception e) {
+ System.err.println("RunExampleTool failed due to: " + e +
+ "; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name()));
+ throw e;
+ }
+
+ String toolOutput = baos.toString(StandardCharsets.UTF_8.name());
+
+ // verify Solr is running on the expected port and verify the collection exists
+ String solrUrl = "http://localhost:" + bindPort + "/solr";
+ String collectionListUrl = solrUrl + "/admin/collections?action=list";
+ if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) {
+ fail("After running Solr cloud example, test collection '" + collectionName +
+ "' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput);
+ }
+
+ // index some docs - to verify all is good for both shards
+ CloudSolrClient cloudClient = null;
+
+ try {
+ cloudClient = getCloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress());
+ String setClusterPolicyCommand = "{" +
+ " 'set-cluster-policy': [" +
+ " {'cores':'<10', 'node':'#ANY'}," +
+ " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'overseer', 'replica':0}" +
+ " ]" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
+ NamedList<Object> response = cloudClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ SolrCLI.CreateCollectionTool createCollectionTool = new SolrCLI.CreateCollectionTool(stdoutSim);
+ String[] createArgs = new String[]{"create_collection", "-name", "newColl", "-configsetsDir", "_default", "-solrUrl", solrUrl};
+ createCollectionTool.runTool(
+ SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(createCollectionTool.getOptions()), createArgs));
+ solrUrl = "http://localhost:" + bindPort + "/solr";
+ collectionListUrl = solrUrl + "/admin/collections?action=list";
+ if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, "newColl")) {
+ toolOutput = baos.toString(StandardCharsets.UTF_8.name());
+ fail("After running Solr cloud example, test collection 'newColl' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput);
+ }
+ } finally {
+ if (cloudClient != null) {
+ try {
+ cloudClient.close();
+ } catch (Exception ignore) {
+ }
+ }
+ }
+
+ File node1SolrHome = new File(solrExampleDir, "cloud/node1/solr");
+ if (!node1SolrHome.isDirectory()) {
fail(node1SolrHome.getAbsolutePath()+" not found! run cloud example failed; tool output: "+toolOutput);
}
@@ -489,6 +602,10 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
String[] deleteArgs = new String[] { "-name", collectionName, "-solrUrl", solrUrl };
deleteTool.runTool(
SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs));
+ deleteTool = new SolrCLI.DeleteTool(stdoutSim);
+ deleteArgs = new String[]{"-name", "newColl", "-solrUrl", solrUrl};
+ deleteTool.runTool(
+ SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs));
// dump all the output written by the SolrCLI commands to stdout
//System.out.println(toolOutput);
@@ -496,7 +613,7 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
// stop the test instance
executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p "+bindPort));
}
-
+
@Test
public void testFailExecuteScript() throws Exception {
File solrHomeDir = new File(ExternalPaths.SERVER_HOME);