You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by is...@apache.org on 2020/07/17 00:03:30 UTC
[lucene-solr] branch jira/solr-14656-master updated: SOLR-14656:
Remove autoscaling framework, part 2
This is an automated email from the ASF dual-hosted git repository.
ishan pushed a commit to branch jira/solr-14656-master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/jira/solr-14656-master by this push:
new 9d946e2 SOLR-14656: Remove autoscaling framework, part 2
9d946e2 is described below
commit 9d946e232a198b9f5adc5cc4a42dbb790eccdb1e
Author: Ishan Chattopadhyaya <is...@apache.org>
AuthorDate: Fri Jul 17 05:33:12 2020 +0530
SOLR-14656: Remove autoscaling framework, part 2
---
.../test/org/apache/solr/cloud/CloudTestUtils.java | 1 +
.../org/apache/solr/cloud/TestUtilizeNode.java | 184 ---------------------
.../test/org/apache/solr/cloud/rule/RulesTest.java | 58 -------
.../apache/solr/util/TestSolrCLIRunExample.java | 117 -------------
4 files changed, 1 insertion(+), 359 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
index 07e279b..9c8e6ff 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CloudTestUtils.java
@@ -120,6 +120,7 @@ public class CloudTestUtils {
* Helper class for sending (JSON) autoscaling requests that can randomize between V1 and V2 requests
*/
@SuppressWarnings({"rawtypes"})
+ // nocommit remove this
public static class AutoScalingRequest extends SolrRequest {
private SolrParams params = null;
/**
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java b/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java
deleted file mode 100644
index 68e1e88..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/TestUtilizeNode.java
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.LogLevel;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.impl.SolrClientDataProvider=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling.PolicyHelper=TRACE")
-public class TestUtilizeNode extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(3)
- .addConfig("conf1", TEST_PATH().resolve("configsets").resolve("cloud-dynamic").resolve("conf"))
- .configure();
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- JettySolrRunner overseerJetty = null;
- String overseerLeader = (String) overSeerStatus.get("leader");
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerJetty = jetty;
- break;
- }
- }
- if (overseerJetty == null) {
- fail("no overseer leader!");
- }
- }
-
- protected String getSolrXml() {
- return "solr.xml";
- }
-
- @Before
- public void beforeTest() throws Exception {
- cluster.deleteAllCollections();
- }
-
- @Test
- public void test() throws Exception {
- cluster.waitForAllNodes(5);
- String coll = "utilizenodecoll";
- CloudSolrClient cloudClient = cluster.getSolrClient();
-
- log.info("Creating Collection...");
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(coll, "conf1", 2, 2);
- cloudClient.request(create);
-
- log.info("Spinning up additional jettyX...");
- JettySolrRunner jettyX = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
-
- assertNoReplicas("jettyX should not yet be utilized: ", coll, jettyX);
-
- if (log.isInfoEnabled()) {
- log.info("Sending UTILIZE command for jettyX ({})", jettyX.getNodeName());
- }
- cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyX.getNodeName()));
-
- // TODO: aparently we can't assert this? ...
- //
- // assertSomeReplicas("jettyX should now be utilized: ", coll, jettyX);
- //
- // ... it appears from the docs that unless there are policy violations,
- // this can be ignored unless jettyX has less "load" then other jetty instances?
- //
- // if the above is true, that means that this test is incredibly weak...
- // unless we know jettyX has at least one replica, then all the subsequent testing of the
- // port blacklist & additional UTILIZE command for jettyY are a waste of time.
- //
- // should we skip spinning up a *new* jettyX, and instead just pick an existing jetty?
-
- if (log.isInfoEnabled()) {
- log.info("jettyX replicas prior to being blacklisted: {}", getReplicaList(coll, jettyX));
- }
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'port':" + jettyX.getLocalPort() +
- " , 'replica':0}" +
- " ]" +
- "}";
- if (log.isInfoEnabled()) {
- log.info("Setting new policy to blacklist jettyX ({}) port={}",
- jettyX.getNodeName(), jettyX.getLocalPort());
- }
- @SuppressWarnings({"rawtypes"})
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- NamedList<Object> response = cloudClient.request(req);
- assertEquals(req + " => " + response,
- "success", response.get("result").toString());
-
- log.info("Spinning up additional jettyY...");
- JettySolrRunner jettyY = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
-
- assertNoReplicas("jettyY should not yet be utilized: ", coll, jettyY);
- if (log.isInfoEnabled()) {
- log.info("jettyX replicas prior to utilizing jettyY: {}", getReplicaList(coll, jettyX));
- log.info("Sending UTILIZE command for jettyY ({})", jettyY.getNodeName()); // logOk
- }
- cloudClient.request(new CollectionAdminRequest.UtilizeNode(jettyY.getNodeName()));
-
- assertSomeReplicas("jettyY should now be utilized: ", coll, jettyY);
- }
-
- /**
- * Gets the list of replicas for the specified collection hosted on the specified node
- * and then asserts that it has no replicas
- */
- private void assertNoReplicas(String prefix, String collectionName, JettySolrRunner jettyNode) throws IOException {
-
- final List<Replica> replicas = getReplicaList(collectionName, jettyNode);
- assertEquals(prefix + " " + jettyNode.getNodeName() + " => " + replicas,
- 0, replicas.size());
- }
-
- /**
- * Gets the list of replicas for the specified collection hosted on the specified node
- * and then asserts that it there is at least one
- */
- private void assertSomeReplicas(String prefix, String collectionName, JettySolrRunner jettyNode) throws IOException {
-
- final List<Replica> replicas = getReplicaList(collectionName, jettyNode);
- assertTrue(prefix + " " + jettyNode.getNodeName() + " => " + replicas,
- 0 < replicas.size());
- }
-
- /**
- * Returns a list of all Replicas for the specified collection hosted on the specified node using
- * an <em>uncached</em> ClusterState call (so it should be authoritative from ZK).
- */
- private List<Replica> getReplicaList(String collectionName, JettySolrRunner jettyNode) throws IOException {
- DocCollection collection = cluster.getSolrClient().getClusterStateProvider()
- // we do *NOT* want to trust the cache, because anytime we call this method we have just
- // done a lot of mucking with the cluster
- .getClusterState().getCollectionOrNull(collectionName, false);
-
- List<Replica> results = new ArrayList<>(3);
- if (collection != null) {
- collection.forEachReplica((s, replica) -> {
- if (replica.getNodeName().equals(jettyNode.getNodeName())) {
- results.add(replica);
- }
- });
- }
- return results;
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
index 7401c85..d929944 100644
--- a/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/rule/RulesTest.java
@@ -28,13 +28,11 @@ import java.util.stream.Collectors;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
import org.apache.solr.cloud.SolrCloudTestCase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -160,62 +158,6 @@ public class RulesTest extends SolrCloudTestCase {
}
@Test
- public void testPortRuleInPresenceOfClusterPolicy() throws Exception {
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- String port = Integer.toString(jetty.getLocalPort());
-
- // this cluster policy prohibits having any replicas on a node with the above port
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'replica': 0, 'port':'" + port + "'}" +
- " ]" +
- "}";
- @SuppressWarnings({"rawtypes"})
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- cluster.getSolrClient().request(req);
-
- // but this collection is created with a replica placement rule that says all replicas must be created
- // on a node with above port (in direct conflict with the cluster policy)
- String rulesColl = "portRuleColl2";
- CollectionAdminRequest.createCollectionWithImplicitRouter(rulesColl, "conf", "shard1", 2)
- .setRule("port:" + port)
- .setSnitch("class:ImplicitSnitch")
- .process(cluster.getSolrClient());
-
- waitForState("Collection should have followed port rule w/ImplicitSnitch, not cluster policy",
- rulesColl, (liveNodes, rulesCollection) -> {
- // first sanity check that the collection exists & the rules/snitch are listed
- if (null == rulesCollection) {
- return false;
- } else {
- @SuppressWarnings({"rawtypes"})
- List list = (List) rulesCollection.get("rule");
- if (null == list || 1 != list.size()) {
- return false;
- }
- if (! port.equals(((Map) list.get(0)).get("port"))) {
- return false;
- }
- list = (List) rulesCollection.get("snitch");
- if (null == list || 1 != list.size()) {
- return false;
- }
- if (! "ImplicitSnitch".equals(((Map)list.get(0)).get("class"))) {
- return false;
- }
- }
- if (2 != rulesCollection.getReplicas().size()) {
- return false;
- }
- // now sanity check that the rules were *obeyed*
- // (and the contradictory policy was ignored)
- return rulesCollection.getReplicas().stream().allMatch
- (replica -> (replica.getNodeName().contains(port) &&
- replica.isActive(liveNodes)));
- });
- }
-
- @Test
public void testPortRule() throws Exception {
JettySolrRunner jetty = cluster.getRandomJetty(random());
diff --git a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
index 0ef5851..5721ecd 100644
--- a/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
+++ b/solr/core/src/test/org/apache/solr/util/TestSolrCLIRunExample.java
@@ -40,16 +40,13 @@ import org.apache.commons.exec.ExecuteResultHandler;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.embedded.JettyConfig;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
import org.apache.solr.cloud.MiniSolrCloudCluster;
import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.util.NamedList;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -515,120 +512,6 @@ public class TestSolrCLIRunExample extends SolrTestCaseJ4 {
}
@Test
- public void testInteractiveSolrCloudExampleWithAutoScalingPolicy() throws Exception {
- File solrHomeDir = new File(ExternalPaths.SERVER_HOME);
- if (!solrHomeDir.isDirectory())
- fail(solrHomeDir.getAbsolutePath() + " not found and is required to run this test!");
-
- Path tmpDir = createTempDir();
- File solrExampleDir = tmpDir.toFile();
-
- File solrServerDir = solrHomeDir.getParentFile();
-
- String[] toolArgs = new String[]{
- "-example", "cloud",
- "-serverDir", solrServerDir.getAbsolutePath(),
- "-exampleDir", solrExampleDir.getAbsolutePath()
- };
-
- int bindPort = -1;
- try (ServerSocket socket = new ServerSocket(0)) {
- bindPort = socket.getLocalPort();
- }
-
- String collectionName = "testCloudExamplePrompt1";
-
- // this test only support launching one SolrCloud node due to how MiniSolrCloudCluster works
- // and the need for setting the host and port system properties ...
- String userInput = "1\n" + bindPort + "\n" + collectionName + "\n2\n2\n_default\n";
-
- // simulate user input from stdin
- InputStream userInputSim = new ByteArrayInputStream(userInput.getBytes(StandardCharsets.UTF_8));
-
- // capture tool output to stdout
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream stdoutSim = new PrintStream(baos, true, StandardCharsets.UTF_8.name());
-
- RunExampleExecutor executor = new RunExampleExecutor(stdoutSim);
- closeables.add(executor);
-
- SolrCLI.RunExampleTool tool = new SolrCLI.RunExampleTool(executor, userInputSim, stdoutSim);
- try {
- tool.runTool(SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(tool.getOptions()), toolArgs));
- } catch (Exception e) {
- System.err.println("RunExampleTool failed due to: " + e +
- "; stdout from tool prior to failure: " + baos.toString(StandardCharsets.UTF_8.name()));
- throw e;
- }
-
- String toolOutput = baos.toString(StandardCharsets.UTF_8.name());
-
- // verify Solr is running on the expected port and verify the collection exists
- String solrUrl = "http://localhost:" + bindPort + "/solr";
- String collectionListUrl = solrUrl + "/admin/collections?action=list";
- if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, collectionName)) {
- fail("After running Solr cloud example, test collection '" + collectionName +
- "' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput);
- }
-
- // index some docs - to verify all is good for both shards
- CloudSolrClient cloudClient = null;
-
- try {
- cloudClient = getCloudSolrClient(executor.solrCloudCluster.getZkServer().getZkAddress());
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- @SuppressWarnings({"rawtypes"})
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- NamedList<Object> response = cloudClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- SolrCLI.CreateCollectionTool createCollectionTool = new SolrCLI.CreateCollectionTool(stdoutSim);
- String[] createArgs = new String[]{"create_collection", "-name", "newColl", "-configsetsDir", "_default", "-solrUrl", solrUrl};
- createCollectionTool.runTool(
- SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(createCollectionTool.getOptions()), createArgs));
- solrUrl = "http://localhost:" + bindPort + "/solr";
- collectionListUrl = solrUrl + "/admin/collections?action=list";
- if (!SolrCLI.safeCheckCollectionExists(collectionListUrl, "newColl")) {
- toolOutput = baos.toString(StandardCharsets.UTF_8.name());
- fail("After running Solr cloud example, test collection 'newColl' not found in Solr at: " + solrUrl + "; tool output: " + toolOutput);
- }
- } finally {
- if (cloudClient != null) {
- try {
- cloudClient.close();
- } catch (Exception ignore) {
- }
- }
- }
-
- File node1SolrHome = new File(solrExampleDir, "cloud/node1/solr");
- if (!node1SolrHome.isDirectory()) {
- fail(node1SolrHome.getAbsolutePath()+" not found! run cloud example failed; tool output: "+toolOutput);
- }
-
- // delete the collection
- SolrCLI.DeleteTool deleteTool = new SolrCLI.DeleteTool(stdoutSim);
- String[] deleteArgs = new String[] { "-name", collectionName, "-solrUrl", solrUrl };
- deleteTool.runTool(
- SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs));
- deleteTool = new SolrCLI.DeleteTool(stdoutSim);
- deleteArgs = new String[]{"-name", "newColl", "-solrUrl", solrUrl};
- deleteTool.runTool(
- SolrCLI.processCommandLineArgs(SolrCLI.joinCommonAndToolOptions(deleteTool.getOptions()), deleteArgs));
-
- // dump all the output written by the SolrCLI commands to stdout
- //System.out.println(toolOutput);
-
- // stop the test instance
- executor.execute(org.apache.commons.exec.CommandLine.parse("bin/solr stop -p "+bindPort));
- }
-
- @Test
public void testFailExecuteScript() throws Exception {
File solrHomeDir = new File(ExternalPaths.SERVER_HOME);
if (!solrHomeDir.isDirectory())