You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2017/10/09 15:30:58 UTC
[06/19] lucene-solr:branch_7x: Merge branch 'feature/autoscaling' of
https://git-wip-us.apache.org/repos/asf/lucene-solr
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
index 507c188..030be51 100644
--- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java
@@ -48,6 +48,7 @@ import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.params.CollectionParams;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.CloudConfig;
+import org.apache.solr.core.CoreContainer;
import org.apache.solr.handler.component.HttpShardHandlerFactory;
import org.apache.solr.update.UpdateShardHandler;
import org.apache.solr.update.UpdateShardHandlerConfig;
@@ -61,11 +62,14 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import org.mockito.Mockito;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.xml.sax.SAXException;
import static org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
@Slow
public class OverseerTest extends SolrTestCaseJ4 {
@@ -607,7 +611,8 @@ public class OverseerTest extends SolrTestCaseJ4 {
updateShardHandlers.add(updateShardHandler);
HttpShardHandlerFactory httpShardHandlerFactory = new HttpShardHandlerFactory();
httpShardHandlerFactorys.add(httpShardHandlerFactory);
- Overseer overseer = new Overseer(httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, null,
+ MockZkController mockZkController = createMockZkController(zkClient, reader);
+ Overseer overseer = new Overseer(httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, mockZkController,
new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").build());
overseers.add(overseer);
ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
@@ -1174,7 +1179,10 @@ public class OverseerTest extends SolrTestCaseJ4 {
updateShardHandlers.add(updateShardHandler);
HttpShardHandlerFactory httpShardHandlerFactory = new HttpShardHandlerFactory();
httpShardHandlerFactorys.add(httpShardHandlerFactory);
- Overseer overseer = new Overseer(httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, null,
+
+ MockZkController zkController = createMockZkController(zkClient, reader);
+
+ Overseer overseer = new Overseer(httpShardHandlerFactory.getShardHandler(), updateShardHandler, "/admin/cores", reader, zkController,
new CloudConfig.CloudConfigBuilder("127.0.0.1", 8983, "").build());
overseers.add(overseer);
ElectionContext ec = new OverseerElectionContext(zkClient, overseer,
@@ -1183,7 +1191,19 @@ public class OverseerTest extends SolrTestCaseJ4 {
overseerElector.joinElection(ec, false);
return zkClient;
}
-
+
+ private MockZkController createMockZkController(SolrZkClient zkClient, ZkStateReader reader) {
+ CoreContainer mockAlwaysUpCoreContainer = mock(CoreContainer.class,
+ Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
+ when(mockAlwaysUpCoreContainer.isShutDown()).thenReturn(Boolean.FALSE); // Allow retry on session expiry
+ MockZkController zkController = mock(MockZkController.class,
+ Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS));
+ when(zkController.getCoreContainer()).thenReturn(mockAlwaysUpCoreContainer);
+ when(zkController.getZkClient()).thenReturn(zkClient);
+ when(zkController.getZkStateReader()).thenReturn(reader);
+ return zkController;
+ }
+
@Test
public void testRemovalOfLastReplica() throws Exception {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
index 1593e78..3274fe2 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
@@ -1021,4 +1021,3 @@ public class ShardSplitTest extends BasicDistributedZkTest {
return client;
}
}
-
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
index e2174de..41c7c40 100644
--- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverTest.java
@@ -16,12 +16,12 @@
*/
package org.apache.solr.cloud;
-import static org.apache.solr.common.util.Utils.makeMap;
-
import java.io.IOException;
+import java.lang.invoke.MethodHandles;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CompletionService;
@@ -30,39 +30,44 @@ import java.util.concurrent.Future;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.util.LuceneTestCase.Slow;
import com.carrotsearch.randomizedtesting.annotations.Nightly;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrQuery;
-import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest.Create;
-import org.apache.solr.client.solrj.request.QueryRequest;
import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.cloud.hdfs.HdfsTestUtil;
+import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.ClusterStateUtil;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.MapSolrParams;
import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.ExecutorUtil;
import org.apache.solr.core.CoreDescriptor;
import org.apache.solr.update.DirectUpdateHandler2;
import org.apache.solr.util.DefaultSolrThreadFactory;
import org.apache.solr.util.BadHdfsThreadsFilter;
+import org.apache.solr.util.LogLevel;
+import org.apache.solr.util.TimeOut;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Nightly
@Slow
@@ -70,8 +75,10 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
@ThreadLeakFilters(defaultFilters = true, filters = {
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.*=DEBUG")
public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBase {
-
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
private static final boolean DEBUG = true;
private static MiniDFSCluster dfsCluster;
@@ -210,9 +217,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
Thread.sleep(5000);
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 120000));
-
- assertSliceAndReplicaCount(collection1);
+ assertSliceAndReplicaCount(collection1, 2, 2, 120000);
assertEquals(4, ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
assertTrue(ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection2) < 4);
@@ -225,7 +230,7 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
// all docs should be queried after failover
cloudClient.commit(); // to query all docs
- assertSingleReplicationAndShardSize(collection4, 5);
+ assertSliceAndReplicaCount(collection4, 5, 1, 120000);
queryAndAssertResultSize(collection4, numDocs, 10000);
// collection1 should still be at 4
@@ -235,21 +240,22 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
assertUlogDir(collections);
- ChaosMonkey.stop(jettys);
+ boolean allowOverseerRestart = random().nextBoolean();
+ List<JettySolrRunner> stoppedJetties = allowOverseerRestart
+ ? jettys.stream().filter(jettySolrRunner -> random().nextBoolean()).collect(Collectors.toList()) : notOverseerJetties();
+ ChaosMonkey.stop(stoppedJetties);
ChaosMonkey.stop(controlJetty);
- assertTrue("Timeout waiting for all not live", ClusterStateUtil.waitForAllReplicasNotLive(cloudClient.getZkStateReader(), 45000));
+ assertTrue("Timeout waiting for all not live", waitingForReplicasNotLive(cloudClient.getZkStateReader(), 45000, stoppedJetties));
- ChaosMonkey.start(jettys);
+ ChaosMonkey.start(stoppedJetties);
ChaosMonkey.start(controlJetty);
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 120000));
-
- assertSliceAndReplicaCount(collection1);
- assertSingleReplicationAndShardSize(collection3, 5);
+ assertSliceAndReplicaCount(collection1, 2, 2, 120000);
+ assertSliceAndReplicaCount(collection3, 5, 1, 120000);
// all docs should be queried
- assertSingleReplicationAndShardSize(collection4, 5);
+ assertSliceAndReplicaCount(collection4, 5, 1, 120000);
queryAndAssertResultSize(collection4, numDocs, 10000);
assertUlogDir(collections);
@@ -257,75 +263,13 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
int jettyIndex = random().nextInt(jettys.size());
ChaosMonkey.stop(jettys.get(jettyIndex));
ChaosMonkey.start(jettys.get(jettyIndex));
-
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 60000));
-
- assertSliceAndReplicaCount(collection1);
-
- assertUlogDir(collections);
-
- assertSingleReplicationAndShardSize(collection3, 5);
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection3, 5, 30000);
-
- assertSingleReplicationAndShardSize(collection4, 5);
- ClusterStateUtil.waitForLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection4, 5, 30000);
-
- //disable autoAddReplicas
- Map m = makeMap(
- "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
- "name", ZkStateReader.AUTO_ADD_REPLICAS,
- "val", "false");
- SolrRequest request = new QueryRequest(new MapSolrParams(m));
- request.setPath("/admin/collections");
- cloudClient.request(request);
-
- int currentCount = ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1);
-
- ChaosMonkey.stop(jettys.get(3));
-
- //solr.xml has defined workLoopDelay=10s and waitAfterExpiration=10s
- //Hence waiting for 30 seconds to be on the safe side.
- Thread.sleep(30000);
- //Ensures that autoAddReplicas has not kicked in.
- assertTrue(currentCount > ClusterStateUtil.getLiveAndActiveReplicaCount(cloudClient.getZkStateReader(), collection1));
-
- //enable autoAddReplicas
- m = makeMap(
- "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
- "name", ZkStateReader.AUTO_ADD_REPLICAS);
-
- request = new QueryRequest(new MapSolrParams(m));
- request.setPath("/admin/collections");
- cloudClient.request(request);
-
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 90000));
- assertSliceAndReplicaCount(collection1);
+ assertSliceAndReplicaCount(collection1, 2, 2, 120000);
assertUlogDir(collections);
-
- // restart all to test core saved state
-
- ChaosMonkey.stop(jettys);
- ChaosMonkey.stop(controlJetty);
-
- assertTrue("Timeout waiting for all not live", ClusterStateUtil.waitForAllReplicasNotLive(cloudClient.getZkStateReader(), 45000));
- ChaosMonkey.start(jettys);
- ChaosMonkey.start(controlJetty);
-
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cloudClient.getZkStateReader(), collection1, 120000));
-
- assertSliceAndReplicaCount(collection1);
-
- assertUlogDir(collections);
-
- assertSliceAndReplicaCount(collection1);
- assertSingleReplicationAndShardSize(collection3, 5);
-
- // all docs should be queried
- assertSingleReplicationAndShardSize(collection4, 5);
- queryAndAssertResultSize(collection4, numDocs, 10000);
+ assertSliceAndReplicaCount(collection3, 5, 1, 120000);
+ assertSliceAndReplicaCount(collection4, 5, 1, 120000);
}
private void queryAndAssertResultSize(String collection, int expectedResultSize, int timeoutMS)
@@ -339,10 +283,14 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
SolrParams queryAll = new SolrQuery("*:*");
cloudClient.setDefaultCollection(collection);
- QueryResponse queryResponse = cloudClient.query(queryAll);
- actualResultSize = queryResponse.getResults().getNumFound();
- if(expectedResultSize == actualResultSize) {
- return;
+ try {
+ QueryResponse queryResponse = cloudClient.query(queryAll);
+ actualResultSize = queryResponse.getResults().getNumFound();
+ if(expectedResultSize == actualResultSize) {
+ return;
+ }
+ } catch (SolrServerException | IOException e) {
+ log.warn("Querying solr threw an exception. This can be expected to happen during restarts.", e);
}
Thread.sleep(1000);
@@ -386,26 +334,83 @@ public class SharedFSAutoReplicaFailoverTest extends AbstractFullDistribZkTestBa
}
}
- private void assertSingleReplicationAndShardSize(String collection, int numSlices) {
- Collection<Slice> slices;
- slices = cloudClient.getZkStateReader().getClusterState().getCollection(collection).getActiveSlices();
- assertEquals(numSlices, slices.size());
- for (Slice slice : slices) {
- assertEquals(1, slice.getReplicas().size());
- }
+ // Overseer failover is not currently guaranteed with MoveReplica or Policy Framework
+ private List<JettySolrRunner> notOverseerJetties() throws IOException, SolrServerException {
+ CollectionAdminResponse response = CollectionAdminRequest.getOverseerStatus().process(cloudClient);
+ String overseerNode = (String) response.getResponse().get("leader");
+ return jettys.stream().filter(jetty -> !(jetty.getCoreContainer() != null && overseerNode.equals(jetty.getNodeName())))
+ .collect(Collectors.toList());
}
- private void assertSliceAndReplicaCount(String collection) {
- Collection<Slice> slices;
- slices = cloudClient.getZkStateReader().getClusterState().getCollection(collection).getActiveSlices();
- assertEquals(2, slices.size());
- for (Slice slice : slices) {
- assertEquals(2, slice.getReplicas().size());
+ private boolean waitingForReplicasNotLive(ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
+ Set<String> nodeNames = jetties.stream()
+ .filter(jetty -> jetty.getCoreContainer() != null)
+ .map(JettySolrRunner::getNodeName)
+ .collect(Collectors.toSet());
+ long timeout = System.nanoTime()
+ + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
+ boolean success = false;
+ while (!success && System.nanoTime() < timeout) {
+ success = true;
+ ClusterState clusterState = zkStateReader.getClusterState();
+ if (clusterState != null) {
+ Map<String, DocCollection> collections = clusterState.getCollectionsMap();
+ for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
+ DocCollection docCollection = entry.getValue();
+ Collection<Slice> slices = docCollection.getSlices();
+ for (Slice slice : slices) {
+ // only look at active shards
+ if (slice.getState() == Slice.State.ACTIVE) {
+ Collection<Replica> replicas = slice.getReplicas();
+ for (Replica replica : replicas) {
+ if (nodeNames.contains(replica.getNodeName())) {
+ boolean live = clusterState.liveNodesContain(replica
+ .getNodeName());
+ if (live) {
+ success = false;
+ }
+ }
+ }
+ }
+ }
+ }
+ if (!success) {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted");
+ }
+ }
+ }
}
+
+ return success;
}
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
+
+ private void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs) throws InterruptedException {
+ TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS);
+ while (!timeOut.hasTimedOut()) {
+ ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
+ Collection<Slice> slices = clusterState.getCollection(collection).getActiveSlices();
+ if (slices.size() == numSlices) {
+ boolean isMatch = true;
+ for (Slice slice : slices) {
+ int count = 0;
+ for (Replica replica : slice.getReplicas()) {
+ if (replica.getState() == Replica.State.ACTIVE && clusterState.liveNodesContain(replica.getNodeName())) {
+ count++;
+ }
+ }
+ if (count < numReplicas) {
+ isMatch = false;
+ }
+ }
+ if (isMatch) return;
+ }
+ Thread.sleep(200);
+ }
+ fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java b/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java
deleted file mode 100644
index 3423420..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/SharedFSAutoReplicaFailoverUtilsTest.java
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.solr.SolrTestCaseJ4;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.apache.solr.cloud.ClusterStateMockUtil.buildClusterState;
-
-public class SharedFSAutoReplicaFailoverUtilsTest extends SolrTestCaseJ4 {
- private static final String NODE6 = "baseUrl6_";
- private static final String NODE6_URL = "http://baseUrl6";
-
- private static final String NODE5 = "baseUrl5_";
- private static final String NODE5_URL = "http://baseUrl5";
-
- private static final String NODE4 = "baseUrl4_";
- private static final String NODE4_URL = "http://baseUrl4";
-
- private static final String NODE3 = "baseUrl3_";
- private static final String NODE3_URL = "http://baseUrl3";
-
- private static final String NODE2 = "baseUrl2_";
- private static final String NODE2_URL = "http://baseUrl2";
-
- private static final String NODE1 = "baseUrl1_";
- private static final String NODE1_URL = "http://baseUrl1";
-
- private List<ClusterStateMockUtil.Result> results;
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
- results = new ArrayList<>();
- }
-
- @After
- public void tearDown() throws Exception {
- super.tearDown();
- for (ClusterStateMockUtil.Result result : results) {
- result.close();
- }
- }
-
- @Test
- public void testGetBestCreateUrlBasics() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr1R*r2", NODE1);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull("Should be no live node to failover to", createUrl);
-
- result = buildClusterState(results, "csr1R*r2", NODE1, NODE2);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull("Only failover candidate node already has a replica", createUrl);
-
- result = buildClusterState(results, "csr1R*r2sr3", NODE1, NODE2, NODE3);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals("Node3 does not have a replica from the bad slice and should be the best choice", NODE3_URL, createUrl);
-
- result = buildClusterState(results, "csr1R*r2Fsr3r4r5", NODE1, NODE2, NODE3);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertTrue(createUrl.equals(NODE3_URL));
-
- result = buildClusterState(results, "csr1*r2r3sr3r3sr4", NODE1, NODE2, NODE3, NODE4);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE4_URL, createUrl);
-
- result = buildClusterState(results, "csr1*r2sr3r3sr4sr4", NODE1, NODE2, NODE3, NODE4);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL));
- }
-
- @Test
- public void testGetBestCreateUrlMultipleCollections() throws Exception {
-
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2csr2", NODE1);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*r2csr2", NODE1);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*r2csr2", NODE1, NODE2);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull(createUrl);
- }
-
- @Test
- public void testGetBestCreateUrlMultipleCollections2() {
-
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr*r2sr3cr2", NODE1);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*r2sr3cr2", NODE1, NODE2, NODE3);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE3_URL, createUrl);
- }
-
-
- @Test
- public void testGetBestCreateUrlMultipleCollections3() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr5r1sr4r2sr3r6csr2*r6sr5r3sr4r3", NODE1, NODE4, NODE5, NODE6);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE1_URL, createUrl);
- }
-
- @Test
- public void testGetBestCreateUrlMultipleCollections4() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr1r4sr3r5sr2r6csr5r6sr4r6sr5*r4", NODE6);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE6_URL, createUrl);
- }
-
- @Test
- public void testFailOverToEmptySolrInstance() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr1*r1sr1csr1", NODE2);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE2_URL, createUrl);
- }
-
- @Test
- public void testFavorForeignSlices() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2csr3r3", NODE2, NODE3);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE3_URL, createUrl);
-
- result = buildClusterState(results, "csr*sr2csr3r3r3r3r3r3r3", NODE2, NODE3);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE2_URL, createUrl);
- }
-
- @Test
- public void testCollectionMaxNodesPerShard() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*sr2", 1, 2, NODE2);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE2_URL, createUrl);
-
- result = buildClusterState(results, "csr*csr2r2", 1, 1, NODE2);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, null);
- assertEquals(NODE2_URL, createUrl);
- }
-
- @Test
- public void testMaxCoresPerNode() {
- ClusterStateMockUtil.Result result = buildClusterState(results, "csr*sr2", 1, 1, NODE2);
- String createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1);
- assertNull(createUrl);
-
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*sr2", 1, 2, NODE2);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
- assertEquals(NODE2_URL, createUrl);
-
- result = buildClusterState(results, "csr*sr2sr3sr4", 1, 1, NODE2, NODE3, NODE4);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 1);
- assertNull(createUrl);
-
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
- assertNull(createUrl);
-
- result = buildClusterState(results, "csr*sr2sr3sr4", 1, 2, NODE2, NODE3, NODE4);
- createUrl = OverseerAutoReplicaFailoverThread.getBestCreateUrl(result.reader, result.badReplica, 2);
- assertTrue(createUrl.equals(NODE3_URL) || createUrl.equals(NODE4_URL));
- }
-}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
new file mode 100644
index 0000000..7b1a0c3
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ClusterStateUtil;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.MapSolrParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.util.LogLevel;
+import org.apache.solr.util.TimeOut;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.solr.common.util.Utils.makeMap;
+
+@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
+public class AutoAddReplicasIntegrationTest extends SolrCloudTestCase {
+ private static final String COLLECTION1 = "testSimple1";
+ private static final String COLLECTION2 = "testSimple2";
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(3)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ JettySolrRunner jetty1 = cluster.getJettySolrRunner(0);
+ JettySolrRunner jetty2 = cluster.getJettySolrRunner(1);
+ JettySolrRunner jetty3 = cluster.getJettySolrRunner(2);
+ CollectionAdminRequest.createCollection(COLLECTION1, "conf", 2, 2)
+ .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
+ .setAutoAddReplicas(true)
+ .setMaxShardsPerNode(2)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection(COLLECTION2, "conf", 2, 2)
+ .setCreateNodeSet(jetty2.getNodeName()+","+jetty3.getNodeName())
+ .setAutoAddReplicas(false)
+ .setMaxShardsPerNode(2)
+ .process(cluster.getSolrClient());
+ // the number of cores in jetty1 (5) will be larger than jetty3 (1)
+ CollectionAdminRequest.createCollection("testSimple3", "conf", 3, 1)
+ .setCreateNodeSet(jetty1.getNodeName())
+ .setAutoAddReplicas(false)
+ .setMaxShardsPerNode(3)
+ .process(cluster.getSolrClient());
+
+ ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
+
+ // start the tests
+ JettySolrRunner lostJetty = random().nextBoolean() ? cluster.getJettySolrRunner(0) : cluster.getJettySolrRunner(1);
+ String lostNodeName = lostJetty.getNodeName();
+ List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION1, zkStateReader, lostNodeName);
+ lostJetty.stop();
+ waitForNodeLeave(lostNodeName);
+ waitForState("Waiting for collection " + COLLECTION1, COLLECTION1, clusterShape(2, 2));
+ checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION1);
+ lostJetty.start();
+ assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 90000));
+
+ // check cluster property is considered
+ disableAutoAddReplicasInCluster();
+ lostNodeName = jetty3.getNodeName();
+ jetty3.stop();
+ waitForNodeLeave(lostNodeName);
+ waitForState("Waiting for collection " + COLLECTION1, COLLECTION1, clusterShape(2, 1));
+ jetty3.start();
+ waitForState("Waiting for collection " + COLLECTION1, COLLECTION1, clusterShape(2, 2));
+ waitForState("Waiting for collection " + COLLECTION2, COLLECTION2, clusterShape(2, 2));
+ enableAutoAddReplicasInCluster();
+
+
+ // test for multiple collections
+ new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("collection", COLLECTION2);
+ params.set("autoAddReplicas", true);
+ return params;
+ }
+ }.process(cluster.getSolrClient());
+
+ lostNodeName = jetty2.getNodeName();
+ replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION2, zkStateReader, lostNodeName);
+ jetty2.stop();
+ waitForNodeLeave(lostNodeName);
+ waitForState("Waiting for collection " + COLLECTION1, COLLECTION1, clusterShape(2, 2));
+ waitForState("Waiting for collection " + COLLECTION2, COLLECTION2, clusterShape(2, 2));
+ checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION2);
+
+ // overseer failover test..
+ }
+
+ private void disableAutoAddReplicasInCluster() throws SolrServerException, IOException {
+ Map m = makeMap(
+ "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
+ "name", ZkStateReader.AUTO_ADD_REPLICAS,
+ "val", "false");
+ QueryRequest request = new QueryRequest(new MapSolrParams(m));
+ request.setPath("/admin/collections");
+ cluster.getSolrClient().request(request);
+ }
+
+ private void enableAutoAddReplicasInCluster() throws SolrServerException, IOException {
+ Map m = makeMap(
+ "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
+ "name", ZkStateReader.AUTO_ADD_REPLICAS);
+ QueryRequest request = new QueryRequest(new MapSolrParams(m));
+ request.setPath("/admin/collections");
+ cluster.getSolrClient().request(request);
+ }
+
+ private void checkSharedFsReplicasMovedCorrectly(List<Replica> replacedHdfsReplicas, ZkStateReader zkStateReader, String collection){
+ DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection);
+ for (Replica replica :replacedHdfsReplicas) {
+ boolean found = false;
+ String dataDir = replica.getStr("dataDir");
+ String ulogDir = replica.getStr("ulogDir");
+ for (Replica replica2 : docCollection.getReplicas()) {
+ if (dataDir.equals(replica2.getStr("dataDir")) && ulogDir.equals(replica2.getStr("ulogDir"))) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) fail("Can not found a replica with same dataDir and ulogDir as " + replica + " from:" + docCollection.getReplicas());
+ }
+ }
+
+ private List<Replica> getReplacedSharedFsReplicas(String collection, ZkStateReader zkStateReader, String lostNodeName) {
+ List<Replica> replacedHdfsReplicas = new ArrayList<>();
+ for (Replica replica : zkStateReader.getClusterState().getCollection(collection).getReplicas()) {
+ String dataDir = replica.getStr("dataDir");
+ if (replica.getNodeName().equals(lostNodeName) && dataDir != null) {
+ replacedHdfsReplicas.add(replica);
+ }
+ }
+
+ return replacedHdfsReplicas;
+ }
+
+ private void waitForNodeLeave(String lostNodeName) throws InterruptedException {
+ ZkStateReader reader = cluster.getSolrClient().getZkStateReader();
+ TimeOut timeOut = new TimeOut(20, TimeUnit.SECONDS);
+ while (reader.getClusterState().getLiveNodes().contains(lostNodeName)) {
+ Thread.sleep(100);
+ if (timeOut.hasTimedOut()) fail("Wait for " + lostNodeName + " to leave failed!");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
new file mode 100644
index 0000000..4a0495d
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.CloudDescriptor;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.cloud.ClusterStateUtil;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.util.TimeOut;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.solr.cloud.autoscaling.AutoScalingHandlerTest.createAutoScalingRequest;
+
+public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ configureCluster(3)
+ .addConfig("conf", configset("cloud-minimal"))
+ .configure();
+ }
+
+ @Test
+ public void testSimple() throws Exception {
+ JettySolrRunner jetty1 = cluster.getJettySolrRunner(0);
+ JettySolrRunner jetty2 = cluster.getJettySolrRunner(1);
+ JettySolrRunner jetty3 = cluster.getJettySolrRunner(2);
+
+ String collection1 = "testSimple1";
+ String collection2 = "testSimple2";
+ CollectionAdminRequest.createCollection(collection1, "conf", 2, 2)
+ .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
+ .setAutoAddReplicas(true)
+ .setMaxShardsPerNode(2)
+ .process(cluster.getSolrClient());
+ CollectionAdminRequest.createCollection(collection2, "conf", 1, 2)
+ .setCreateNodeSet(jetty2.getNodeName()+","+jetty3.getNodeName())
+ .setAutoAddReplicas(false)
+ .setMaxShardsPerNode(1)
+ .process(cluster.getSolrClient());
+ // the number of cores in jetty1 (5) will be larger than jetty3 (1)
+ CollectionAdminRequest.createCollection("testSimple3", "conf", 3, 1)
+ .setCreateNodeSet(jetty1.getNodeName())
+ .setAutoAddReplicas(false)
+ .setMaxShardsPerNode(3)
+ .process(cluster.getSolrClient());
+
+ // we remove the implicit created trigger, so the replicas won't be moved
+ String removeTriggerCommand = "{" +
+ "'remove-trigger' : {" +
+ "'name' : '.auto_add_replicas'," +
+ "'removeListeners': true" +
+ "}" +
+ "}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeTriggerCommand);
+ NamedList response = cluster.getSolrClient().request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ JettySolrRunner lostJetty = random().nextBoolean()? jetty1 : jetty2;
+ String lostNodeName = lostJetty.getNodeName();
+ List<CloudDescriptor> cloudDescriptors = lostJetty.getCoreContainer().getCores().stream()
+ .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor())
+ .collect(Collectors.toList());
+ lostJetty.stop();
+ waitForNodeLeave(lostNodeName);
+
+ List<SolrRequest> operations = getOperations(jetty3, lostNodeName);
+ assertOperations(collection1, operations, lostNodeName, cloudDescriptors, null);
+
+ lostJetty.start();
+ ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000);
+ String setClusterPreferencesCommand = "{" +
+ "'set-cluster-preferences': [" +
+ "{'minimize': 'cores','precision': 0}]" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPreferencesCommand);
+ response = cluster.getSolrClient().request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ lostJetty = random().nextBoolean()? jetty1 : jetty2;
+ lostNodeName = lostJetty.getNodeName();
+ cloudDescriptors = lostJetty.getCoreContainer().getCores().stream()
+ .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor())
+ .collect(Collectors.toList());
+ lostJetty.stop();
+ waitForNodeLeave(lostNodeName);
+
+ operations = getOperations(jetty3, lostNodeName);
+ assertOperations(collection1, operations, lostNodeName, cloudDescriptors, jetty3);
+
+ lostJetty.start();
+ assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000));
+
+ new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
+ @Override
+ public SolrParams getParams() {
+ ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
+ params.set("collection", collection1);
+ params.set("autoAddReplicas", false);
+ return params;
+ }
+ }.process(cluster.getSolrClient());
+ lostJetty = jetty1;
+ lostNodeName = lostJetty.getNodeName();
+ lostJetty.stop();
+ waitForNodeLeave(lostNodeName);
+ operations = getOperations(jetty3, lostNodeName);
+ assertNull(operations);
+ }
+
+ private void waitForNodeLeave(String lostNodeName) throws InterruptedException {
+ ZkStateReader reader = cluster.getSolrClient().getZkStateReader();
+ TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS);
+ while (reader.getClusterState().getLiveNodes().contains(lostNodeName)) {
+ Thread.sleep(100);
+ if (timeOut.hasTimedOut()) fail("Wait for " + lostNodeName + " to leave failed!");
+ }
+ }
+
+ private List<SolrRequest> getOperations(JettySolrRunner actionJetty, String lostNodeName) {
+ AutoAddReplicasPlanAction action = new AutoAddReplicasPlanAction();
+ TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName));
+ ActionContext context = new ActionContext(actionJetty.getCoreContainer(), null, new HashMap<>());
+ action.process(lostNode, context);
+ List<SolrRequest> operations = (List) context.getProperty("operations");
+ return operations;
+ }
+
+ private void assertOperations(String collection, List<SolrRequest> operations, String lostNodeName,
+ List<CloudDescriptor> cloudDescriptors, JettySolrRunner destJetty) {
+ assertEquals("Replicas of " + collection + " is not fully moved, operations="+operations,
+ cloudDescriptors.stream().filter(cd -> cd.getCollectionName().equals(collection)).count(), operations.size());
+ for (SolrRequest solrRequest : operations) {
+ assertTrue(solrRequest instanceof CollectionAdminRequest.MoveReplica);
+ SolrParams params = solrRequest.getParams();
+
+ assertEquals(params.get("collection"), collection);
+
+ String replica = params.get("replica");
+ boolean found = false;
+ Iterator<CloudDescriptor> it = cloudDescriptors.iterator();
+ while (it.hasNext()) {
+ CloudDescriptor cd = it.next();
+ if (cd.getCollectionName().equals(collection) && cd.getCoreNodeName().equals(replica)) {
+ found = true;
+ it.remove();
+ break;
+ }
+ }
+ assertTrue("Can not find "+replica+ " in node " + lostNodeName, found);
+
+ String targetNode = params.get("targetNode");
+ assertFalse("Target node match the lost one " + lostNodeName, lostNodeName.equals(targetNode));
+ if (destJetty != null) {
+ assertEquals("Target node is not as expectation", destJetty.getNodeName(), targetNode);
+ }
+ }
+
+ for (CloudDescriptor cd : cloudDescriptors) {
+ if (cd.getCollectionName().equals(collection)) {
+ fail("Exist replica which is not moved " + cd);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/ccf1bf72/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
index 719b0ac..335fdbc 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
@@ -22,10 +22,13 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
@@ -38,11 +41,14 @@ import org.apache.solr.common.util.ContentStream;
import org.apache.solr.common.util.ContentStreamBase;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.Utils;
+import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.data.Stat;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
+import static org.apache.solr.common.util.Utils.getObjectByPath;
/**
* Test for AutoScalingHandler
@@ -54,6 +60,31 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
configureCluster(2)
.addConfig(CONFIGSET_NAME, configset("cloud-minimal"))
.configure();
+ testAutoAddReplicas();
+ }
+
+ private static void testAutoAddReplicas() throws Exception {
+ TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS);
+ while (!timeOut.hasTimedOut()) {
+ byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ ZkNodeProps loaded = ZkNodeProps.load(data);
+ Map triggers = (Map) loaded.get("triggers");
+ if (triggers != null && triggers.containsKey(".auto_add_replicas")) {
+ Map<String, Object> autoAddReplicasTrigger = (Map<String, Object>) triggers.get(".auto_add_replicas");
+ assertNotNull(autoAddReplicasTrigger);
+ List<Map<String, Object>> actions = (List<Map<String, Object>>) autoAddReplicasTrigger.get("actions");
+ assertNotNull(actions);
+ assertEquals(2, actions.size());
+ assertEquals("auto_add_replicas_plan", actions.get(0).get("name").toString());
+ assertEquals("solr.AutoAddReplicasPlanAction", actions.get(0).get("class").toString());
+ break;
+ } else {
+ Thread.sleep(300);
+ }
+ }
+ if (timeOut.hasTimedOut()) {
+ fail("Timeout waiting for .auto_add_replicas being created");
+ }
}
@Before
@@ -63,6 +94,333 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
}
@Test
+ public void testSuspendTrigger() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String suspendEachCommand = "{\n" +
+ "\t\"suspend-trigger\" : {\n" +
+ "\t\t\"name\" : \"" + Policy.EACH + "\"\n" +
+ "\t}\n" +
+ "}";
+ String resumeEachCommand = "{\n" +
+ "\t\"resume-trigger\" : {\n" +
+ "\t\t\"name\" : \"" + Policy.EACH + "\"\n" +
+ "\t}\n" +
+ "}";
+ // these should be no-ops because there are no triggers, and it should succeed
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendEachCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ assertEquals(response.get("changed").toString(), "[]");
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeEachCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ assertEquals(response.get("changed").toString(), "[]");
+
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '10m'," +
+ "'enabled' : true}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '10m'," +
+ "'enabled' : true" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ String suspendTriggerCommand = "{\n" +
+ "\t\"suspend-trigger\" : {\n" +
+ "\t\t\"name\" : \"node_lost_trigger\"\n" +
+ "\t}\n" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ assertEquals(response.get("changed").toString(), "[node_lost_trigger]");
+
+ Stat stat = new Stat();
+ byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat, true);
+ ZkNodeProps loaded = ZkNodeProps.load(data);
+ Map<String, Object> triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(2, countNotImplicitTriggers(triggers));
+ assertTrue(triggers.containsKey("node_lost_trigger"));
+ assertTrue(triggers.containsKey("node_added_trigger"));
+ Map<String, Object> nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ assertEquals("false", nodeLostTrigger.get("enabled").toString());
+ Map<String, Object> nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
+ assertEquals(4, nodeAddedTrigger.size());
+ assertEquals("true", nodeAddedTrigger.get("enabled").toString());
+
+ suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : '" + Policy.EACH + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ List<String> changed = (List<String>)response.get("changed");
+ assertEquals(1, changed.size());
+ assertTrue(changed.contains("node_added_trigger"));
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(2, countNotImplicitTriggers(triggers));
+ nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ assertEquals("false", nodeLostTrigger.get("enabled").toString());
+ nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
+ assertEquals(4, nodeAddedTrigger.size());
+ assertEquals("false", nodeAddedTrigger.get("enabled").toString());
+
+ String resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : 'node_added_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ changed = (List<String>)response.get("changed");
+ assertEquals(1, changed.size());
+ assertTrue(changed.contains("node_added_trigger"));
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(2, countNotImplicitTriggers(triggers));
+ nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ assertEquals("false", nodeLostTrigger.get("enabled").toString());
+ nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
+ assertEquals(4, nodeAddedTrigger.size());
+ assertEquals("true", nodeAddedTrigger.get("enabled").toString());
+
+ resumeTriggerCommand = "{" +
+ "'resume-trigger' : {" +
+ "'name' : '" + Policy.EACH + "'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ changed = (List<String>)response.get("changed");
+ assertEquals(1, changed.size());
+ assertTrue(changed.contains("node_lost_trigger"));
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(2, countNotImplicitTriggers(triggers));
+ nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ assertEquals("true", nodeLostTrigger.get("enabled").toString());
+ nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
+ assertEquals(4, nodeAddedTrigger.size());
+ assertEquals("true", nodeAddedTrigger.get("enabled").toString());
+
+ suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'timeout' : '1h'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ changed = (List<String>)response.get("changed");
+ assertEquals(1, changed.size());
+ assertTrue(changed.contains("node_lost_trigger"));
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(2, countNotImplicitTriggers(triggers));
+ nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(5, nodeLostTrigger.size());
+ assertEquals("false", nodeLostTrigger.get("enabled").toString());
+ assertTrue(nodeLostTrigger.containsKey("resumeAt"));
+ }
+
+ @Test
+ public void test() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '10m'," +
+ "'enabled' : true," +
+ "'actions' : [" +
+ "{" +
+ "'name' : 'compute_plan'," +
+ "'class' : 'solr.ComputePlanAction'" +
+ "}]}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
+ byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ ZkNodeProps loaded = ZkNodeProps.load(data);
+ Map<String, Object> triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(1, countNotImplicitTriggers(triggers));
+ assertTrue(triggers.containsKey("node_lost_trigger"));
+ Map<String, Object> nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ List<Map<String, String>> actions = (List<Map<String, String>>) nodeLostTrigger.get("actions");
+ assertNotNull(actions);
+ assertEquals(1, actions.size());
+ assertEquals("600", nodeLostTrigger.get("waitFor").toString());
+
+ setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_lost_trigger'," +
+ "'event' : 'nodeLost'," +
+ "'waitFor' : '20m'," +
+ "'enabled' : false" +
+ "}}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(1, countNotImplicitTriggers(triggers));
+ assertTrue(triggers.containsKey("node_lost_trigger"));
+ nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
+ assertEquals(4, nodeLostTrigger.size());
+ assertEquals("1200", nodeLostTrigger.get("waitFor").toString());
+ assertEquals("false", nodeLostTrigger.get("enabled").toString());
+ actions = (List<Map<String, String>>) nodeLostTrigger.get("actions");
+ assertNotNull(actions);
+ assertEquals(2, actions.size());
+
+ String setListenerCommand = "{" +
+ "'set-listener' : " +
+ "{" +
+ "'name' : 'xyz'," +
+ "'trigger' : 'node_lost_trigger'," +
+ "'stage' : ['STARTED','ABORTED','SUCCEEDED']," +
+ "'beforeAction' : 'execute_plan'," +
+ "'class' : 'org.apache.solr.cloud.autoscaling.HttpTriggerListener'," +
+ "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ Map<String, Object> listeners = (Map<String, Object>) loaded.get("listeners");
+ assertNotNull(listeners);
+ assertEquals(2, listeners.size());
+ assertTrue(listeners.containsKey("xyz"));
+ Map<String, Object> xyzListener = (Map<String, Object>) listeners.get("xyz");
+ assertEquals(6, xyzListener.size());
+ assertEquals("org.apache.solr.cloud.autoscaling.HttpTriggerListener", xyzListener.get("class").toString());
+
+ String removeTriggerCommand = "{" +
+ "'remove-trigger' : {" +
+ "'name' : 'node_lost_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeTriggerCommand);
+ try {
+ solrClient.request(req);
+ fail("expected exception");
+ } catch (HttpSolrClient.RemoteExecutionException e) {
+ // expected
+ assertTrue(String.valueOf(getObjectByPath(e.getMetaData(),
+ false, "error/details[0]/errorMessages[0]")).contains("Cannot remove trigger: node_lost_trigger because it has active listeners: ["));
+ }
+
+ String removeListenerCommand = "{\n" +
+ "\t\"remove-listener\" : {\n" +
+ "\t\t\"name\" : \"xyz\"\n" +
+ "\t}\n" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeListenerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ listeners = (Map<String, Object>) loaded.get("listeners");
+ assertNotNull(listeners);
+ assertEquals(1, listeners.size());
+
+ removeTriggerCommand = "{" +
+ "'remove-trigger' : {" +
+ "'name' : 'node_lost_trigger'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, removeTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null, true);
+ loaded = ZkNodeProps.load(data);
+ triggers = (Map<String, Object>) loaded.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(0, countNotImplicitTriggers(triggers));
+
+ setListenerCommand = "{" +
+ "'set-listener' : {" +
+ "'name' : 'xyz'," +
+ "'trigger' : 'node_lost_trigger'," +
+ "'stage' : ['STARTED','ABORTED','SUCCEEDED']," +
+ "'beforeAction' : 'execute_plan'," +
+ "'class' : 'org.apache.solr.cloud.autoscaling.AutoScaling$HttpTriggerListener'," +
+ "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setListenerCommand);
+ try {
+ solrClient.request(req);
+ fail("should have thrown Exception");
+ } catch (HttpSolrClient.RemoteSolrException e) {
+ // expected
+ assertTrue(String.valueOf(getObjectByPath(((HttpSolrClient.RemoteExecutionException) e).getMetaData(),
+ false, "error/details[0]/errorMessages[0]")).contains("A trigger with the name node_lost_trigger does not exist"));
+ }
+ }
+
+ @Test
+ public void testErrorHandling() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+
+ String setClusterPolicyCommand = "{" +
+ " 'set-cluster-policy': [" +
+ " {'cores':'<10', 'node':'#ANY'}," +
+ " {'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'overseer', 'replica':0}" +
+ " ]" +
+ "}";
+ try {
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
+ solrClient.request(req);
+ fail("expect exception");
+ } catch (HttpSolrClient.RemoteExecutionException e) {
+ String message = String.valueOf(Utils.getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"));
+ assertTrue(message.contains("replica is required in"));
+ }
+
+ }
+ @Test
public void testPolicyAndPreferences() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
// add multiple policies
@@ -177,29 +535,21 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertNotNull(clusterPolicy);
assertEquals(3, clusterPolicy.size());
}
- public void testErrorHandling() throws Exception {
- CloudSolrClient solrClient = cluster.getSolrClient();
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- try {
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- solrClient.request(req);
- fail("expect exception");
- } catch (HttpSolrClient.RemoteExecutionException e) {
- String message = String.valueOf(Utils.getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"));
- assertTrue(message.contains("replica is required in"));
- }
- }
@Test
public void testReadApi() throws Exception {
CloudSolrClient solrClient = cluster.getSolrClient();
+ // first trigger
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger1'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '0s'," +
+ "'enabled' : true" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
String setClusterPolicyCommand = "{" +
" 'set-cluster-policy': [" +
@@ -208,8 +558,8 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" {'nodeRole':'overseer', 'replica':0}" +
" ]" +
"}";
- SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- NamedList<Object> response = solrClient.request(req);
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
+ response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
String setPreferencesCommand = "{" +
@@ -230,6 +580,9 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
" ]," +
" 'policy1':[" +
" {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
+ " ]," +
+ " 'policy2':[" +
+ " {'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}" +
" ]" +
"}}";
req = createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand);
@@ -239,6 +592,16 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
response = solrClient.request(req);
+ Map triggers = (Map) response.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(1, countNotImplicitTriggers(triggers));
+ assertTrue(triggers.containsKey("node_added_trigger1"));
+ Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1");
+ assertEquals(4, node_added_trigger1.size());
+ assertEquals(0L, node_added_trigger1.get("waitFor"));
+ assertEquals(true, node_added_trigger1.get("enabled"));
+ assertEquals(2, ((List)node_added_trigger1.get("actions")).size());
+
List<Map> clusterPrefs = (List<Map>) response.get("cluster-preferences");
assertNotNull(clusterPrefs);
assertEquals(4, clusterPrefs.size());
@@ -249,7 +612,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
Map policies = (Map) response.get("policies");
assertNotNull(policies);
- assertEquals(2, policies.size());
+ assertEquals(3, policies.size());
assertNotNull(policies.get("xyz"));
assertNotNull(policies.get("policy1"));
@@ -284,18 +647,7 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
assertNotNull(violations);
assertEquals(0, violations.size());
- String setEmptyClusterPolicyCommand = "{" +
- " 'set-cluster-policy': []" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setEmptyClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, "{set-cluster-policy : []}");
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // lets create a collection which violates the rule replicas < 2
+ // assert that when a cluster policy is in effect, using maxShardsPerNode throws an exception
try {
CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
create.setMaxShardsPerNode(10);
@@ -303,9 +655,19 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
fail();
} catch (Exception e) {
assertTrue(e.getMessage().contains("'maxShardsPerNode>0' is not supported when autoScaling policies are used"));
-
}
+ // temporarily increase replica limit in cluster policy so that we can create a collection with 6 replicas
+ String tempClusterPolicyCommand = "{" +
+ " 'set-cluster-policy': [" +
+ " {'cores':'<10', 'node':'#ANY'}," +
+ " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}," +
+ " {'nodeRole':'overseer', 'replica':0}" +
+ " ]" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
// lets create a collection which violates the rule replicas < 2
CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6);
@@ -317,11 +679,6 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
-
// get the diagnostics output again
req = createAutoScalingRequest(SolrRequest.METHOD.GET, "/diagnostics", null);
response = solrClient.request(req);
@@ -335,18 +692,99 @@ public class AutoScalingHandlerTest extends SolrCloudTestCase {
for (Map<String, Object> violation : violations) {
assertEquals("readApiTestViolations", violation.get("collection"));
assertEquals("shard1", violation.get("shard"));
- assertEquals(Utils.makeMap("replica", "3", "delta", -1), violation.get("violation"));
+ assertEquals(-1l, getObjectByPath(violation, true, "violation/delta"));
+ assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
assertNotNull(violation.get("clause"));
}
}
+ @Test
+ public void testConcurrentUpdates() throws Exception {
+ int COUNT = 50;
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ CountDownLatch updateLatch = new CountDownLatch(COUNT * 2);
+ Runnable r = () -> {
+ for (int i = 0; i < COUNT; i++) {
+ String setTriggerCommand = "{" +
+ "'set-trigger' : {" +
+ "'name' : 'node_added_trigger1'," +
+ "'event' : 'nodeAdded'," +
+ "'waitFor' : '0s'," +
+ "'enabled' : true" +
+ "}}";
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.POST, setTriggerCommand);
+ NamedList<Object> response = null;
+ try {
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+ } catch (Exception e) {
+ fail(e.toString());
+ } finally {
+ updateLatch.countDown();
+ }
+ }
+ };
+ Thread t1 = new Thread(r);
+ Thread t2 = new Thread(r);
+ t1.start();
+ t2.start();
+ boolean await = updateLatch.await(60, TimeUnit.SECONDS);
+ assertTrue("not all updates executed in time, remaining=" + updateLatch.getCount(), await);
+ SolrRequest req = createAutoScalingRequest(SolrRequest.METHOD.GET, null);
+ NamedList<Object> response = solrClient.request(req);
+
+ Map triggers = (Map) response.get("triggers");
+ assertNotNull(triggers);
+ assertEquals(1, countNotImplicitTriggers(triggers));
+ assertTrue(triggers.containsKey("node_added_trigger1"));
+ Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1");
+ assertEquals(4, node_added_trigger1.size());
+ assertEquals(0L, node_added_trigger1.get("waitFor"));
+ assertEquals(true, node_added_trigger1.get("enabled"));
+ assertEquals(2, ((List)node_added_trigger1.get("actions")).size());
+
+ }
+
+ private int countNotImplicitTriggers(Map triggers) {
+ if (triggers == null) return 0;
+ int count = 0;
+ for (Object trigger : triggers.keySet()) {
+ if (!trigger.toString().startsWith(".")) count++;
+ }
+ return count;
+ }
+
+ @Test
+ public void testDeleteUsedPolicy() throws Exception {
+ CloudSolrClient solrClient = cluster.getSolrClient();
+ // add multiple policies
+ String setPolicyCommand = "{'set-policy': {" +
+ " 'nodelete':[" +
+ " {'nodeRole':'overseer', 'replica':0}]}}";
+ solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, setPolicyCommand));
+ CollectionAdminRequest.createCollection("COLL1", "conf", 1, 1)
+ .setPolicy("nodelete")
+ .process(cluster.getSolrClient());
+ String removePolicyCommand = "{remove-policy : nodelete}";
+ createAutoScalingRequest(SolrRequest.METHOD.POST, removePolicyCommand);
+ try {
+ solrClient.request(createAutoScalingRequest(SolrRequest.METHOD.POST, removePolicyCommand));
+ fail("should have failed");
+ } catch (HttpSolrClient.RemoteExecutionException e) {
+ assertTrue(String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"))
+ .contains("is being used by collection"));
+ } catch (Exception e) {
+ fail("Only RemoteExecutionException expected");
+ }
+ solrClient.request(CollectionAdminRequest.deleteCollection("COLL1"));
+ }
public static SolrRequest createAutoScalingRequest(SolrRequest.METHOD m, String message) {
return createAutoScalingRequest(m, null, message);
}
static SolrRequest createAutoScalingRequest(SolrRequest.METHOD m, String subPath, String message) {
- boolean useV1 = random().nextBoolean();
+ boolean useV1 = false;
String path = useV1 ? "/admin/autoscaling" : "/cluster/autoscaling";
path += subPath != null ? subPath : "";
return useV1