You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2020/10/01 07:36:22 UTC
[lucene-solr] 02/06: @876 Remove autoscaling tests.
This is an automated email from the ASF dual-hosted git repository.
markrmiller pushed a commit to branch reference_impl_dev
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
commit 1d951afc1d151451cd82f927ae8ec44f8f180c63
Author: markrmiller@gmail.com <ma...@gmail.com>
AuthorDate: Wed Sep 30 19:40:16 2020 -0500
@876 Remove autoscaling tests.
---
.../AutoAddReplicasIntegrationTest.java | 476 ------
.../autoscaling/AutoAddReplicasPlanActionTest.java | 262 ----
.../cloud/autoscaling/AutoScalingHandlerTest.java | 1098 --------------
.../solr/cloud/autoscaling/CapturedEvent.java | 65 -
.../cloud/autoscaling/ComputePlanActionTest.java | 775 ----------
.../cloud/autoscaling/ExecutePlanActionTest.java | 376 -----
.../HdfsAutoAddReplicasIntegrationTest.java | 49 -
.../cloud/autoscaling/HttpTriggerListenerTest.java | 210 ---
.../IndexSizeTriggerMixedBoundsTest.java | 374 -----
.../IndexSizeTriggerSizeEstimationTest.java | 324 ----
.../cloud/autoscaling/IndexSizeTriggerTest.java | 782 ----------
.../autoscaling/MetricTriggerIntegrationTest.java | 246 ----
.../solr/cloud/autoscaling/MetricTriggerTest.java | 140 --
.../NodeAddedTriggerIntegrationTest.java | 322 ----
.../cloud/autoscaling/NodeAddedTriggerTest.java | 344 -----
.../NodeLostTriggerIntegrationTest.java | 359 -----
.../cloud/autoscaling/NodeLostTriggerTest.java | 395 -----
.../autoscaling/NodeMarkersRegistrationTest.java | 361 -----
.../cloud/autoscaling/RestoreTriggerStateTest.java | 169 ---
.../ScheduledMaintenanceTriggerTest.java | 381 -----
.../ScheduledTriggerIntegrationTest.java | 151 --
.../cloud/autoscaling/ScheduledTriggerTest.java | 143 --
.../SearchRateTriggerIntegrationTest.java | 747 ----------
.../cloud/autoscaling/SearchRateTriggerTest.java | 360 -----
.../cloud/autoscaling/SystemLogListenerTest.java | 294 ----
.../solr/cloud/autoscaling/TestPolicyCloud.java | 545 -------
.../TriggerCooldownIntegrationTest.java | 233 ---
.../cloud/autoscaling/TriggerEventQueueTest.java | 100 --
.../cloud/autoscaling/TriggerIntegrationTest.java | 729 ---------
.../TriggerSetPropertiesIntegrationTest.java | 278 ----
.../autoscaling/sim/SimSolrCloudTestCase.java | 257 ----
.../sim/TestSimClusterStateProvider.java | 230 ---
.../autoscaling/sim/TestSimComputePlanAction.java | 390 -----
.../sim/TestSimDistribStateManager.java | 386 -----
.../autoscaling/sim/TestSimDistributedQueue.java | 222 ---
.../autoscaling/sim/TestSimExecutePlanAction.java | 225 ---
.../autoscaling/sim/TestSimExtremeIndexing.java | 148 --
.../sim/TestSimGenericDistributedQueue.java | 40 -
.../cloud/autoscaling/sim/TestSimLargeCluster.java | 878 -----------
.../autoscaling/sim/TestSimNodeAddedTrigger.java | 331 -----
.../autoscaling/sim/TestSimNodeLostTrigger.java | 349 -----
.../cloud/autoscaling/sim/TestSimPolicyCloud.java | 381 -----
.../cloud/autoscaling/sim/TestSimScenario.java | 171 ---
.../autoscaling/sim/TestSimTriggerIntegration.java | 1555 --------------------
.../solr/cloud/autoscaling/sim/TestSimUtils.java | 97 --
.../autoscaling/sim/TestSnapshotCloudManager.java | 278 ----
46 files changed, 17026 deletions(-)
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
deleted file mode 100644
index 92732c6..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasIntegrationTest.java
+++ /dev/null
@@ -1,476 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import static org.apache.solr.common.util.Utils.makeMap;
-
-import java.lang.invoke.MethodHandles;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.QueryRequest;
-import org.apache.solr.client.solrj.request.V2Request;
-import org.apache.solr.cloud.MiniSolrCloudCluster;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.CollectionStatePredicate;
-import org.apache.solr.common.cloud.ClusterStateUtil;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.MapSolrParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-@org.apache.solr.util.LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.autoscaling.NodeLostTrigger=TRACE;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG")
-@Ignore // nocommit this is removed in master
-public class AutoAddReplicasIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
-
- protected String getConfigSet() {
- return "cloud-minimal";
- }
-
- @Before
- public void setupCluster() throws Exception {
- configureCluster(3)
- .addConfig("conf", configset(getConfigSet()))
- .withSolrXml(TEST_PATH().resolve("solr.xml"))
- .configure();
-
- new V2Request.Builder("/cluster")
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload("{set-obj-property:{defaults : {cluster: {useLegacyReplicaAssignment:true}}}}")
- .build()
- .process(cluster.getSolrClient());
-
- new V2Request.Builder("/cluster/autoscaling")
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload("{'set-trigger':{'name':'.auto_add_replicas','event':'nodeLost','waitFor':'5s','enabled':'true','actions':[{'name':'auto_add_replicas_plan','class':'solr.AutoAddReplicasPlanAction'},{'name':'auto_add_replicas_plan','class':'solr.ExecutePlanAction'}]}}")
- .build()
- .process(cluster.getSolrClient());
- }
-
- @After
- public void tearDown() throws Exception {
- try {
- shutdownCluster();
- } finally {
- super.tearDown();
- }
- }
-
- /**
- * Test that basic autoAddReplicaLogic kicks in when a node is lost
- */
- @Test
- public void testSimple() throws Exception {
- final String COLLECTION = "test_simple";
- final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
- final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
- if (log.isInfoEnabled()) {
- log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
- jetty1.getNodeName(), jetty1.getLocalPort(),
- jetty2.getNodeName(), jetty2.getLocalPort());
- }
-
- CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(true)
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(COLLECTION, 2, 4);
-
- // start the tests
- JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
- String lostNodeName = lostJetty.getNodeName();
- List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
- if (log.isInfoEnabled()) {
- log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
- waitForNodeLeave(lostNodeName);
-
- waitForState(COLLECTION + "=(2,4) w/o down replicas",
- COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
-
- checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);
-
- if (log.isInfoEnabled()) {
- log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.start();
-
- waitForNodeLive(lostJetty);
-
- assertTrue("Timeout waiting for all live and active",
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
-
- }
-
- /**
- * Test that basic autoAddReplicaLogic logic is <b>not</b> used if the cluster prop for it is disabled
- * (even if sys prop is set after collection is created)
- */
- @Test
- public void testClusterPropOverridesCollecitonProp() throws Exception {
- final String COLLECTION = "test_clusterprop";
- final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
- final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
-
- if (log.isInfoEnabled()) {
- log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
- jetty1.getNodeName(), jetty1.getLocalPort(),
- jetty2.getNodeName(), jetty2.getLocalPort());
- }
-
- CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(true)
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(COLLECTION, 2, 4);
-
- // check cluster property is considered
- disableAutoAddReplicasInCluster();
-
- JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
- String lostNodeName = lostJetty.getNodeName();
- List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
-
- if (log.isInfoEnabled()) {
- log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
-
- waitForNodeLeave(lostNodeName);
-
- waitForState(COLLECTION + "=(2,2)", COLLECTION,
- clusterShape(2, 2), 90, TimeUnit.SECONDS);
-
-
- if (log.isInfoEnabled()) {
- log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.start();
-
- waitForNodeLive(lostJetty);
-
- assertTrue("Timeout waiting for all live and active",
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
-
- waitForState(COLLECTION + "=(2,4) w/o down replicas",
- COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
-
- }
-
- /**
- * Test that we can modify a collection after creation to add autoAddReplicas.
- */
- @Test
- public void testAddCollectionPropAfterCreation() throws Exception {
- final String COLLECTION = "test_addprop";
- final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
- final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
-
- if (log.isInfoEnabled()) {
- log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
- jetty1.getNodeName(), jetty1.getLocalPort(),
- jetty2.getNodeName(), jetty2.getLocalPort());
- }
-
- CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(false) // NOTE: false
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(COLLECTION, 2, 4);
-
- log.info("Modifying {} to use autoAddReplicas", COLLECTION);
- new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
- @Override
- public SolrParams getParams() {
- ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
- params.set("collection", COLLECTION);
- params.set("autoAddReplicas", true);
- return params;
- }
- }.process(cluster.getSolrClient());
-
- JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
- String lostNodeName = lostJetty.getNodeName();
- List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
-
- if (log.isInfoEnabled()) {
- log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
-
- waitForNodeLeave(lostNodeName);
-
- waitForState(COLLECTION + "=(2,4) w/o down replicas",
- COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
- checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);
-
- if (log.isInfoEnabled()) {
- log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.start();
-
- waitForNodeLive(lostJetty);
-
- assertTrue("Timeout waiting for all live and active",
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
- }
-
- /**
- * Test a specific sequence of problematic events:
- * <ul>
- * <li>create a collection with autoAddReplicas=<b>false</b></li>
- * <li>stop a nodeX in use by the collection</li>
- * <li>re-start nodeX</li>
- * <li>set autoAddReplicas=<b>true</b></li>
- * <li>re-stop nodeX</li>
- * </ul>
- */
- @Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13811")
- public void testRapidStopStartStopWithPropChange() throws Exception {
-
- // This is the collection we'll be focused on in our testing...
- final String COLLECTION = "test_stoptwice";
- // This is a collection we'll use as a "marker" to ensure we "wait" for the
- // autoAddReplicas logic (via NodeLostTrigger) to kick in at least once before proceeding...
- final String ALT_COLLECTION = "test_dummy";
-
- final ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
- final JettySolrRunner jetty1 = cluster.getJettySolrRunner(1);
- final JettySolrRunner jetty2 = cluster.getJettySolrRunner(2);
-
- if (log.isInfoEnabled()) {
- log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", COLLECTION,
- jetty1.getNodeName(), jetty1.getLocalPort(),
- jetty2.getNodeName(), jetty2.getLocalPort());
- }
-
- CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(false) // NOTE: false
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
-
- if (log.isInfoEnabled()) {
- log.info("Creating {} using jetty1:{}/{} and jetty2:{}/{}", ALT_COLLECTION,
- jetty1.getNodeName(), jetty1.getLocalPort(),
- jetty2.getNodeName(), jetty2.getLocalPort());
- }
-
- CollectionAdminRequest.createCollection(ALT_COLLECTION, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(true) // NOTE: true
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(COLLECTION, 2, 4);
- cluster.waitForActiveCollection(ALT_COLLECTION, 2, 4);
-
- JettySolrRunner lostJetty = random().nextBoolean() ? jetty1 : jetty2;
- String lostNodeName = lostJetty.getNodeName();
- List<Replica> replacedHdfsReplicas = getReplacedSharedFsReplicas(COLLECTION, zkStateReader, lostNodeName);
-
- if (log.isInfoEnabled()) {
- log.info("Stopping random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
- waitForNodeLeave(lostNodeName);
-
- // ensure that our marker collection indicates that the autoAddReplicas logic
- // has detected the down node and done some processing
- waitForState(ALT_COLLECTION + "=(2,4) w/o down replicas",
- ALT_COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
-
- waitForState(COLLECTION + "=(2,2)", COLLECTION, clusterShape(2, 2));
-
- if (log.isInfoEnabled()) {
- log.info("Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.start();
- // save time, don't bother waiting for lostJetty to start until after updating collection prop...
-
- log.info("Modifying {} to use autoAddReplicas", COLLECTION);
- new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
- @Override
- public SolrParams getParams() {
- ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
- params.set("collection", COLLECTION);
- params.set("autoAddReplicas", true);
- return params;
- }
- }.process(cluster.getSolrClient());
-
- // make sure lostJetty is fully up before stopping again...
- waitForNodeLive(lostJetty);
-
- if (log.isInfoEnabled()) {
- log.info("Re-Stopping (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
- waitForNodeLeave(lostNodeName);
-
- // TODO: this is the problematic situation...
- // wether or not NodeLostTrigger noticed that lostJetty was re-started and shutdown *again*
- // and that the new auoAddReplicas=true since the last time lostJetty was shutdown is respected
- waitForState(COLLECTION + "=(2,4) w/o down replicas",
- COLLECTION, clusterShapeNoDownReplicas(2,4), 90, TimeUnit.SECONDS);
- checkSharedFsReplicasMovedCorrectly(replacedHdfsReplicas, zkStateReader, COLLECTION);
-
- if (log.isInfoEnabled()) {
- log.info("Re-Re-starting (same) random node: {} / {}", lostNodeName, lostJetty.getLocalPort());
- }
- lostJetty.start();
-
- waitForNodeLive(lostJetty);
-
- assertTrue("Timeout waiting for all live and active",
- ClusterStateUtil.waitForAllActiveAndLiveReplicas(zkStateReader, 90000));
- }
-
- private void disableAutoAddReplicasInCluster() throws SolrServerException, IOException {
- Map m = makeMap(
- "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
- "name", ZkStateReader.AUTO_ADD_REPLICAS,
- "val", "false");
- QueryRequest request = new QueryRequest(new MapSolrParams(m));
- request.setPath("/admin/collections");
- cluster.getSolrClient().request(request);
- }
-
- private void enableAutoAddReplicasInCluster() throws SolrServerException, IOException {
- Map m = makeMap(
- "action", CollectionParams.CollectionAction.CLUSTERPROP.toLower(),
- "name", ZkStateReader.AUTO_ADD_REPLICAS);
- QueryRequest request = new QueryRequest(new MapSolrParams(m));
- request.setPath("/admin/collections");
- cluster.getSolrClient().request(request);
- }
-
- private void checkSharedFsReplicasMovedCorrectly(List<Replica> replacedHdfsReplicas, ZkStateReader zkStateReader, String collection){
- DocCollection docCollection = zkStateReader.getClusterState().getCollection(collection);
- for (Replica replica :replacedHdfsReplicas) {
- boolean found = false;
- String dataDir = replica.getStr("dataDir");
- String ulogDir = replica.getStr("ulogDir");
- for (Replica replica2 : docCollection.getReplicas()) {
- if (dataDir.equals(replica2.getStr("dataDir")) && ulogDir.equals(replica2.getStr("ulogDir"))) {
- found = true;
- break;
- }
- }
- if (!found) fail("Can not found a replica with same dataDir and ulogDir as " + replica + " from:" + docCollection.getReplicas());
- }
- }
-
- private List<Replica> getReplacedSharedFsReplicas(String collection, ZkStateReader zkStateReader, String lostNodeName) {
- List<Replica> replacedHdfsReplicas = new ArrayList<>();
- for (Replica replica : zkStateReader.getClusterState().getCollection(collection).getReplicas()) {
- String dataDir = replica.getStr("dataDir");
- if (replica.getNodeName().equals(lostNodeName) && dataDir != null) {
- replacedHdfsReplicas.add(replica);
- }
- }
-
- return replacedHdfsReplicas;
- }
-
- /**
- * {@link MiniSolrCloudCluster#waitForNode} Doesn't check isRunning first, and we don't want to
- * use {@link MiniSolrCloudCluster#waitForAllNodes} because we don't want to waste cycles checking
- * nodes we aren't messing with
- */
- private void waitForNodeLive(final JettySolrRunner jetty)
- throws InterruptedException, TimeoutException, IOException {
- if (log.isInfoEnabled()) {
- log.info("waitForNodeLive: {}/{}", jetty.getNodeName(), jetty.getLocalPort());
- }
-
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while(!timeout.hasTimedOut()) {
- if (jetty.isRunning()) {
- break;
- }
- try {
- Thread.sleep(100);
- } catch (InterruptedException e) {
- // ignore
- }
- }
- if (timeout.hasTimedOut()) {
- throw new TimeoutException("Waiting for Jetty to stop timed out");
- }
- cluster.waitForNode(jetty, 30);
- }
-
- private void waitForNodeLeave(String lostNodeName) throws InterruptedException, TimeoutException {
- log.info("waitForNodeLeave: {}", lostNodeName);
- ZkStateReader reader = cluster.getSolrClient().getZkStateReader();
- reader.waitForLiveNodes(30, TimeUnit.SECONDS, (o, n) -> !n.contains(lostNodeName));
- }
-
-
- private static CollectionStatePredicate clusterShapeNoDownReplicas(final int expectedShards,
- final int expectedReplicas) {
- return (liveNodes, collectionState)
- -> (clusterShape(expectedShards, expectedReplicas).matches(liveNodes, collectionState)
- && collectionState.getReplicas().size() == expectedReplicas);
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
deleted file mode 100644
index 14a07c1..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoAddReplicasPlanActionTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Optional;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.V2Request;
-import org.apache.solr.cloud.CloudDescriptor;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrException;
-import org.apache.solr.common.cloud.ClusterStateUtil;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.SuppressForbidden;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-@Ignore // nocommit this is removed in master
-public class AutoAddReplicasPlanActionTest extends SolrCloudTestCase{
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- System.setProperty("solr.httpclient.retries", "4");
- System.setProperty("solr.retries.on.forward", "1");
- System.setProperty("solr.retries.to.followers", "1");
-
- }
-
- @Before
- public void beforeTest() throws Exception {
- configureCluster(3)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- new V2Request.Builder("/cluster")
- .withMethod(SolrRequest.METHOD.POST)
- .withPayload("{set-obj-property:{defaults : {cluster: {useLegacyReplicaAssignment:true}}}}")
- .build()
- .process(cluster.getSolrClient());
- }
-
- @After
- public void afterTest() throws Exception {
- shutdownCluster();
- }
-
- @Test
- @Ignore // nocommit debug
- //Commented out 11-Dec-2018 @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13028")
- public void testSimple() throws Exception {
- JettySolrRunner jetty1 = cluster.getJettySolrRunner(0);
- JettySolrRunner jetty2 = cluster.getJettySolrRunner(1);
- JettySolrRunner jetty3 = cluster.getJettySolrRunner(2);
-
- String collection1 = "testSimple1";
- String collection2 = "testSimple2";
- String collection3 = "testSimple3";
- CollectionAdminRequest.createCollection(collection1, "conf", 2, 2)
- .setCreateNodeSet(jetty1.getNodeName()+","+jetty2.getNodeName())
- .setAutoAddReplicas(true)
- .setMaxShardsPerNode(2)
- .process(cluster.getSolrClient());
- CollectionAdminRequest.createCollection(collection2, "conf", 1, 2)
- .setCreateNodeSet(jetty2.getNodeName()+","+jetty3.getNodeName())
- .setAutoAddReplicas(false)
- .setMaxShardsPerNode(1)
- .process(cluster.getSolrClient());
- // the number of cores in jetty1 (6) will be larger than jetty3 (1)
- CollectionAdminRequest.createCollection(collection3, "conf", 3, 1)
- .setCreateNodeSet(jetty1.getNodeName())
- .setAutoAddReplicas(false)
- .setMaxShardsPerNode(3)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collection1, 2, 4);
- cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection(collection3, 3, 3);
-
- // we remove the implicit created trigger, so the replicas won't be moved
- String removeTriggerCommand = "{" +
- "'remove-trigger' : {" +
- "'name' : '.auto_add_replicas'," +
- "'removeListeners': true" +
- "}" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand);
- NamedList response = cluster.getSolrClient().request(req);
- assertEquals(response.get("result").toString(), "success");
-
- JettySolrRunner lostJetty = cluster.getRandomJetty(random(), jetty3);
- String lostNodeName = lostJetty.getNodeName();
- List<CloudDescriptor> cloudDescriptors = lostJetty.getCoreContainer().getCores().stream()
- .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor())
- .collect(Collectors.toList());
-
- ZkStateReader reader = cluster.getSolrClient().getZkStateReader();
-
- lostJetty.stop();
-
- cluster.waitForJettyToStop(lostJetty);
-
- reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName));
-
-
- List<SolrRequest> operations = getOperations(cluster.getRandomJetty(random(), lostJetty), lostNodeName);
- assertOperations(collection1, operations, lostNodeName, cloudDescriptors, null);
-
- lostJetty.start();
- cluster.waitForNode(lostJetty, 15);
-
- cluster.waitForActiveCollection(collection1, 2, 4);
- cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection(collection3, 3, 3);
-
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 10000));
-
- String setClusterPreferencesCommand = "{" +
- "'set-cluster-preferences': [" +
- "{'minimize': 'cores','precision': 0}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand);
-
- // you can hit a stale connection from pool when restarting jetty
- try (CloudSolrClient cloudClient = new CloudSolrClient.Builder(Collections.singletonList(cluster.getZkServer().getZkAddress()),
- Optional.empty())
- .withSocketTimeout(15000).withConnectionTimeout(5000).build()) {
- response = cloudClient.request(req);
- }
-
- assertEquals(response.get("result").toString(), "success");
-
- lostJetty = cluster.getRandomJetty(random(), jetty3);
- String lostNodeName2 = lostJetty.getNodeName();
- cloudDescriptors = lostJetty.getCoreContainer().getCores().stream()
- .map(solrCore -> solrCore.getCoreDescriptor().getCloudDescriptor())
- .collect(Collectors.toList());
-
-
-
- lostJetty.stop();
- cluster.waitForJettyToStop(lostJetty);
- reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName2));
-
- try {
- operations = getOperations(jetty3, lostNodeName2);
- } catch (SolrException e) {
- // we might get a stale connection from the pool after jetty restarts
- operations = getOperations(jetty3, lostNodeName2);
- }
-
- assertOperations(collection1, operations, lostNodeName2, cloudDescriptors, jetty3);
-
- lostJetty.start();
- cluster.waitForNode(lostJetty, 10000);
- cluster.waitForActiveCollection(collection1, 2, 4);
- cluster.waitForActiveCollection(collection2, 1, 2);
- cluster.waitForActiveCollection(collection3, 3, 3);
-
- assertTrue("Timeout waiting for all live and active", ClusterStateUtil.waitForAllActiveAndLiveReplicas(cluster.getSolrClient().getZkStateReader(), 30000));
-
- new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.MODIFYCOLLECTION) {
- @Override
- public SolrParams getParams() {
- ModifiableSolrParams params = (ModifiableSolrParams) super.getParams();
- params.set("collection", collection1);
- params.set("autoAddReplicas", false);
- return params;
- }
- }.process(cluster.getSolrClient());
- lostJetty = jetty1;
- String lostNodeName3 = lostJetty.getNodeName();
-
- lostJetty.stop();
- cluster.waitForJettyToStop(lostJetty);
- reader.waitForLiveNodes(30, TimeUnit.SECONDS, missingLiveNode(lostNodeName3));
-
- operations = getOperations(jetty3, lostNodeName3);
- assertNull(operations);
- }
-
- @SuppressForbidden(reason = "Needs currentTimeMillis to create unique id")
- private List<SolrRequest> getOperations(JettySolrRunner actionJetty, String lostNodeName) throws Exception {
- try (AutoAddReplicasPlanAction action = new AutoAddReplicasPlanAction()) {
- action.configure(actionJetty.getCoreContainer().getResourceLoader(), actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), new HashMap<>());
- TriggerEvent lostNode = new NodeLostTrigger.NodeLostEvent(TriggerEventType.NODELOST, ".auto_add_replicas", Collections.singletonList(System.currentTimeMillis()), Collections.singletonList(lostNodeName), CollectionParams.CollectionAction.MOVEREPLICA.toLower());
- ActionContext context = new ActionContext(actionJetty.getCoreContainer().getZkController().getSolrCloudManager(), null, new HashMap<>());
- action.process(lostNode, context);
- List<SolrRequest> operations = (List) context.getProperty("operations");
- return operations;
- }
- }
-
- private void assertOperations(String collection, List<SolrRequest> operations, String lostNodeName,
- List<CloudDescriptor> cloudDescriptors, JettySolrRunner destJetty) {
- assertEquals("Replicas of " + collection + " is not fully moved, operations="+operations,
- cloudDescriptors.stream().filter(cd -> cd.getCollectionName().equals(collection)).count(), operations.size());
- for (SolrRequest solrRequest : operations) {
- assertTrue(solrRequest instanceof CollectionAdminRequest.MoveReplica);
- SolrParams params = solrRequest.getParams();
-
- assertEquals(params.get("collection"), collection);
-
- String replica = params.get("replica");
- boolean found = false;
- Iterator<CloudDescriptor> it = cloudDescriptors.iterator();
- while (it.hasNext()) {
- CloudDescriptor cd = it.next();
- if (cd.getCollectionName().equals(collection) && cd.getCoreNodeName().equals(replica)) {
- found = true;
- it.remove();
- break;
- }
- }
- assertTrue("Can not find "+replica+ " in node " + lostNodeName, found);
-
- String targetNode = params.get("targetNode");
- assertFalse("Target node match the lost one " + lostNodeName, lostNodeName.equals(targetNode));
- if (destJetty != null) {
- assertEquals("Target node is not as expectation", destJetty.getNodeName(), targetNode);
- }
- }
-
- for (CloudDescriptor cd : cloudDescriptors) {
- if (cd.getCollectionName().equals(collection)) {
- fail("Exist replica which is not moved " + cd);
- }
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
deleted file mode 100644
index c30a460..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/AutoScalingHandlerTest.java
+++ /dev/null
@@ -1,1098 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.CollectionAdminResponse;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.MapSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.data.Stat;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.Type.repair;
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-import static org.apache.solr.common.util.Utils.getObjectByPath;
-
-/**
- * Test for AutoScalingHandler
- */
-
-@Ignore // nocommit this is removed in master
-public class AutoScalingHandlerTest extends SolrCloudTestCase {
- final static String CONFIGSET_NAME = "conf";
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig(CONFIGSET_NAME, configset("cloud-minimal"))
- .configure();
- testAutoAddReplicas();
- }
-
- private static void testAutoAddReplicas() throws Exception {
- TimeOut timeOut = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- while (!timeOut.hasTimedOut()) {
- byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- ZkNodeProps loaded = ZkNodeProps.load(data);
- Map triggers = (Map) loaded.get("triggers");
- if (triggers != null && triggers.containsKey(".auto_add_replicas")) {
- Map<String, Object> autoAddReplicasTrigger = (Map<String, Object>) triggers.get(".auto_add_replicas");
- assertNotNull(autoAddReplicasTrigger);
- List<Map<String, Object>> actions = (List<Map<String, Object>>) autoAddReplicasTrigger.get("actions");
- assertNotNull(actions);
- assertEquals(2, actions.size());
- assertEquals("auto_add_replicas_plan", actions.get(0).get("name").toString());
- assertEquals("solr.AutoAddReplicasPlanAction", actions.get(0).get("class").toString());
- break;
- } else {
- Thread.sleep(300);
- }
- }
- if (timeOut.hasTimedOut()) {
- fail("Timeout waiting for .auto_add_replicas being created");
- }
- }
-
- @Before
- public void beforeTest() throws Exception {
- // clear any persisted auto scaling configuration
- zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
- }
-
- public void testSuggestionsWithPayload() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String COLLNAME = "testSuggestionsWithPayload.COLL";
- CollectionAdminResponse adminResponse = CollectionAdminRequest.createCollection(COLLNAME, CONFIGSET_NAME, 1, 2)
- .setMaxShardsPerNode(4)
- .process(solrClient);
- cluster.waitForActiveCollection(COLLNAME, 1, 2);
- DocCollection collection = solrClient.getClusterStateProvider().getCollection(COLLNAME);
- Replica aReplica = collection.getReplicas().get(0);
-
- String configPayload = "{\n" +
- " 'cluster-policy': [{'replica': 0, 'node': '_NODE'}]\n" +
- "}";
- configPayload = configPayload.replaceAll("_NODE", aReplica.getNodeName());
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/suggestions", configPayload);
- NamedList<Object> response = solrClient.request(req);
- assertFalse(((Collection) response.get("suggestions")).isEmpty());
- String replicaName = response._getStr("suggestions[0]/operation/command/move-replica/replica", null);
- boolean[] passed = new boolean[]{false};
- collection.forEachReplica((s, replica) -> {
- if (replica.getName().equals(replicaName) && replica.getNodeName().equals(aReplica.getNodeName())) {
- passed[0] = true;
- }
- });
- assertTrue(passed[0]);
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/suggestions", configPayload, new MapSolrParams(Collections.singletonMap("type", repair.name())));
- response = solrClient.request(req);
- assertTrue(((Collection) response.get("suggestions")).isEmpty());
-
- CollectionAdminRequest.deleteCollection(COLLNAME)
- .process(cluster.getSolrClient());
- }
- public void testDiagnosticsWithPayload() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String COLLNAME = "testDiagnosticsWithPayload.COLL";
- CollectionAdminResponse adminResponse = CollectionAdminRequest.createCollection(COLLNAME, CONFIGSET_NAME, 1, 2)
- .setMaxShardsPerNode(4)
- .process(solrClient);
- cluster.waitForActiveCollection(COLLNAME, 1, 2);
- DocCollection collection = solrClient.getClusterStateProvider().getCollection(COLLNAME);
- Replica aReplica = collection.getReplicas().get(0);
-
- String configPayload = "{\n" +
- " 'cluster-policy': [{'replica': 0, 'node': '_NODE'}]\n" +
- "}";
- configPayload = configPayload.replaceAll("_NODE", aReplica.getNodeName());
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, "/diagnostics", configPayload);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response._getStr("diagnostics/violations[0]/node",null),response._getStr("diagnostics/violations[0]/node",null));
- CollectionAdminRequest.deleteCollection(COLLNAME)
- .process(cluster.getSolrClient());
- }
-
- @Test
- public void testSuspendTrigger() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String suspendEachCommand = "{\n" +
- "\t\"suspend-trigger\" : {\n" +
- "\t\t\"name\" : \"" + Policy.EACH + "\"\n" +
- "\t}\n" +
- "}";
- String resumeEachCommand = "{\n" +
- "\t\"resume-trigger\" : {\n" +
- "\t\t\"name\" : \"" + Policy.EACH + "\"\n" +
- "\t}\n" +
- "}";
- // these should be no-ops because there are no triggers, and it should succeed
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendEachCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- assertEquals(response.get("changed").toString(), "[]");
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeEachCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- assertEquals(response.get("changed").toString(), "[]");
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '10m'," +
- "'enabled' : true}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '10m'," +
- "'enabled' : true" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String suspendTriggerCommand = "{\n" +
- "\t\"suspend-trigger\" : {\n" +
- "\t\t\"name\" : \"node_lost_trigger\"\n" +
- "\t}\n" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- assertEquals(response.get("changed").toString(), "[node_lost_trigger]");
-
- Stat stat = new Stat();
- byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, stat);
- ZkNodeProps loaded = ZkNodeProps.load(data);
- Map<String, Object> triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(2, countNotImplicitTriggers(triggers));
- assertTrue(triggers.containsKey("node_lost_trigger"));
- assertTrue(triggers.containsKey("node_added_trigger"));
- Map<String, Object> nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- assertEquals("false", nodeLostTrigger.get("enabled").toString());
- Map<String, Object> nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
- assertEquals(4, nodeAddedTrigger.size());
- assertEquals("true", nodeAddedTrigger.get("enabled").toString());
-
- suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : '" + Policy.EACH + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- List<String> changed = (List<String>) response.get("changed");
- assertEquals(1, changed.size());
- assertTrue(changed.contains("node_added_trigger"));
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(2, countNotImplicitTriggers(triggers));
- nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- assertEquals("false", nodeLostTrigger.get("enabled").toString());
- nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
- assertEquals(4, nodeAddedTrigger.size());
- assertEquals("false", nodeAddedTrigger.get("enabled").toString());
-
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'node_added_trigger'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- changed = (List<String>) response.get("changed");
- assertEquals(1, changed.size());
- assertTrue(changed.contains("node_added_trigger"));
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(2, countNotImplicitTriggers(triggers));
- nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- assertEquals("false", nodeLostTrigger.get("enabled").toString());
- nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
- assertEquals(4, nodeAddedTrigger.size());
- assertEquals("true", nodeAddedTrigger.get("enabled").toString());
-
- resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : '" + Policy.EACH + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- changed = (List<String>) response.get("changed");
- assertEquals(1, changed.size());
- assertTrue(changed.contains("node_lost_trigger"));
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(2, countNotImplicitTriggers(triggers));
- nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- assertEquals("true", nodeLostTrigger.get("enabled").toString());
- nodeAddedTrigger = (Map<String, Object>) triggers.get("node_added_trigger");
- assertEquals(4, nodeAddedTrigger.size());
- assertEquals("true", nodeAddedTrigger.get("enabled").toString());
-
- suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'timeout' : '1h'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- changed = (List<String>) response.get("changed");
- assertEquals(1, changed.size());
- assertTrue(changed.contains("node_lost_trigger"));
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(2, countNotImplicitTriggers(triggers));
- nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(5, nodeLostTrigger.size());
- assertEquals("false", nodeLostTrigger.get("enabled").toString());
- assertTrue(nodeLostTrigger.containsKey("resumeAt"));
- }
-
- @Test
- public void test() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '10m'," +
- "'enabled' : true," +
- "'actions' : [" +
- "{" +
- "'name' : 'compute_plan'," +
- "'class' : 'solr.ComputePlanAction'" +
- "}]}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
-
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- ZkNodeProps loaded = ZkNodeProps.load(data);
- Map<String, Object> triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(1, countNotImplicitTriggers(triggers));
- assertTrue(triggers.containsKey("node_lost_trigger"));
- Map<String, Object> nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- List<Map<String, String>> actions = (List<Map<String, String>>) nodeLostTrigger.get("actions");
- assertNotNull(actions);
- assertEquals(1, actions.size());
- assertEquals("600", nodeLostTrigger.get("waitFor").toString());
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '20m'," +
- "'enabled' : false" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(1, countNotImplicitTriggers(triggers));
- assertTrue(triggers.containsKey("node_lost_trigger"));
- nodeLostTrigger = (Map<String, Object>) triggers.get("node_lost_trigger");
- assertEquals(4, nodeLostTrigger.size());
- assertEquals("1200", nodeLostTrigger.get("waitFor").toString());
- assertEquals("false", nodeLostTrigger.get("enabled").toString());
- actions = (List<Map<String, String>>) nodeLostTrigger.get("actions");
- assertNotNull(actions);
- assertEquals(2, actions.size());
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'xyz'," +
- "'trigger' : 'node_lost_trigger'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED']," +
- "'beforeAction' : 'execute_plan'," +
- "'class' : 'org.apache.solr.cloud.autoscaling.HttpTriggerListener'," +
- "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- Map<String, Object> listeners = (Map<String, Object>) loaded.get("listeners");
- assertNotNull(listeners);
- assertEquals(2, listeners.size());
- assertTrue(listeners.containsKey("xyz"));
- Map<String, Object> xyzListener = (Map<String, Object>) listeners.get("xyz");
- assertEquals(6, xyzListener.size());
- assertEquals("org.apache.solr.cloud.autoscaling.HttpTriggerListener", xyzListener.get("class").toString());
-
- String removeTriggerCommand = "{" +
- "'remove-trigger' : {" +
- "'name' : 'node_lost_trigger'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand);
- try {
- solrClient.request(req);
- fail("expected exception");
- } catch (BaseHttpSolrClient.RemoteExecutionException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(e.getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("Cannot remove trigger: node_lost_trigger because it has active listeners: ["));
- }
-
- String removeListenerCommand = "{\n" +
- "\t\"remove-listener\" : {\n" +
- "\t\t\"name\" : \"xyz\"\n" +
- "\t}\n" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- listeners = (Map<String, Object>) loaded.get("listeners");
- assertNotNull(listeners);
- assertEquals(1, listeners.size());
-
- removeTriggerCommand = "{" +
- "'remove-trigger' : {" +
- "'name' : 'node_lost_trigger'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- triggers = (Map<String, Object>) loaded.get("triggers");
- assertNotNull(triggers);
- assertEquals(0, countNotImplicitTriggers(triggers));
-
- setListenerCommand = "{" +
- "'set-listener' : {" +
- "'name' : 'xyz'," +
- "'trigger' : 'node_lost_trigger'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED']," +
- "'beforeAction' : 'execute_plan'," +
- "'class' : 'org.apache.solr.cloud.autoscaling.AutoScaling$HttpTriggerListener'," +
- "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- try {
- solrClient.request(req);
- fail("should have thrown Exception");
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("A trigger with the name node_lost_trigger does not exist"));
- }
- }
-
- @Test
- public void testErrorHandling() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- try {
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- solrClient.request(req);
- fail("expect exception");
- } catch (BaseHttpSolrClient.RemoteExecutionException e) {
- String message = String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"));
- assertTrue(message.contains("replica is required in"));
- }
-
- }
-
- @Test
- public void testValidation() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- // unknown trigger properties
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '10m'," +
- "'enabled' : true," +
- "'foo': 'bar'," +
- "'actions' : [" +
- "{" +
- "'name' : 'compute_plan'," +
- "'class' : 'solr.ComputePlanAction'" +
- "}]}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
-
- try {
- solrClient.request(req);
- fail("should have thrown Exception");
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property"));
- }
-
- // invalid trigger properties
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '10m'," +
- "'enabled' : true," +
- "'aboveRate': 'foo'," +
- "'actions' : [" +
- "{" +
- "'name' : 'compute_plan'," +
- "'class' : 'solr.ComputePlanAction'" +
- "}]}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
-
- try {
- solrClient.request(req);
- fail("should have thrown Exception");
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("aboveRate=Invalid configuration value: 'foo'"));
- }
-
- // unknown trigger action properties
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '10m'," +
- "'enabled' : true," +
- "'actions' : [" +
- "{" +
- "'name' : 'compute_plan'," +
- "'foo' : 'bar'," +
- "'class' : 'solr.ComputePlanAction'" +
- "}]}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
-
- try {
- solrClient.request(req);
- fail("should have thrown Exception");
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property"));
- }
-
- // unknown trigger listener properties
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '10m'," +
- "'enabled' : true," +
- "'actions' : [" +
- "{" +
- "'name' : 'compute_plan'," +
- "'class' : 'solr.ComputePlanAction'" +
- "}]}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
-
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'xyz'," +
- "'trigger' : 'node_lost_trigger'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED']," +
- "'foo' : 'bar'," +
- "'beforeAction' : 'execute_plan'," +
- "'class' : 'org.apache.solr.cloud.autoscaling.HttpTriggerListener'," +
- "'url' : 'http://xyz.com/on_node_lost?node={$LOST_NODE_NAME}'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- try {
- solrClient.request(req);
- fail("should have thrown Exception");
- } catch (BaseHttpSolrClient.RemoteSolrException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(((BaseHttpSolrClient.RemoteExecutionException) e).getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("foo=unknown property"));
- }
- }
-
- @Test
- public void testPolicyAndPreferences() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- // add multiple policies
- String setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'!overseer', 'replica':0}" +
- " ]," +
- " 'policy1':[" +
- " {'cores':'<2', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand);
- NamedList<Object> response = null;
- try {
- solrClient.request(req);
- fail("Adding a policy with 'cores' attribute should not have succeeded.");
- } catch (BaseHttpSolrClient.RemoteExecutionException e) {
- String message = e.getMetaData()._getStr("error/details[0]/errorMessages[0]",null);
-
- // expected
- assertTrue(message.contains("cores is only allowed in 'cluster-policy'"));
- }
-
- setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'!overseer', 'replica':0}" +
- " ]," +
- " 'policy1':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- byte[] data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- ZkNodeProps loaded = ZkNodeProps.load(data);
- Map<String, Object> policies = (Map<String, Object>) loaded.get("policies");
- assertNotNull(policies);
- assertNotNull(policies.get("xyz"));
- assertNotNull(policies.get("policy1"));
-
- // update default policy
- setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[" +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}" +
- " ]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- policies = (Map<String, Object>) loaded.get("policies");
- List conditions = (List) policies.get("xyz");
- assertEquals(1, conditions.size());
-
- // remove policy
- String removePolicyCommand = "{remove-policy : policy1}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- policies = (Map<String, Object>) loaded.get("policies");
- assertNull(policies.get("policy1"));
-
- // set preferences
- String setPreferencesCommand = "{" +
- " 'set-cluster-preferences': [" +
- " {'minimize': 'cores', 'precision': 3}," +
- " {'maximize': 'freedisk','precision': 100}," +
- " {'minimize': 'sysLoadAvg','precision': 10}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- List preferences = (List) loaded.get("cluster-preferences");
- assertEquals(3, preferences.size());
-
- // set preferences
- setPreferencesCommand = "{" +
- " 'set-cluster-preferences': [" +
- " {'minimize': 'sysLoadAvg','precision': 10}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- preferences = (List) loaded.get("cluster-preferences");
- assertEquals(1, preferences.size());
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'!overseer', 'replica':0}" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- List clusterPolicy = (List) loaded.get("cluster-policy");
- assertNotNull(clusterPolicy);
- assertEquals(3, clusterPolicy.size());
-
- setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'replica':0, put : on-each-node, nodeset:{'nodeRole':'overseer'} }" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- data = zkClient().getData(SOLR_AUTOSCALING_CONF_PATH, null, null);
- loaded = ZkNodeProps.load(data);
- clusterPolicy = (List) loaded.get("cluster-policy");
- assertNotNull(clusterPolicy);
- assertEquals(3, clusterPolicy.size());
- }
-
- @Test
- // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 17-Aug-2018
- @Ignore // nocommit flakey
- public void testReadApi() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- // first trigger
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger1'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '0s'," +
- "'enabled' : true" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setPreferencesCommand = "{" +
- " 'set-cluster-preferences': [" +
- " {'minimize': 'cores', 'precision': 3}," +
- " {'maximize': 'freedisk','precision': 100}," +
- " {'minimize': 'sysLoadAvg','precision': 10}," +
- " {'minimize': 'heapUsage','precision': 10}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPreferencesCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setPolicyCommand = "{'set-policy': {" +
- " 'xyz':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
- " 'policy1':[{'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}]," +
- " 'policy2':[{'replica':'<7', 'shard': '#EACH', 'node': '#ANY'}]}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
-
- Map triggers = (Map) response.get("triggers");
- assertNotNull(triggers);
- assertEquals(1, countNotImplicitTriggers(triggers));
- assertTrue(triggers.containsKey("node_added_trigger1"));
- Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1");
- assertEquals(4, node_added_trigger1.size());
- assertEquals(0L, node_added_trigger1.get("waitFor"));
- assertEquals(true, node_added_trigger1.get("enabled"));
- assertEquals(2, ((List)node_added_trigger1.get("actions")).size());
-
- List<Map> clusterPrefs = (List<Map>) response.get("cluster-preferences");
- assertNotNull(clusterPrefs);
- assertEquals(4, clusterPrefs.size());
-
- List<Map> clusterPolicy = (List<Map>) response.get("cluster-policy");
- assertNotNull(clusterPolicy);
- assertEquals(2, clusterPolicy.size());
-
- Map policies = (Map) response.get("policies");
- assertNotNull(policies);
- assertEquals(3, policies.size());
- assertNotNull(policies.get("xyz"));
- assertNotNull(policies.get("policy1"));
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null);
- response = solrClient.request(req);
-
- Map<String, Object> diagnostics = (Map<String, Object>) response.get("diagnostics");
- List sortedNodes = (List) response._get("diagnostics/sortedNodes", null);
- assertNotNull(sortedNodes);
-
- assertEquals(2, sortedNodes.size());
- for (int i = 0; i < 2; i++) {
- Map node = (Map) sortedNodes.get(i);
- assertNotNull(node);
- assertNotNull(node.get("node"));
- assertNotNull(node.get("cores"));
- assertEquals(0d, node.get("cores"));
- assertNotNull(node.get("freedisk"));
- assertNotNull(node.get("replicas"));
- assertTrue(node.get("freedisk") instanceof Double);
- assertNotNull(node.get("sysLoadAvg"));
- assertTrue(node.get("sysLoadAvg") instanceof Double);
- assertNotNull(node.get("heapUsage"));
- assertTrue(node.get("heapUsage") instanceof Double);
- }
-
- List<Map<String, Object>> violations = (List<Map<String, Object>>) diagnostics.get("violations");
- assertNotNull(violations);
- assertEquals(0, violations.size());
-
- violations = (List<Map<String, Object>>) diagnostics.get("violations");
- assertNotNull(violations);
- assertEquals(0, violations.size());
-
- // temporarily increase replica limit in cluster policy so that we can create a collection with 6 replicas
- String tempClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<4', 'shard': '#EACH', 'node': '#ANY'}"+
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, tempClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // lets create a collection which violates the rule replicas < 2
- CollectionAdminRequest.Create create = CollectionAdminRequest.Create.createCollection("readApiTestViolations", CONFIGSET_NAME, 1, 6)
- .setMaxShardsPerNode(3);
- CollectionAdminResponse adminResponse = create.process(solrClient);
- cluster.waitForActiveCollection("readApiTestViolations", 1, 6);
- assertTrue(adminResponse.isSuccess());
-
- // reset the original cluster policy
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // get the diagnostics output again
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null);
- response = solrClient.request(req);
- diagnostics = (Map<String, Object>) response.get("diagnostics");
- sortedNodes = (List) diagnostics.get("sortedNodes");
- assertNotNull(sortedNodes);
-
- violations = (List<Map<String, Object>>) diagnostics.get("violations");
- assertNotNull(violations);
- assertEquals(2, violations.size());
- for (Map<String, Object> violation : violations) {
- assertEquals("readApiTestViolations", violation.get("collection"));
- assertEquals("shard1", violation.get("shard"));
- assertEquals(1.0d, getObjectByPath(violation, true, "violation/delta"));
- assertEquals(3l, getObjectByPath(violation, true, "violation/replica/NRT"));
- assertNotNull(violation.get("clause"));
- }
- if (log.isInfoEnabled()) {
- log.info("Before starting new jetty ,{}", cluster.getJettySolrRunners()
- .stream()
- .map(jettySolrRunner -> jettySolrRunner.getNodeName()).collect(Collectors.toList()));
- }
- JettySolrRunner runner1 = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- if (log.isInfoEnabled()) {
- log.info("started new jetty {}", runner1.getNodeName());
- }
-
- response = waitForResponse(namedList -> {
- List l = (List) namedList._get("diagnostics/liveNodes",null);
- if (l != null && l.contains(runner1.getNodeName())) return true;
- return false;
- },
- AutoScalingRequest.create(SolrRequest.METHOD.GET, "/diagnostics", null),
- 200,
- 20,
- runner1.getNodeName() + " could not come up ");
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, "/suggestions", null);
- response = solrClient.request(req);
- List l = (List) response.get("suggestions");
- assertNotNull(l);
- assertEquals(2, l.size());
- for (int i = 0; i < l.size(); i++) {
- Object suggestion = l.get(i);
- assertEquals("violation", getObjectByPath(suggestion, true, "type"));
- assertEquals("POST", getObjectByPath(suggestion, true, "operation/method"));
- assertEquals("/c/readApiTestViolations", getObjectByPath(suggestion, true, "operation/path"));
- String node = (String) getObjectByPath(suggestion, true, "operation/command/move-replica/targetNode");
- assertNotNull(node);
- assertEquals(runner1.getNodeName(), node);
- }
- CollectionAdminRequest.deleteCollection("readApiTestViolations")
- .process(cluster.getSolrClient());
- }
-
- @Test
- public void testConcurrentUpdates() throws Exception {
- int COUNT = 50;
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- CountDownLatch updateLatch = new CountDownLatch(COUNT * 2);
- Runnable r = () -> {
- for (int i = 0; i < COUNT; i++) {
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger1'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '0s'," +
- "'enabled' : true" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = null;
- try {
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- } catch (Exception e) {
- fail(e.toString());
- } finally {
- updateLatch.countDown();
- }
- }
- };
- Thread t1 = new Thread(r);
- Thread t2 = new Thread(r);
- t1.start();
- t2.start();
- boolean await = updateLatch.await(60, TimeUnit.SECONDS);
- assertTrue("not all updates executed in time, remaining=" + updateLatch.getCount(), await);
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- NamedList<Object> response = solrClient.request(req);
-
- Map triggers = (Map) response.get("triggers");
- assertNotNull(triggers);
- assertEquals(1, countNotImplicitTriggers(triggers));
- assertTrue(triggers.containsKey("node_added_trigger1"));
- Map node_added_trigger1 = (Map) triggers.get("node_added_trigger1");
- assertEquals(4, node_added_trigger1.size());
- assertEquals(0L, node_added_trigger1.get("waitFor"));
- assertEquals(true, node_added_trigger1.get("enabled"));
- assertEquals(2, ((List)node_added_trigger1.get("actions")).size());
-
- }
-
- private int countNotImplicitTriggers(Map triggers) {
- if (triggers == null) return 0;
- int count = 0;
- for (Object trigger : triggers.keySet()) {
- if (!trigger.toString().startsWith(".")) count++;
- }
- return count;
- }
-
- @Test
- public void testDeleteUsedPolicy() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- // add multiple policies
- String setPolicyCommand = "{'set-policy': {" +
- " 'nodelete':[" +
- " {'nodeRole':'overseer', 'replica':0}]}}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPolicyCommand));
- CollectionAdminRequest.createCollection("COLL1", "conf", 1, 1)
- .setPolicy("nodelete")
- .process(cluster.getSolrClient());
- String removePolicyCommand = "{remove-policy : nodelete}";
- AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand);
- try {
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, removePolicyCommand));
- fail("should have failed");
- } catch (BaseHttpSolrClient.RemoteExecutionException e) {
- assertTrue(String.valueOf(getObjectByPath(e.getMetaData(), true, "error/details[0]/errorMessages[0]"))
- .contains("is being used by collection"));
- } catch (Exception e) {
- fail("Only RemoteExecutionException expected");
- }
- solrClient.request(CollectionAdminRequest.deleteCollection("COLL1"));
- }
-
- @Test
- public void testSetProperties() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"pqr\" : \"abc\"\n" +
- "\t}\n" +
- "}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- NamedList<Object> response = solrClient.request(req);
- Map properties = (Map) response.get("properties");
- assertNotNull(properties);
- assertEquals(1, properties.size());
- assertEquals("abc", properties.get("pqr"));
-
- setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"xyz\" : 123\n" +
- "\t}\n" +
- "}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
- properties = (Map) response.get("properties");
- assertNotNull(properties);
- assertEquals(2, properties.size());
- assertEquals("abc", properties.get("pqr"));
- assertEquals(123L, properties.get("xyz"));
-
- setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"xyz\" : 456\n" +
- "\t}\n" +
- "}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
- properties = (Map) response.get("properties");
- assertNotNull(properties);
- assertEquals(2, properties.size());
- assertEquals("abc", properties.get("pqr"));
- assertEquals(456L, properties.get("xyz"));
-
- setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"xyz\" : null\n" +
- "\t}\n" +
- "}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
- properties = (Map) response.get("properties");
- assertNotNull(properties);
- assertEquals(1, properties.size());
- assertEquals("abc", properties.get("pqr"));
-
- setPropertiesCommand = "{\n" +
- "\t\"set-properties\" : {\n" +
- "\t\t\"" + AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS + "\" : 5\n" +
- "\t\t\"" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "\" : 10\n" +
- "\t\t\"" + AutoScalingParams.TRIGGER_CORE_POOL_SIZE + "\" : 10\n" +
- "\t\t\"" + AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS + "\" : 5\n" +
- "\t}\n" +
- "}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
- properties = (Map) response.get("properties");
- assertNotNull(properties);
- assertEquals(5, properties.size());
- assertEquals("abc", properties.get("pqr"));
- assertEquals(5L, properties.get(AutoScalingParams.TRIGGER_SCHEDULE_DELAY_SECONDS));
- assertEquals(10L, properties.get(AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS));
- assertEquals(10L, properties.get(AutoScalingParams.TRIGGER_CORE_POOL_SIZE));
- assertEquals(5L, properties.get(AutoScalingParams.ACTION_THROTTLE_PERIOD_SECONDS));
- }
-
- public void testUpdatePolicy() throws IOException, SolrServerException {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setPropertiesCommand = "{'set-cluster-policy': [" +
- "{'cores': '<4','node': '#ANY'}]}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- NamedList<Object> response = solrClient.request(req);
- assertEquals("<4", response._get("cluster-policy[0]/cores", null));
- assertEquals("#ANY", response._get("cluster-policy[0]/node", null));
- setPropertiesCommand = "{'set-cluster-policy': [" +
- "{'cores': '<3','node': '#ANY'}]}";
- solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- req = AutoScalingRequest.create(SolrRequest.METHOD.GET, null);
- response = solrClient.request(req);
- assertEquals("<3", response._get("cluster-policy[0]/cores", null));
- assertEquals("#ANY", response._get("cluster-policy[0]/node", null));
-
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
deleted file mode 100644
index d5c3127..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/CapturedEvent.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-
-/**
- *
- */
-public class CapturedEvent {
- public final AutoScalingConfig.TriggerListenerConfig config;
- public final TriggerEventProcessorStage stage;
- public final String actionName;
- public final TriggerEvent event;
- public final String message;
- public final Map<String, Object> context = new HashMap<>();
- public final long timestamp;
-
- public CapturedEvent(long timestamp, ActionContext context, AutoScalingConfig.TriggerListenerConfig config, TriggerEventProcessorStage stage, String actionName,
- TriggerEvent event, String message) {
- if (context != null) {
- context._forEachEntry((o, o2) -> CapturedEvent.this.context.put((String) o, o2));
- TriggerEvent.fixOps("properties." + TriggerEvent.REQUESTED_OPS, this.context);
- TriggerEvent.fixOps("properties." + TriggerEvent.UNSUPPORTED_OPS, this.context);
- }
- this.config = config;
- this.stage = stage;
- this.actionName = actionName;
- this.event = event;
- this.message = message;
- this.timestamp = timestamp;
- }
-
- @Override
- public String toString() {
- return "CapturedEvent{" +
- "timestamp=" + timestamp +
- ", stage=" + stage +
- ", actionName='" + actionName + '\'' +
- ", event=" + event +
- ", context=" + context +
- ", config=" + config +
- ", message='" + message + '\'' +
- '}';
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
deleted file mode 100644
index 00ab43c..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ComputePlanActionTest.java
+++ /dev/null
@@ -1,775 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.NodeStateProvider;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.cloud.rule.ImplicitSnitch;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
-import org.junit.*;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.*;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import static java.nio.charset.StandardCharsets.UTF_8;
-import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
-
-/**
- * Test for {@link ComputePlanAction}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.cloud.Overseer=DEBUG;org.apache.solr.cloud.overseer=DEBUG;org.apache.solr.client.solrj.impl.SolrClientDataProvider=DEBUG;")
-@Ignore // nocommit this is removed in master
-public class ComputePlanActionTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final AtomicBoolean fired = new AtomicBoolean(false);
- private static final int NODE_COUNT = 1;
- private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
- private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
- private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- private static SolrCloudManager cloudManager;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(NODE_COUNT)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
- // remove everything from autoscaling.json in ZK
- zkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH, "{}".getBytes(UTF_8), true);
-
- if (cluster.getJettySolrRunners().size() > NODE_COUNT) {
- // stop some to get to original state
- int numJetties = cluster.getJettySolrRunners().size();
- for (int i = 0; i < numJetties - NODE_COUNT; i++) {
- JettySolrRunner randomJetty = cluster.getRandomJetty(random());
- List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
- for (int i1 = 0; i1 < jettySolrRunners.size(); i1++) {
- JettySolrRunner jettySolrRunner = jettySolrRunners.get(i1);
- if (jettySolrRunner == randomJetty) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i1);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
- }
- }
-
- cluster.deleteAllCollections();
-
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setClusterPreferencesCommand = "{" +
- "'set-cluster-preferences': [" +
- "{'minimize': 'cores'}," +
- "{'maximize': 'freedisk','precision': 100}]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPreferencesCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
-
- reset();
- }
-
- private void reset() {
- fired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- actionContextPropsRef.set(null);
- eventRef.set(null);
- AssertingTriggerAction.expectedNode = null;
- }
-
- private void deleteChildrenRecursively(String path) throws Exception {
- cloudManager.getDistribStateManager().removeRecursively(path, true, false);
- }
-
- @After
- public void printState() throws Exception {
- log.debug("-------------_ FINAL STATE --------------");
- SolrCloudManager cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- for (String node: cloudManager.getClusterStateProvider().getLiveNodes()) {
- Map<String, Object> values = cloudManager.getNodeStateProvider().getNodeValues(node, ImplicitSnitch.tags);
- if (log.isDebugEnabled()) {
- log.debug("* Node values: {}\n{}", node, Utils.toJSONString(values));
- }
- }
- if (log.isDebugEnabled()) {
- log.debug("* Live nodes: {}", cloudManager.getClusterStateProvider().getLiveNodes());
- }
- ClusterState state = cloudManager.getClusterStateProvider().getClusterState();
- if (log.isDebugEnabled()) {
- state.forEachCollection(coll -> log.debug("* Collection {} state: {}", coll.getName(), coll));
- }
- }
-
- @AfterClass
- public static void cleanUpAfterClass() throws Exception {
- cloudManager = null;
- }
-
- @Test
- @LuceneTestCase.AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- public void testNodeLost() throws Exception {
- // let's start a node so that we have at least two
- JettySolrRunner runner = cluster.startJettySolrRunner();
- String node = runner.getNodeName();
- AssertingTriggerAction.expectedNode = node;
-
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '7s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeLost",
- "conf",1, 2);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testNodeLost", clusterShape(1, 2));
-
- ClusterState clusterState = cluster.getSolrClient().getZkStateReader().getClusterState();
- DocCollection collection = clusterState.getCollection("testNodeLost");
- List<Replica> replicas = collection.getReplicas(node);
- assertNotNull(replicas);
- assertFalse(replicas.isEmpty());
-
- // start another node because because when the other node goes away, the cluster policy requires only
- // 1 replica per node and none on the overseer
- JettySolrRunner node2 = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- assertTrue(node2.getNodeName() + "is not live yet", cluster.getSolrClient().getZkStateReader().getClusterState().liveNodesContain(node2.getNodeName()) );
-
- // stop the original node
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i);
- if (jettySolrRunner == runner) {
- cluster.stopJettySolrRunner(i);
- break;
- }
- }
- log.info("Stopped_node : {}", node);
- cluster.waitForAllNodes(10);
-
- assertTrue("Trigger was not fired even after 10 seconds", triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
- List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null , "+ getNodeStateProviderState() + eventRef.get(), operations);
- assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size());
- SolrRequest solrRequest = operations.get(0);
- SolrParams params = solrRequest.getParams();
- assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
- String replicaToBeMoved = params.get("replica");
- assertEquals("Unexpected node in computed operation", replicas.get(0).getName(), replicaToBeMoved);
-
- // shutdown the extra node that we had started
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i);
- if (jettySolrRunner == node2) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
- }
- static String getNodeStateProviderState() {
- String result = "SolrClientNodeStateProvider.DEBUG";
- if(SolrClientNodeStateProvider.INST != null) {
- result+= Utils.toJSONString(SolrClientNodeStateProvider.INST);
- }
- return result;
-
- }
-
- // commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
- public void testNodeWithMultipleReplicasLost() throws Exception {
- // start 3 more nodes
- cluster.startJettySolrRunner();
- cluster.startJettySolrRunner();
- cluster.startJettySolrRunner();
-
- cluster.waitForAllNodes(30);
-
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeWithMultipleReplicasLost",
- "conf", 2, 3);
- create.setMaxShardsPerNode(2);
- create.process(solrClient);
-
- cluster.waitForActiveCollection("testNodeWithMultipleReplicasLost", 2, 6);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testNodeWithMultipleReplicasLost", clusterShape(2, 6));
-
- ClusterState clusterState = cluster.getSolrClient().getZkStateReader().getClusterState();
- DocCollection docCollection = clusterState.getCollection("testNodeWithMultipleReplicasLost");
-
- // lets find a node with at least 2 replicas
- String stoppedNodeName = null;
- List<Replica> replicasToBeMoved = null;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jettySolrRunner = cluster.getJettySolrRunners().get(i);
- List<Replica> replicas = docCollection.getReplicas(jettySolrRunner.getNodeName());
- if (replicas != null && replicas.size() == 2) {
- stoppedNodeName = jettySolrRunner.getNodeName();
- replicasToBeMoved = replicas;
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
- assertNotNull(stoppedNodeName);
-
- assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertTrue(fired.get());
-
- TriggerEvent triggerEvent = eventRef.get();
- assertNotNull(triggerEvent);
- assertEquals(TriggerEventType.NODELOST, triggerEvent.getEventType());
- // TODO assertEquals(stoppedNodeName, triggerEvent.getProperty(TriggerEvent.NODE_NAME));
-
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
- List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null "+ getNodeStateProviderState() + actionContextPropsRef.get(), operations);
- if (log.isInfoEnabled()) {
- operations.forEach(solrRequest -> log.info(solrRequest.getParams().toString()));
- }
- assertEquals("ComputePlanAction should have computed exactly 2 operation", 2, operations.size());
-
- for (SolrRequest solrRequest : operations) {
- SolrParams params = solrRequest.getParams();
- assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
- String moved = params.get("replica");
- assertTrue(replicasToBeMoved.stream().anyMatch(replica -> replica.getName().equals(moved)));
- }
- }
-
- @Test
- // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- public void testNodeAdded() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // the default policy limits 1 replica per node, we need more right now
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<3', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testNodeAdded",
- "conf",1, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testNodeAdded", (liveNodes, collectionState) -> collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
-
- // reset to the original policy which has only 1 replica per shard per node
- setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // start a node so that the 'violation' created by the previous policy update is fixed
- JettySolrRunner runner = cluster.startJettySolrRunner();
- assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
- List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null" + getNodeStateProviderState() + context, operations);
- assertEquals("ComputePlanAction should have computed exactly 1 operation", 1, operations.size());
- SolrRequest request = operations.get(0);
- SolrParams params = request.getParams();
- assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
- String nodeAdded = params.get("targetNode");
- assertEquals("Unexpected node in computed operation", runner.getNodeName(), nodeAdded);
- }
-
- public static class AssertingTriggerAction implements TriggerAction {
- static volatile String expectedNode;
-
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
-
- }
-
- @Override
- public void init() {
-
- }
-
- @Override
- public String getName() {
- return null;
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext context) {
- if (expectedNode != null) {
- Collection nodes = (Collection) event.getProperty(TriggerEvent.NODE_NAMES);
- if (nodes == null || !nodes.contains(expectedNode)) return;//this is not the event we are looking for
- }
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- actionContextPropsRef.set(context.getProperties());
- triggerFiredLatch.countDown();
- }
- }
-
- @Override
- public void close() throws IOException {
-
- }
- }
-
- @Test
- //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
- public void testSelectedCollectionsByName() throws Exception {
- String collectionsFilter = "'testSelected1,testSelected2'";
- testCollectionsPredicate(collectionsFilter, Collections.emptyMap());
- }
-
- @Test
- //2018-06-18 (commented) @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 09-Apr-2018
- public void testSelectedCollectionsByPolicy() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setSearchPolicyCommand = "{" +
- " 'set-policy': {" +
- " 'search': [" +
- " {'replica':'<5', 'shard': '#EACH', 'node': '#ANY'}," +
- " ]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setSearchPolicyCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String collectionsFilter = "{'policy': 'search'}";
- Map<String, String> createCollectionParameters = new HashMap<>();
- createCollectionParameters.put("testSelected1", "search");
- createCollectionParameters.put("testSelected2", "search");
- testCollectionsPredicate(collectionsFilter, createCollectionParameters);
- }
-
- private void testCollectionsPredicate(String collectionsFilter, Map<String, String> createCollectionParameters) throws Exception {
- if (log.isInfoEnabled()) {
- log.info("Found number of jetties: {}", cluster.getJettySolrRunners().size());
- }
- // start 3 more nodes
- cluster.startJettySolrRunner();
- cluster.startJettySolrRunner();
- cluster.startJettySolrRunner();
-
- cluster.waitForAllNodes(30);
-
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction', 'collections' : " + collectionsFilter + "}," +
- "{'name':'test','class':'" + ComputePlanActionTest.AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("testSelected1",
- "conf", 2, 2);
- if (createCollectionParameters.get("testSelected1") != null) {
- create.setPolicy(createCollectionParameters.get("testSelected1"));
- }
- create.process(solrClient);
-
- create = CollectionAdminRequest.createCollection("testSelected2",
- "conf", 2, 2);
- if (createCollectionParameters.get("testSelected2") != null) {
- create.setPolicy(createCollectionParameters.get("testSelected2"));
- }
- create.process(solrClient);
-
- create = CollectionAdminRequest.createCollection("testSelected3",
- "conf", 2, 2);
- if (createCollectionParameters.get("testSelected3") != null) {
- create.setPolicy(createCollectionParameters.get("testSelected3"));
- }
- create.process(solrClient);
-
- cluster.waitForActiveCollection("testSelected1", 2, 4);
- cluster.waitForActiveCollection("testSelected2", 2, 4);
- cluster.waitForActiveCollection("testSelected3", 2, 4);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testSelected1", clusterShape(2, 4));
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testSelected2", clusterShape(2, 4));
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "testSelected3", clusterShape(2, 4));
-
- // find a node that has replicas from all collections
- SolrCloudManager cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- NodeStateProvider stateProvider = cloudManager.getNodeStateProvider();
- List<String> nodes = new ArrayList<>();
- cloudManager.getClusterStateProvider().getLiveNodes().forEach(n -> {
- Map<String, Map<String, List<ReplicaInfo>>> map = stateProvider.getReplicaInfo(n, ImplicitSnitch.tags);
- if (map.containsKey("testSelected3") && map.containsKey("testSelected2") && map.containsKey("testSelected1")) {
- nodes.add(n);
- }
- });
- assertTrue(nodes.size() > 0);
- // kill first such node
- String node = nodes.get(0);
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- if (cluster.getJettySolrRunner(i).getNodeName().equals(node)) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
- assertTrue("Trigger was not fired even after 5 seconds", triggerFiredLatch.await(5, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
- List<SolrRequest> operations = (List<SolrRequest>) context.get("operations");
- assertNotNull("The operations computed by ComputePlanAction should not be null. " + getNodeStateProviderState() + context, operations);
- assertEquals("ComputePlanAction should have computed exactly 2 operations", 2, operations.size());
- SolrRequest request = operations.get(0);
- SolrParams params = request.getParams();
- assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
- assertFalse("not expected testSelected3", "testSelected3".equals(params.get("collection")));
- request = operations.get(1);
- params = request.getParams();
- assertEquals("Expected MOVEREPLICA action after adding node", MOVEREPLICA, CollectionParams.CollectionAction.get(params.get("action")));
- assertFalse("not expected testSelected3", "testSelected3".equals(params.get("collection")));
- }
-
- @Test
- public void testNodeAddedTriggerWithAddReplicaPreferredOp_1Shard() throws Exception {
- String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOp_1Shard";
- int numShards = 1;
- int numCollections = 5;
-
- nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections);
- }
-
- @Test
- public void testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard() throws Exception {
- String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOpReplicaType_1Shard";
- int numShards = 1;
- int numCollections = 5;
-
- nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(collectionNamePrefix, numShards, numCollections);
- }
-
- @Test
- // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- public void testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard() throws Exception {
- String collectionNamePrefix = "testNodeAddedTriggerWithAddReplicaPreferredOp_2Shard";
- int numShards = 2;
- int numCollections = 5;
-
- nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections);
- }
- private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections) throws Exception {
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
-
- nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand);
- }
-
- private void nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(String collectionNamePrefix, int numShards, int numCollections) throws Exception {
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," +
- "'" + AutoScalingParams.REPLICA_TYPE + "':'" + Replica.Type.PULL + "'," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
-
- nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 0, 1, 0);
- }
-
- private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand) throws Exception {
- nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 1, null, null);
- }
- private void nodeAddedTriggerWithAddReplicaPreferredOp(String collectionNamePrefix, int numShards, int numCollections, String setTriggerCommand, String setClusterPolicyCommand, Integer nNrtReplicas, Integer nTlogReplicas, Integer nPullReplicas) throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0",
- "conf", numShards, nNrtReplicas, nTlogReplicas, nPullReplicas).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- collectionNamePrefix + "_0", (liveNodes, collectionState) ->
- collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(10);
- assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map actionContext = actionContextPropsRef.get();
- List operations = (List) actionContext.get("operations");
- assertNotNull(operations);
- assertEquals(numShards, operations.size());
- Set<String> affectedShards = new HashSet<>(2);
- for (Object operation : operations) {
- assertTrue(operation instanceof CollectionAdminRequest.AddReplica);
- CollectionAdminRequest.AddReplica addReplica = (CollectionAdminRequest.AddReplica) operation;
- assertEquals(newNode.getNodeName(), addReplica.getNode());
- assertEquals(collectionNamePrefix + "_0", addReplica.getCollection());
- affectedShards.add(addReplica.getShard());
- }
- assertEquals(numShards, affectedShards.size());
-
- for (int i = 1; i < numCollections; i++) {
- create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_" + i,
- "conf", numShards, 2).setMaxShardsPerNode(numShards * 2);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- collectionNamePrefix + "_" + i, (liveNodes, collectionState) ->
- collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
- }
-
- reset();
-
- newNode = cluster.startJettySolrRunner();
- assertTrue(triggerFiredLatch.await(30, TimeUnit.SECONDS));
- assertTrue(fired.get());
- actionContext = actionContextPropsRef.get();
- operations = (List) actionContext.get("operations");
- assertNotNull(operations);
- assertEquals(numCollections * numShards, operations.size());
- Set<String> affectedCollections = new HashSet<>(numCollections);
- affectedShards = new HashSet<>(numShards);
- Set<Pair<String, String>> affectedCollShards = new HashSet<>(numCollections * numShards);
- for (Object operation : operations) {
- assertTrue(operation instanceof CollectionAdminRequest.AddReplica);
- CollectionAdminRequest.AddReplica addReplica = (CollectionAdminRequest.AddReplica) operation;
- assertEquals(newNode.getNodeName(), addReplica.getNode());
- affectedCollections.add(addReplica.getCollection());
- affectedShards.add(addReplica.getShard());
- affectedCollShards.add(new Pair<>(addReplica.getCollection(), addReplica.getShard()));
- }
- assertEquals(numCollections, affectedCollections.size());
- assertEquals(numShards, affectedShards.size());
- assertEquals(numCollections * numShards, affectedCollShards.size());
- }
-
- @Test
- // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- @Ignore // nocommit
- public void testNodeLostTriggerWithDeleteNodePreferredOp() throws Exception {
- String collectionNamePrefix = "testNodeLostTriggerWithDeleteNodePreferredOp";
- int numCollections = 1 + random().nextInt(3), numShards = 1 + random().nextInt(3);
-
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'" + AutoScalingParams.PREFERRED_OP + "':'deletenode'," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'execute_plan','class':'solr.ExecutePlanAction'}" +
- "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'nodeRole':'overseer', 'replica':0}" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- // cache the node name because it won't be available once the node is shutdown
- String newNodeName = newNode.getNodeName();
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionNamePrefix + "_0",
- "conf", numShards, 2).setMaxShardsPerNode(numShards * 2);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- collectionNamePrefix + "_0", (liveNodes, collectionState) ->
- collectionState.getReplicas().stream().allMatch(replica -> replica.isActive(liveNodes)));
-
- cluster.stopJettySolrRunner(newNode);
- cluster.waitForJettyToStop(newNode);
- assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map actionContext = actionContextPropsRef.get();
- List operations = (List) actionContext.get("operations");
- assertNotNull(operations);
- assertEquals(1, operations.size());
- for (Object operation : operations) {
- assertTrue(operation instanceof CollectionAdminRequest.DeleteNode);
- CollectionAdminRequest.DeleteNode deleteNode = (CollectionAdminRequest.DeleteNode) operation;
- SolrParams deleteNodeParams = deleteNode.getParams();
- assertEquals(newNodeName, deleteNodeParams.get("node"));
- }
-
- waitForState("Timed out waiting for all shards to have only 1 replica",
- collectionNamePrefix + "_0", clusterShape(numShards, numShards));
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
deleted file mode 100644
index 56c009c..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ExecutePlanActionTest.java
+++ /dev/null
@@ -1,376 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.stream.Collectors;
-
-import com.google.common.collect.Lists;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.BaseCloudSolrClient;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.MiniSolrCloudCluster;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TestInjection;
-import org.apache.zookeeper.data.Stat;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-/**
- * Test for {@link ExecutePlanAction}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class ExecutePlanActionTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final int NODE_COUNT = 2;
-
- private SolrResourceLoader loader;
- private SolrCloudManager cloudManager;
-
- public static class StartAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- startedProcessing.countDown();
- }
- }
-
- private static CountDownLatch startedProcessing = new CountDownLatch(1);
-
- public static class FinishAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- finishedProcessing.countDown();
- }
- }
-
- private static CountDownLatch finishedProcessing = new CountDownLatch(1);
-
- @BeforeClass
- public static void setupCluster() throws Exception {
-
- }
-
- @Before
- public void setUp() throws Exception {
- super.setUp();
-
- configureCluster(NODE_COUNT)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- // clear any persisted auto scaling configuration
- Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
-
-
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
-
- finishedProcessing = new CountDownLatch(1);
- startedProcessing = new CountDownLatch(1);
- }
-
-
- @After
- public void tearDown() throws Exception {
- shutdownCluster();
- super.tearDown();
- TestInjection.reset();
- }
-
- @Test
- public void testExecute() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String collectionName = "testExecute";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 1, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
-
- JettySolrRunner sourceNode = cluster.getRandomJetty(random());
- String sourceNodeName = sourceNode.getNodeName();
- ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
- DocCollection docCollection = clusterState.getCollection(collectionName);
- List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
- assertNotNull(replicas);
- assertFalse(replicas.isEmpty());
-
- List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
- .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
- assertFalse(otherJetties.isEmpty());
- JettySolrRunner survivor = otherJetties.get(0);
-
- try (ExecutePlanAction action = new ExecutePlanAction()) {
- action.configure(loader, cloudManager, Collections.singletonMap("name", "execute_plan"));
-
- // used to signal if we found that ExecutePlanAction did in fact create the right znode before executing the operation
- AtomicBoolean znodeCreated = new AtomicBoolean(false);
-
- CollectionAdminRequest.AsyncCollectionAdminRequest moveReplica = new CollectionAdminRequest.MoveReplica(collectionName, replicas.get(0).getName(), survivor.getNodeName());
- CollectionAdminRequest.AsyncCollectionAdminRequest mockRequest = new CollectionAdminRequest.AsyncCollectionAdminRequest(CollectionParams.CollectionAction.OVERSEERSTATUS) {
- @Override
- public void setAsyncId(String asyncId) {
- super.setAsyncId(asyncId);
- String parentPath = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/xyz/execute_plan";
- try {
- if (zkClient().exists(parentPath)) {
- java.util.List<String> children = zkClient().getChildren(parentPath, null, true);
- if (!children.isEmpty()) {
- String child = children.get(0);
- byte[] data = zkClient().getData(parentPath + "/" + child, null, null);
- Map m = (Map) Utils.fromJSON(data);
- if (m.containsKey("requestid")) {
- znodeCreated.set(m.get("requestid").equals(asyncId));
- }
- }
- }
- } catch (Exception e) {
- throw new RuntimeException(e);
- }
-
- }
- };
- List<CollectionAdminRequest.AsyncCollectionAdminRequest> operations = Lists.asList(moveReplica, new CollectionAdminRequest.AsyncCollectionAdminRequest[]{mockRequest});
- NodeLostTrigger.NodeLostEvent nodeLostEvent = new NodeLostTrigger.NodeLostEvent
- (TriggerEventType.NODELOST, "mock_trigger_name",
- Collections.singletonList(cloudManager.getTimeSource().getTimeNs()),
- Collections.singletonList(sourceNodeName),
- CollectionParams.CollectionAction.MOVEREPLICA.toLower());
- ActionContext actionContext = new ActionContext(survivor.getCoreContainer().getZkController().getSolrCloudManager(), null,
- new HashMap<>(Collections.singletonMap("operations", operations)));
- action.process(nodeLostEvent, actionContext);
-
-// assertTrue("ExecutePlanAction should have stored the requestid in ZK before executing the request", znodeCreated.get());
- List<NamedList<Object>> responses = (List<NamedList<Object>>) actionContext.getProperty("responses");
- assertNotNull(responses);
- assertEquals(2, responses.size());
- NamedList<Object> response = responses.get(0);
- assertNull(response.get("failure"));
- assertNotNull(response.get("success"));
- }
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
- }
-
- @Test
- public void testIntegration() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'execute_plan','class':'solr.ExecutePlanAction'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String collectionName = "testIntegration";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 1, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- collectionName, clusterShape(1, 2));
-
- JettySolrRunner sourceNode = cluster.getRandomJetty(random());
- String sourceNodeName = sourceNode.getNodeName();
- ClusterState clusterState = solrClient.getZkStateReader().getClusterState();
- DocCollection docCollection = clusterState.getCollection(collectionName);
- List<Replica> replicas = docCollection.getReplicas(sourceNodeName);
- assertNotNull(replicas);
- assertFalse(replicas.isEmpty());
-
- List<JettySolrRunner> otherJetties = cluster.getJettySolrRunners().stream()
- .filter(jettySolrRunner -> jettySolrRunner != sourceNode).collect(Collectors.toList());
- assertFalse(otherJetties.isEmpty());
- JettySolrRunner survivor = otherJetties.get(0);
-
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner runner = cluster.getJettySolrRunner(i);
- if (runner == sourceNode) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- }
- }
-
- waitForState("Timed out waiting for replicas of collection to be 2 again",
- collectionName, clusterShape(1, 2));
-
- clusterState = solrClient.getZkStateReader().getClusterState();
- docCollection = clusterState.getCollection(collectionName);
- List<Replica> replicasOnSurvivor = docCollection.getReplicas(survivor.getNodeName());
- assertNotNull(replicasOnSurvivor);
- assertEquals(docCollection.toString(), 2, replicasOnSurvivor.size());
- }
-
- @Test
- public void testTaskTimeout() throws Exception {
- int DELAY = TEST_NIGHTLY ? 1000 : 100;
- boolean taskTimeoutFail = random().nextBoolean();
- TestInjection.delayInExecutePlanAction = DELAY;
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String triggerName = "node_lost_trigger2";
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : '" + triggerName + "'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'execute_plan','class':'solr.ExecutePlanAction', 'taskTimeoutSeconds' : '1','taskTimeoutFail':'" + taskTimeoutFail + "'}," +
- "{'name':'finish','class':'" + FinishAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String collectionName = "testTaskTimeout";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 1, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- collectionName, clusterShape(1, 2));
-
- JettySolrRunner sourceNode = cluster.getRandomJetty(random());
-
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner runner = cluster.getJettySolrRunner(i);
- if (runner == sourceNode) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- }
- }
-
- boolean await = finishedProcessing.await(15000, TimeUnit.MILLISECONDS);
- if (taskTimeoutFail) {
- assertFalse("finished processing event but should fail", await);
- } else {
- assertTrue("did not finish processing event in time", await);
- }
- String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName + "/execute_plan";
- assertTrue(path + " does not exist", zkClient().exists(path));
- List<String> requests = zkClient().getChildren(path, null, true);
- assertFalse("some requests should be still present", requests.isEmpty());
-
- // in either case the task will complete and move the replica as needed
- waitForState("Timed out waiting for replicas of collection to be 2 again",
- collectionName, clusterShape(1, 2));
- }
-
- @Test
- public void testTaskFail() throws Exception {
- TestInjection.failInExecutePlanAction = true;
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String triggerName = "node_lost_trigger3";
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : '" + triggerName + "'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'start', 'class' : '" + StartAction.class.getName() + "'}," +
- "{'name':'compute_plan','class':'solr.ComputePlanAction'}," +
- "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," +
- "{'name':'finish','class':'" + FinishAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String collectionName = "testTaskFail";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 1, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
-
- // don't stop the jetty that runs our SolrCloudManager
- JettySolrRunner runner = cluster.stopJettySolrRunner(1);
- cluster.waitForJettyToStop(runner);
-
- boolean await = startedProcessing.await(10, TimeUnit.SECONDS);
- assertTrue("did not start processing event in time", await);
- await = finishedProcessing.await(2, TimeUnit.SECONDS);
- assertFalse("finished processing event but should fail", await);
-
- String path = ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH + "/" + triggerName + "/execute_plan";
- assertTrue(path + " does not exist", zkClient().exists(path));
- List<String> requests = zkClient().getChildren(path, null, true);
- assertTrue("there should be no requests pending but got " + requests, requests.isEmpty());
-
- // the task never completed - we actually lost a replica
- try {
- CloudUtil.waitForState(cloudManager, collectionName, 2, TimeUnit.SECONDS, BaseCloudSolrClient.expectedShardsAndActiveReplicas(1, 2));
- fail("completed a task that should have failed");
- } catch (TimeoutException te) {
- // expected
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
deleted file mode 100644
index 7083c91..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HdfsAutoAddReplicasIntegrationTest.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.LuceneTestCase.Slow;
-import org.apache.solr.cloud.hdfs.HdfsTestUtil;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-@Slow
-@LuceneTestCase.Nightly
-public class HdfsAutoAddReplicasIntegrationTest extends AutoAddReplicasIntegrationTest {
- private static MiniDFSCluster dfsCluster;
-
- @BeforeClass
- public static void setupClass() throws Exception {
- dfsCluster = HdfsTestUtil.setupClass(createTempDir().toFile().getAbsolutePath());
- }
-
- @AfterClass
- public static void teardownClass() throws Exception {
- try {
- HdfsTestUtil.teardownClass(dfsCluster);
- } finally {
- dfsCluster = null;
- }
- }
-
- @Override
- protected String getConfigSet() {
- return "cloud-hdfs";
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java
deleted file mode 100644
index 8d8f74d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/HttpTriggerListenerTest.java
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import javax.servlet.ServletException;
-import javax.servlet.ServletInputStream;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.LogLevel;
-import org.eclipse.jetty.server.Request;
-import org.eclipse.jetty.server.Server;
-import org.eclipse.jetty.server.handler.AbstractHandler;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- *
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@SolrTestCaseJ4.SuppressSSL
-@Ignore // nocommit this is removed in master
-public class HttpTriggerListenerTest extends SolrCloudTestCase {
-
- private static CountDownLatch triggerFiredLatch;
-
- private MockService mockService;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @Before
- public void setupTest() throws Exception {
- mockService = new MockService();
- mockService.start();
- triggerFiredLatch = new CountDownLatch(1);
- }
-
- @After
- public void teardownTest() throws Exception {
- if (mockService != null) {
- mockService.close();
- }
- }
-
- @Test
- public void testHttpListenerIntegration() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '0s'," +
- "'enabled' : true," +
- "'actions' : [" +
- "{'name':'test','class':'" + TestDummyAction.class.getName() + "'}" +
- "]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'foo'," +
- "'trigger' : 'node_added_trigger'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," +
- "'beforeAction' : 'test'," +
- "'afterAction' : ['test']," +
- "'class' : '" + HttpTriggerListener.class.getName() + "'," +
- "'url' : '" + mockService.server.getURI().toString() + "/${config.name:invalid}/${config.properties.beforeAction:invalid}/${stage}'," +
- "'payload': 'actionName=${actionName}, source=${event.source}, type=${event.eventType}'," +
- "'header.X-Foo' : '${config.name:invalid}'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertEquals(mockService.requests.toString(), 0, mockService.requests.size());
-
- cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
-
- Thread.sleep(5000);
-
- assertEquals(mockService.requests.toString(), 4, mockService.requests.size());
- mockService.requests.forEach(s -> assertTrue(s.contains("Content-Type: application/json")));
- mockService.requests.forEach(s -> assertTrue(s.contains("X-Foo: foo")));
- mockService.requests.forEach(s -> assertTrue(s.contains("source=node_added_trigger")));
- mockService.requests.forEach(s -> assertTrue(s.contains("type=NODEADDED")));
-
- String request = mockService.requests.get(0);
- assertTrue(request, request.startsWith("/foo/test/STARTED"));
- assertTrue(request, request.contains("actionName=,")); // empty actionName
-
- request = mockService.requests.get(1);
- assertTrue(request, request.startsWith("/foo/test/BEFORE_ACTION"));
- assertTrue(request, request.contains("actionName=test,")); // actionName
-
- request = mockService.requests.get(2);
- assertTrue(request, request.startsWith("/foo/test/AFTER_ACTION"));
- assertTrue(request, request.contains("actionName=test,")); // actionName
-
- request = mockService.requests.get(3);
- assertTrue(request, request.startsWith("/foo/test/SUCCEEDED"));
- assertTrue(request, request.contains("actionName=,")); // empty actionName
- }
-
- public static class TestDummyAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) {
- triggerFiredLatch.countDown();
- }
- }
-
- private static class MockService extends Thread {
- public final List<String> requests = new ArrayList<>();
- private Server server;
-
- public void start() {
- server = new Server(new InetSocketAddress("localhost", 0));
- server.setHandler(new AbstractHandler() {
- @Override
- public void handle(String s, Request request, HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse) throws IOException, ServletException {
- StringBuilder stringBuilder = new StringBuilder();
- stringBuilder.append(httpServletRequest.getRequestURI());
- Enumeration<String> headerNames = httpServletRequest.getHeaderNames();
- while (headerNames.hasMoreElements()) {
- stringBuilder.append('\n');
- String name = headerNames.nextElement();
- stringBuilder.append(name);
- stringBuilder.append(": ");
- stringBuilder.append(httpServletRequest.getHeader(name));
- }
- stringBuilder.append("\n\n");
- ServletInputStream is = request.getInputStream();
- byte[] httpInData = new byte[request.getContentLength()];
- int len = -1;
- while ((len = is.read(httpInData)) != -1) {
- stringBuilder.append(new String(httpInData, 0, len, StandardCharsets.UTF_8));
- }
- requests.add(stringBuilder.toString());
- httpServletResponse.setStatus(HttpServletResponse.SC_OK);
- request.setHandled(true);
- }
- });
- try {
- server.start();
- for (int i = 0; i < 30; i++) {
- Thread.sleep(1000);
- if (server.isRunning()) {
- break;
- }
- if (server.isFailed()) {
- throw new Exception("MockService startup failed - the test will fail...");
- }
- }
- } catch (Exception e) {
- throw new RuntimeException("Exception starting MockService", e);
- }
- }
-
- void close() throws Exception {
- if (server != null) {
- server.stop();
- }
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java
deleted file mode 100644
index 43169fd..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerMixedBoundsTest.java
+++ /dev/null
@@ -1,374 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TestUtil;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.request.UpdateRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.UpdateParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@LuceneTestCase.Slow
-@Ignore // nocommit this is removed in master
-public class IndexSizeTriggerMixedBoundsTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static SolrCloudManager cloudManager;
- private static SolrClient solrClient;
-
- private static int SPEED = 1;
-
- static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
- static volatile CountDownLatch listenerCreated;
- static volatile CountDownLatch finished;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- solrClient = cluster.getSolrClient();
- }
-
- @Before
- public void setDefaults() throws Exception {
- cluster.deleteAllCollections();
- cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
- listenerEvents.clear();
- listenerCreated = new CountDownLatch(1);
- finished = new CountDownLatch(1);
- }
-
- public static class CapturingTriggerListener extends TriggerListenerBase {
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- listenerCreated.countDown();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- CapturedEvent ev = new CapturedEvent(cloudManager.getTimeSource().getTimeNs(), context, config, stage, actionName, event, message);
- log.info("=======> {}", ev);
- lst.add(ev);
- }
- }
-
- public static class FinishedProcessingListener extends TriggerListenerBase {
-
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- finished.countDown();
- }
- }
-
- @Test
- public void testMixedBounds() throws Exception {
- String collectionName = "testMixedBounds_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
-
- for (int j = 0; j < 10; j++) {
- UpdateRequest ureq = new UpdateRequest();
- ureq.setParam("collection", collectionName);
- for (int i = 0; i < 100; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100) + "-" + j);
- doc.addField("foo", TestUtil.randomSimpleString(random(), 130, 130));
- ureq.add(doc);
- }
- solrClient.request(ureq);
- }
- solrClient.commit(collectionName);
-
- // check the actual size of shard to set the threshold
- QueryResponse rsp = solrClient.query(params(CommonParams.QT, "/admin/metrics", "group", "core"));
- NamedList<Object> nl = rsp.getResponse();
- nl = (NamedList<Object>)nl.get("metrics");
- int maxSize = 0;
- for (Iterator<Map.Entry<String, Object>> it = nl.iterator(); it.hasNext(); ) {
- Map.Entry<String, Object> e = it.next();
- NamedList<Object> metrics = (NamedList<Object>)e.getValue();
- Object o = metrics.get("INDEX.sizeInBytes");
- assertNotNull("INDEX.sizeInBytes missing: " + metrics, o);
- assertTrue("not a number", o instanceof Number);
- if (maxSize < ((Number)o).intValue()) {
- maxSize = ((Number)o).intValue();
- }
- }
- assertTrue("maxSize should be non-zero", maxSize > 0);
-
- int aboveBytes = maxSize * 2 / 3;
-
- // need to wait for recovery after splitting
- long waitForSeconds = 10 + random().nextInt(5);
-
- // the trigger is initially disabled so that we have time to add listeners
- // and have them capture all events once the trigger is enabled
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger4'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- // don't hit this limit when indexing
- "'aboveDocs' : 10000," +
- // hit this limit when deleting
- "'belowDocs' : 100," +
- // hit this limit when indexing
- "'aboveBytes' : " + aboveBytes + "," +
- // don't hit this limit when deleting
- "'belowBytes' : 10," +
- "'enabled' : false," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'capturing4'," +
- "'trigger' : 'index_size_trigger4'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : ['compute_plan','execute_plan']," +
- "'afterAction' : ['compute_plan','execute_plan']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'index_size_trigger4'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // now enable the trigger
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'index_size_trigger4'" +
- "}" +
- "}";
- log.info("-- resuming trigger");
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
- log.info("-- suspending trigger");
- // suspend the trigger to avoid generating more events
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger4'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertEquals(1, listenerEvents.size());
- List<CapturedEvent> events = listenerEvents.get("capturing4");
- assertNotNull("'capturing4' events not found", events);
- assertEquals("events: " + events, 6, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage);
-
- // collection should have 2 inactive and 4 active shards
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(6, 2, true, true));
-
- // check ops
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops", 2, ops.size());
- boolean shard1 = false;
- boolean shard2 = false;
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 1, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- if (p.second().equals("shard1")) {
- shard1 = true;
- } else if (p.second().equals("shard2")) {
- shard2 = true;
- } else {
- fail("unexpected shard name " + p.second());
- }
- }
- assertTrue("shard1 should be split", shard1);
- assertTrue("shard2 should be split", shard2);
-
- // now delete most of docs to trigger belowDocs condition
- listenerEvents.clear();
- finished = new CountDownLatch(1);
-
- // suspend the trigger first so that we can safely delete all docs
- suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger4'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals("success", response.get("result").toString());
-
- log.info("-- deleting documents");
- for (int j = 0; j < 10; j++) {
- UpdateRequest ureq = new UpdateRequest();
- ureq.setParam("collection", collectionName);
- for (int i = 0; i < 98; i++) {
- ureq.deleteById("id-" + (i * 100) + "-" + j);
- }
- solrClient.request(ureq);
- }
- // make sure the actual index size is reduced by deletions, otherwise we may still violate aboveBytes
- UpdateRequest ur = new UpdateRequest();
- ur.setParam(UpdateParams.COMMIT, "true");
- ur.setParam(UpdateParams.EXPUNGE_DELETES, "true");
- ur.setParam(UpdateParams.OPTIMIZE, "true");
- ur.setParam(UpdateParams.MAX_OPTIMIZE_SEGMENTS, "1");
- ur.setParam(UpdateParams.WAIT_SEARCHER, "true");
- ur.setParam(UpdateParams.OPEN_SEARCHER, "true");
- log.info("-- requesting optimize / expungeDeletes / commit");
- solrClient.request(ur, collectionName);
-
- // add some docs so that every shard gets an update
- // we can reduce the number of docs here but this also works
- for (int j = 0; j < 1; j++) {
- UpdateRequest ureq = new UpdateRequest();
- ureq.setParam("collection", collectionName);
- for (int i = 0; i < 98; i++) {
- ureq.add("id", "id-" + (i * 100) + "-" + j);
- }
- solrClient.request(ureq);
- }
-
- log.info("-- requesting commit");
- solrClient.commit(collectionName, true, true);
-
- // resume the trigger
- log.info("-- resuming trigger");
- // resume trigger
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
- log.info("-- suspending trigger");
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertEquals(1, listenerEvents.size());
- events = listenerEvents.get("capturing4");
- assertNotNull("'capturing4' events not found", events);
- assertEquals("events: " + events, 6, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage);
-
- // check ops
- ops = (List<TriggerEvent.Op>) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertTrue("number of ops: " + ops, ops.size() > 0);
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 2, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- }
-
- // TODO: fix this once MERGESHARDS is supported
- List<TriggerEvent.Op> unsupportedOps = (List<TriggerEvent.Op>)events.get(2).context.get("properties.unsupportedOps");
- assertNotNull("should have unsupportedOps", unsupportedOps);
- assertEquals(unsupportedOps.toString() + "\n" + ops, ops.size(), unsupportedOps.size());
- unsupportedOps.forEach(op -> assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction()));
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java
deleted file mode 100644
index 40b7b84..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerSizeEstimationTest.java
+++ /dev/null
@@ -1,324 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.autoscaling.sim.SimUtils;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.SolrCoreMetricManager;
-import org.apache.solr.util.LogLevel;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@LuceneTestCase.Slow
-@Ignore // nocommit this is removed in master
-public class IndexSizeTriggerSizeEstimationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static SolrCloudManager cloudManager;
- private static SolrClient solrClient;
- private static TimeSource timeSource;
-
- private static int SPEED = 1;
-
- static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
- static CountDownLatch listenerCreated = new CountDownLatch(1);
- static CountDownLatch finished = new CountDownLatch(1);
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- useFactory(null);
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- solrClient = cluster.getSolrClient();
- timeSource = cloudManager.getTimeSource();
- }
-
- @After
- public void restoreDefaults() throws Exception {
- cluster.deleteAllCollections();
- cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
- cloudManager.getTimeSource().sleep(5000);
- listenerEvents.clear();
- listenerCreated = new CountDownLatch(1);
- finished = new CountDownLatch(1);
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- solrClient = null;
- cloudManager = null;
- }
-
- public static class CapturingTriggerListener extends TriggerListenerBase {
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- listenerCreated.countDown();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message);
- log.info("=======> {}", ev);
- lst.add(ev);
- }
- }
-
- public static class FinishedProcessingListener extends TriggerListenerBase {
-
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- finished.countDown();
- }
- }
-
- @Test
- public void testEstimatedIndexSize() throws Exception {
- String collectionName = "testEstimatedIndexSize_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
-
- int NUM_DOCS = 20;
- for (int i = 0; i < NUM_DOCS; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100));
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- // get the size of the leader's index
- DocCollection coll = cloudManager.getClusterStateProvider().getCollection(collectionName);
- Replica leader = coll.getSlice("shard1").getLeader();
- String replicaName = Utils.parseMetricsReplicaName(collectionName, leader.getCoreName());
- assertNotNull("replicaName could not be constructed from " + leader, replicaName);
- final String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, "shard1", replicaName, null);
- Set<String> tags = SimUtils.COMMON_REPLICA_TAGS.stream()
- .map(s -> "metrics:" + registry + ":" + s).collect(Collectors.toSet());
- Map<String, Object> sizes = cloudManager.getNodeStateProvider().getNodeValues(leader.getNodeName(), tags);
- String commitSizeTag = "metrics:" + registry + ":SEARCHER.searcher.indexCommitSize";
- String numDocsTag = "metrics:" + registry + ":SEARCHER.searcher.numDocs";
- String maxDocTag = "metrics:" + registry + ":SEARCHER.searcher.maxDoc";
- assertNotNull(sizes.toString(), sizes.get(commitSizeTag));
- assertNotNull(sizes.toString(), sizes.get(numDocsTag));
- assertNotNull(sizes.toString(), sizes.get(maxDocTag));
- long commitSize = ((Number)sizes.get(commitSizeTag)).longValue();
- long maxDoc = ((Number)sizes.get(maxDocTag)).longValue();
- long numDocs = ((Number)sizes.get(numDocsTag)).longValue();
-
- assertEquals("maxDoc != numDocs", maxDoc, numDocs);
- assertTrue("unexpected numDocs=" + numDocs, numDocs > NUM_DOCS / 3);
-
- long aboveBytes = commitSize * 9 / 10;
- long waitForSeconds = 3 + random().nextInt(5);
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger7'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'splitMethod' : 'link'," +
- "'aboveBytes' : " + aboveBytes + "," +
- "'enabled' : false," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'capturing7'," +
- "'trigger' : 'index_size_trigger7'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : ['compute_plan','execute_plan']," +
- "'afterAction' : ['compute_plan','execute_plan']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'index_size_trigger7'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // enable the trigger
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'index_size_trigger7'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals("success", response.get("result").toString());
-
- // aboveBytes was set to be slightly lower than the actual size of at least one shard, so
- // we're expecting a SPLITSHARD - but with 'link' method the actual size of the resulting shards
- // will likely not go down. However, the estimated size of the latest commit point will go down
- // (see SOLR-12941).
-
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- boolean await = finished.await(15000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
- // suspend the trigger
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger7'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals("success", response.get("result").toString());
-
- assertEquals(1, listenerEvents.size());
- List<CapturedEvent> events = listenerEvents.get("capturing7");
- assertNotNull(listenerEvents.toString(), events);
- assertFalse("empty events?", events.isEmpty());
- CapturedEvent ev = events.get(0);
- List<TriggerEvent.Op> ops = (List< TriggerEvent.Op>)ev.event.properties.get(TriggerEvent.REQUESTED_OPS);
- assertNotNull("no requested ops in " + ev, ops);
- assertFalse("empty list of ops in " + ev, ops.isEmpty());
- Set<String> parentShards = new HashSet<>();
- ops.forEach(op -> {
- assertTrue(op.toString(), op.getAction() == CollectionParams.CollectionAction.SPLITSHARD);
- Collection<Pair<String, String>> hints = (Collection<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("no hints in op " + op, hints);
- hints.forEach(h -> parentShards.add(h.second()));
- });
-
- // allow for recovery of at least some sub-shards
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- coll = cloudManager.getClusterStateProvider().getCollection(collectionName);
-
- int checkedSubShards = 0;
-
- for (String parentShard : parentShards) {
- for (String subShard : Arrays.asList(parentShard + "_0", parentShard + "_1")) {
- leader = coll.getSlice(subShard).getLeader();
- if (leader == null) {
- // no leader yet - skip it
- }
- checkedSubShards++;
- replicaName = Utils.parseMetricsReplicaName(collectionName, leader.getCoreName());
- assertNotNull("replicaName could not be constructed from " + leader, replicaName);
- final String subregistry = SolrCoreMetricManager.createRegistryName(true, collectionName, subShard, replicaName, null);
- Set<String> subtags = SimUtils.COMMON_REPLICA_TAGS.stream()
- .map(s -> "metrics:" + subregistry + ":" + s).collect(Collectors.toSet());
- sizes = cloudManager.getNodeStateProvider().getNodeValues(leader.getNodeName(), subtags);
- commitSizeTag = "metrics:" + subregistry + ":SEARCHER.searcher.indexCommitSize";
- numDocsTag = "metrics:" + subregistry + ":SEARCHER.searcher.numDocs";
- maxDocTag = "metrics:" + subregistry + ":SEARCHER.searcher.maxDoc";
- assertNotNull(sizes.toString(), sizes.get(commitSizeTag));
- assertNotNull(sizes.toString(), sizes.get(numDocsTag));
- assertNotNull(sizes.toString(), sizes.get(maxDocTag));
- long subCommitSize = ((Number)sizes.get(commitSizeTag)).longValue();
- long subMaxDoc = ((Number)sizes.get(maxDocTag)).longValue();
- long subNumDocs = ((Number)sizes.get(numDocsTag)).longValue();
- assertTrue("subNumDocs=" + subNumDocs + " should be less than subMaxDoc=" + subMaxDoc +
- " due to link split", subNumDocs < subMaxDoc);
- assertTrue("subCommitSize=" + subCommitSize + " should be still greater than aboveBytes=" + aboveBytes +
- " due to link split", subCommitSize > aboveBytes);
- // calculate estimated size using the same formula
- long estimatedSize = IndexSizeTrigger.estimatedSize(subMaxDoc, subNumDocs, subCommitSize);
- assertTrue("estimatedSize=" + estimatedSize + " should be lower than aboveBytes=" + aboveBytes,
- estimatedSize < aboveBytes);
- }
- }
-
- assertTrue("didn't find any leaders in new sub-shards", checkedSubShards > 0);
-
- // reset & resume
- listenerEvents.clear();
- finished = new CountDownLatch(1);
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals("success", response.get("result").toString());
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- // estimated shard size should fall well below the aboveBytes, even though the real commitSize
- // still remains larger due to the splitMethod=link side-effects
- await = finished.await(10000 / SPEED, TimeUnit.MILLISECONDS);
- assertFalse("should not fire the trigger again! " + listenerEvents, await);
-
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
deleted file mode 100644
index dc86d8d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ /dev/null
@@ -1,782 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-import java.lang.invoke.MethodHandles;
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.Suggester;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CommonAdminParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Pair;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.update.SolrIndexSplitter;
-import org.apache.solr.util.LogLevel;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@LuceneTestCase.Slow
-@Ignore // nocommit this is removed in master
-public class IndexSizeTriggerTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private static Field[] FIELDS = TriggerBase.class.getFields();
- private static SolrCloudManager cloudManager;
- private static SolrClient solrClient;
- private static TimeSource timeSource;
- private static SolrResourceLoader loader;
-
- private static int SPEED;
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the processor to fire on first run! event=" + event);
- return true;
- };
- private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(2);
-
- static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
- static CountDownLatch listenerCreated = new CountDownLatch(1);
- static CountDownLatch finished = new CountDownLatch(1);
- static boolean realCluster;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- realCluster = TEST_NIGHTLY ? random().nextBoolean() : false;
- if (realCluster) {
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- solrClient = cluster.getSolrClient();
- loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
- SPEED = 1;
- } else {
- SPEED = 50;
- cloudManager = SimCloudManager.createCluster(2, TimeSource.get("simTime:" + SPEED));
- // wait for defaults to be applied - due to accelerated time sometimes we may miss this
- cloudManager.getTimeSource().sleep(10000);
- AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig();
- assertFalse("autoscaling config is empty", cfg.isEmpty());
- solrClient = ((SimCloudManager)cloudManager).simGetSolrClient();
- loader = ((SimCloudManager) cloudManager).getLoader();
- }
- timeSource = cloudManager.getTimeSource();
- }
-
- @After
- public void restoreDefaults() throws Exception {
- if (!realCluster) {
- if (log.isInfoEnabled()) {
- log.info(((SimCloudManager) cloudManager).dumpClusterState(true));
- }
- ((SimCloudManager) cloudManager).getSimClusterStateProvider().simDeleteAllCollections();
- ((SimCloudManager) cloudManager).simClearSystemCollection();
- ((SimCloudManager) cloudManager).getSimClusterStateProvider().simResetLeaderThrottles();
- ((SimCloudManager) cloudManager).simRestartOverseer(null);
- cloudManager.getTimeSource().sleep(500);
- ((SimCloudManager) cloudManager).simResetOpCounts();
- } else {
- cluster.deleteAllCollections();
- }
- cloudManager.getDistribStateManager().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), -1);
- cloudManager.getTimeSource().sleep(5000);
- listenerEvents.clear();
- listenerCreated = new CountDownLatch(1);
- finished = new CountDownLatch(1);
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- if (cloudManager != null && !realCluster) {
- cloudManager.close();
- }
- solrClient = null;
- cloudManager = null;
- loader = null;
- }
-
- @Test
- public void testTrigger() throws Exception {
- String collectionName = "testTrigger_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- if (SPEED == 1) {
- cluster.waitForActiveCollection(collectionName, 2, 4);
- } else {
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
- }
-
- long waitForSeconds = 3 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
- try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger1")) {
- trigger.configure(loader, cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- for (int i = 0; i < 25; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = timeSource.getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("IndexSizeTrigger was fired more than once!");
- }
- return true;
- });
- trigger.run();
- TriggerEvent ev = eventRef.get();
- // waitFor delay - should not produce any event yet
- assertNull("waitFor not elapsed but produced an event", ev);
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- trigger.run();
- ev = eventRef.get();
- assertNotNull("should have fired an event", ev);
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) ev.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops: " + ops, 2, ops.size());
- boolean shard1 = false;
- boolean shard2 = false;
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 1, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- if (p.second().equals("shard1")) {
- shard1 = true;
- } else if (p.second().equals("shard2")) {
- shard2 = true;
- } else {
- fail("unexpected shard name " + p.second());
- }
- Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
- assertNotNull("params are null: " + op, params);
-
- // verify default split configs
- assertEquals("splitMethod: " + op, SolrIndexSplitter.SplitMethod.LINK.toLower(),
- params.get(CommonAdminParams.SPLIT_METHOD));
- assertEquals("splitByPrefix: " + op, false, params.get(CommonAdminParams.SPLIT_BY_PREFIX));
- }
- assertTrue("shard1 should be split", shard1);
- assertTrue("shard2 should be split", shard2);
- }
- }
-
- public static class CapturingTriggerListener extends TriggerListenerBase {
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- listenerCreated.countDown();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message);
- log.info("=======> {}", ev);
- lst.add(ev);
- }
- }
-
- public static class FinishedProcessingListener extends TriggerListenerBase {
-
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- finished.countDown();
- }
- }
-
- @Test
- public void testSplitIntegration() throws Exception {
- String collectionName = "testSplitIntegration_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- if (SPEED == 1) {
- cluster.waitForActiveCollection(collectionName, 2, 4);
- } else {
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
- }
-
- long waitForSeconds = 6 + random().nextInt(5);
- // add disabled trigger
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger2'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'aboveDocs' : 10," +
- "'belowDocs' : 4," +
- "'enabled' : false," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'capturing2'," +
- "'trigger' : 'index_size_trigger2'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : ['compute_plan','execute_plan']," +
- "'afterAction' : ['compute_plan','execute_plan']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'index_size_trigger2'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
-
- for (int i = 0; i < 50; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- // enable the trigger
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'index_size_trigger2'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- boolean await = finished.await(60000, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
- CloudUtil.waitForState(cloudManager, collectionName, 20, TimeUnit.SECONDS, CloudUtil.clusterShape(6, 2, true, true));
- assertEquals(1, listenerEvents.size());
- List<CapturedEvent> events = listenerEvents.get("capturing2");
- assertNotNull("'capturing2' events not found", events);
- assertEquals("events: " + events, 6, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage);
- // check ops
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops", 2, ops.size());
- boolean shard1 = false;
- boolean shard2 = false;
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 1, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- if (p.second().equals("shard1")) {
- shard1 = true;
- } else if (p.second().equals("shard2")) {
- shard2 = true;
- } else {
- fail("unexpected shard name " + p.second());
- }
- }
-
-
- if (events.size() == 6) {
- assertTrue("shard1 should be split", shard1);
- assertTrue("shard2 should be split", shard2);
- } else {
- assertTrue("shard1 or shard2 should be split", shard1 || shard2);
- }
-
- }
-
- @Test
- public void testMergeIntegration() throws Exception {
- String collectionName = "testMergeIntegration_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
-
- if (SPEED == 1) {
- cluster.waitForActiveCollection(collectionName, 2, 4);
- } else {
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
- }
-
- for (int i = 0; i < 20; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + (i * 100));
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- long waitForSeconds = 3 + random().nextInt(5);
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger3'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'aboveDocs' : 40," +
- "'belowDocs' : 4," +
- "'enabled' : false," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'capturing3'," +
- "'trigger' : 'index_size_trigger3'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : ['compute_plan','execute_plan']," +
- "'afterAction' : ['compute_plan','execute_plan']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'index_size_trigger3'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // delete some docs to trigger a merge
- for (int i = 0; i < 15; i++) {
- solrClient.deleteById(collectionName, "id-" + (i * 100));
- }
- solrClient.commit(collectionName);
-
- // enable the trigger
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'index_size_trigger3'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals("success", response.get("result").toString());
-
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
- assertEquals(1, listenerEvents.size());
- List<CapturedEvent> events = listenerEvents.get("capturing3");
- assertNotNull("'capturing3' events not found", events);
- assertEquals("events: " + events, 6, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(3).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(4).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(5).stage);
- // check ops
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) events.get(4).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertTrue("number of ops: " + ops, ops.size() > 0);
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 2, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- }
-
- // TODO: fix this once MERGESHARDS is supported
- List<TriggerEvent.Op> unsupportedOps = (List<TriggerEvent.Op>)events.get(2).context.get("properties.unsupportedOps");
- assertNotNull("should have unsupportedOps", unsupportedOps);
- assertEquals(unsupportedOps.toString() + "\n" + ops, ops.size(), unsupportedOps.size());
- unsupportedOps.forEach(op -> assertEquals(CollectionParams.CollectionAction.MERGESHARDS, op.getAction()));
- }
-
- @Test
- public void testMaxOps() throws Exception {
- String collectionName = "testMaxOps_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 5, 2).setMaxShardsPerNode(10);
- create.process(solrClient);
-
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(5, 2, false, true));
-
- long waitForSeconds = 3 + random().nextInt(5);
- // add disabled trigger
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger5'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'aboveDocs' : 10," +
- "'enabled' : false," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'capturing5'," +
- "'trigger' : 'index_size_trigger5'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : ['compute_plan']," +
- "'afterAction' : ['compute_plan']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'index_size_trigger5'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
-
- for (int i = 0; i < 200; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- // enable the trigger
- String resumeTriggerCommand = "{" +
- "'resume-trigger' : {" +
- "'name' : 'index_size_trigger5'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, resumeTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
-
- boolean await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
-
- // suspend the trigger
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger5'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertEquals(1, listenerEvents.size());
- List<CapturedEvent> events = listenerEvents.get("capturing5");
- assertNotNull("'capturing5' events not found", events);
- assertEquals("events: " + events, 4, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage);
- // check ops
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops: " + ops, 5, ops.size());
-
- listenerEvents.clear();
- finished = new CountDownLatch(1);
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'index_size_trigger5'," +
- "'event' : 'indexSize'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'aboveDocs' : 10," +
- "'maxOps' : 3," +
- "'enabled' : true," +
- "'actions' : [{'name' : 'compute_plan', 'class' : 'solr.ComputePlanAction'}]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- await = finished.await(60000 / SPEED, TimeUnit.MILLISECONDS);
- assertTrue("did not finish processing in time", await);
-
- // suspend the trigger
- suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger5'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertEquals(1, listenerEvents.size());
- events = listenerEvents.get("capturing5");
- assertNotNull("'capturing5' events not found", events);
- assertEquals("events: " + events, 4, events.size());
- assertEquals(TriggerEventProcessorStage.STARTED, events.get(0).stage);
- assertEquals(TriggerEventProcessorStage.BEFORE_ACTION, events.get(1).stage);
- assertEquals(TriggerEventProcessorStage.AFTER_ACTION, events.get(2).stage);
- assertEquals(TriggerEventProcessorStage.SUCCEEDED, events.get(3).stage);
- // check ops
- ops = (List<TriggerEvent.Op>) events.get(2).event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops: " + ops, 3, ops.size());
- }
-
- //test that split parameters can be overridden
- @Test
- public void testSplitConfig() throws Exception {
- String collectionName = "testSplitConfig_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
- CloudUtil.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
- CloudUtil.clusterShape(2, 2, false, true));
-
- long waitForSeconds = 3 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
- props.put(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower());
- props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, true);
-
- try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger6")) {
- trigger.configure(loader, cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- for (int i = 0; i < 25; i++) {
- SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
- solrClient.add(collectionName, doc);
- }
- solrClient.commit(collectionName);
-
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = timeSource.getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("IndexSizeTrigger was fired more than once!");
- }
- return true;
- });
- trigger.run();
- TriggerEvent ev = eventRef.get();
- // waitFor delay - should not produce any event yet
- assertNull("waitFor not elapsed but produced an event", ev);
- timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
- trigger.run();
- ev = eventRef.get();
- assertNotNull("should have fired an event", ev);
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) ev.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("should contain requestedOps", ops);
- assertEquals("number of ops: " + ops, 2, ops.size());
- boolean shard1 = false;
- boolean shard2 = false;
- for (TriggerEvent.Op op : ops) {
- assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
- Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
- assertNotNull("hints", hints);
- assertEquals("hints", 1, hints.size());
- Pair<String, String> p = hints.iterator().next();
- assertEquals(collectionName, p.first());
- if (p.second().equals("shard1")) {
- shard1 = true;
- } else if (p.second().equals("shard2")) {
- shard2 = true;
- } else {
- fail("unexpected shard name " + p.second());
- }
- Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
- assertNotNull("params are null: " + op, params);
-
- // verify overrides for split config
- assertEquals("splitMethod: " + op, SolrIndexSplitter.SplitMethod.REWRITE.toLower(),
- params.get(CommonAdminParams.SPLIT_METHOD));
- assertEquals("splitByPrefix: " + op, true, params.get(CommonAdminParams.SPLIT_BY_PREFIX));
- }
- assertTrue("shard1 should be split", shard1);
- assertTrue("shard2 should be split", shard2);
- }
-
- }
-
- //validates that trigger configuration will fail for invalid split configs
- @Test
- public void testInvalidSplitConfig() throws Exception {
- long waitForSeconds = 3 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
- props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, "hello");
-
- try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger7")) {
- trigger.configure(loader, cloudManager, props);
- fail("Trigger configuration should have failed with invalid property.");
- } catch (TriggerValidationException e) {
- assertTrue(e.getDetails().containsKey(IndexSizeTrigger.SPLIT_BY_PREFIX));
- }
-
- props.put(IndexSizeTrigger.SPLIT_BY_PREFIX, true);
- props.put(CommonAdminParams.SPLIT_METHOD, "hello");
- try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger8")) {
- trigger.configure(loader, cloudManager, props);
- fail("Trigger configuration should have failed with invalid property.");
- } catch (TriggerValidationException e) {
- assertTrue(e.getDetails().containsKey(IndexSizeTrigger.SPLIT_METHOD_PROP));
- }
- }
-
- // make sure all defined properties are added to valid properties (SOLR-13264)
- @Test
- public void testValidProperties() throws Exception {
-
- final Set<String> propFields = new HashSet<>();
-
- final TriggerBase trigger = new IndexSizeTrigger("index_size_trigger");
- for (final Field field : trigger.getClass().getFields()) {
- if (field.getName().endsWith("_PROP")) {
- propFields.add(field.get(trigger).toString());
- }
- }
- propFields.removeAll(trigger.getValidProperties());
-
- assertTrue("Invalid _PROP constants: "+propFields.toString(), propFields.isEmpty());
- }
-
- private Map<String, Object> createTriggerProps(long waitForSeconds) {
- Map<String, Object> props = new HashMap<>();
- props.put("event", "indexSize");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
- props.put(IndexSizeTrigger.ABOVE_DOCS_PROP, 10);
- props.put(IndexSizeTrigger.BELOW_DOCS_PROP, 2);
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
deleted file mode 100644
index d84ae3a..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerIntegrationTest.java
+++ /dev/null
@@ -1,246 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrInputDocument;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.SolrCoreMetricManager;
-import org.apache.solr.util.LogLevel;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
-
-/**
- * Integration test for {@link MetricTrigger}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class MetricTriggerIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final TimeSource timeSource = TimeSource.NANO_TIME;
-
- static final Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
- private static CountDownLatch triggerFiredLatch;
- private static int waitForSeconds = 1;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
-
- listenerEvents.clear();
- triggerFiredLatch = new CountDownLatch(1);
- }
-
- @Test
- @Ignore // nocommit debug
- // commented 4-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
- // commented out on: 24-Dec-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- public void testMetricTrigger() throws Exception {
- String collectionName = "testMetricTrigger";
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
- "conf", 2, 2).setMaxShardsPerNode(2);
- create.process(solrClient);
- solrClient.setDefaultCollection(collectionName);
-
- cluster.waitForActiveCollection(collectionName, 2, 4);
-
- DocCollection docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- String shardId = "shard1";
- Replica replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
- String coreName = replica.getCoreName();
- String replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
- long waitForSeconds = 2 + random().nextInt(5);
- String registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
- String tag = "metrics:" + registry + ":INDEX.sizeInBytes";
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'metric_trigger'," +
- "'event' : 'metric'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'metric': '" + tag + "'" +
- "'above' : 100.0," +
- "'collection': '" + collectionName + "'" +
- "'shard':'" + shardId + "'" +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name':'test','class':'" + MetricAction.class.getName() + "'}" +
- "]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListenerCommand1 = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'metric_trigger'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute', 'test']," +
- "'class' : '" + TestTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand1);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // start more nodes so that we have at least 4
- for (int i = cluster.getJettySolrRunners().size(); i < 4; i++) {
- cluster.startJettySolrRunner();
- }
- cluster.waitForAllNodes(10);
-
- List<SolrInputDocument> docs = new ArrayList<>(500);
- for (int i = 0; i < 500; i++) {
- docs.add(new SolrInputDocument("id", String.valueOf(i), "x_s", "x" + i));
- }
- solrClient.add(docs);
- solrClient.commit();
-
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
- assertEquals(listenerEvents.toString(), 4, listenerEvents.get("srt").size());
- CapturedEvent ev = listenerEvents.get("srt").get(0);
- long now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- assertEquals(collectionName, ev.event.getProperties().get("collection"));
-
- // find a new replica and create its metric name
- docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- replica = docCollection.getSlice(shardId).getReplicas().iterator().next();
- coreName = replica.getCoreName();
- replicaName = Utils.parseMetricsReplicaName(collectionName, coreName);
- registry = SolrCoreMetricManager.createRegistryName(true, collectionName, shardId, replicaName, null);
- tag = "metrics:" + registry + ":INDEX.sizeInBytes";
-
- triggerFiredLatch = new CountDownLatch(1);
- listenerEvents.clear();
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'metric_trigger'," +
- "'event' : 'metric'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'metric': '" + tag + "'," +
- "'above' : 100.0," +
- "'collection': '" + collectionName + "'," +
- "'shard':'" + shardId + "'," +
- "'preferredOperation':'addreplica'," +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name':'test','class':'" + MetricAction.class.getName() + "'}" +
- "]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- // wait for listener to capture the SUCCEEDED stage
- Thread.sleep(2000);
- assertEquals(listenerEvents.toString(), 5, listenerEvents.get("srt").size());
- ev = listenerEvents.get("srt").get(0);
- now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- assertEquals(collectionName, ev.event.getProperties().get("collection"));
- docCollection = solrClient.getZkStateReader().getClusterState().getCollection(collectionName);
- assertEquals(6, docCollection.getReplicas().size());
- }
-
- public static class MetricAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- try {
- long currentTimeNanos = context.getCloudManager().getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail(event.source + " was fired before the configured waitFor period");
- }
- triggerFiredLatch.countDown();
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
- }
-
- public static class TestTriggerListener extends TriggerListenerBase {
- private TimeSource timeSource;
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- timeSource = cloudManager.getTimeSource();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- lst.add(new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message));
-
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java
deleted file mode 100644
index 52a7642..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/MetricTriggerTest.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.ZkDistributedQueueFactory;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.CoreDescriptor;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.SolrCoreMetricManager;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-@Ignore // nocommit this is removed in master
-public class MetricTriggerTest extends SolrCloudTestCase {
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the listener to fire on first run!");
- return true;
- };
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(1)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME,
- "conf", 1, 1);
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
- cluster.waitForActiveCollection(SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME, 1, 1);
- }
-
- @Test
- public void test() throws Exception {
- CoreDescriptor coreDescriptor = cluster.getJettySolrRunner(0).getCoreContainer().getCoreDescriptors().iterator().next();
- String shardId = coreDescriptor.getCloudDescriptor().getShardId();
- String coreName = coreDescriptor.getName();
- String replicaName = Utils.parseMetricsReplicaName(SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME, coreName);
- long waitForSeconds = 2 + random().nextInt(5);
- String registry = SolrCoreMetricManager.createRegistryName(true, SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME, shardId, replicaName, null);
- String tag = "metrics:" + registry + ":ADMIN./admin/file.requests";
-
- Map<String, Object> props = createTriggerProps(waitForSeconds, tag, 1.0d, null, SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME, null, null);
-
- final List<TriggerEvent> events = new ArrayList<>();
- SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
- SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
- try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient())) {
- try (MetricTrigger metricTrigger = new MetricTrigger("metricTrigger")) {
- metricTrigger.configure(loader, cloudManager, props);
- metricTrigger.setProcessor(noFirstRunProcessor);
- metricTrigger.run();
- metricTrigger.setProcessor(event -> events.add(event));
- assertEquals(0, events.size());
- Thread.sleep(waitForSeconds * 1000 + 2000);
- metricTrigger.run();
- assertEquals(1, events.size());
- }
-
- events.clear();
- tag = "metrics:" + registry + ":ADMIN./admin/file.handlerStart";
- props = createTriggerProps(waitForSeconds, tag, null, 100.0d, SolrTestCaseJ4.DEFAULT_TEST_COLLECTION_NAME, null, null);
- try (MetricTrigger metricTrigger = new MetricTrigger("metricTrigger")) {
- metricTrigger.configure(loader, cloudManager, props);
- metricTrigger.setProcessor(noFirstRunProcessor);
- metricTrigger.run();
- metricTrigger.setProcessor(event -> events.add(event));
- assertEquals(0, events.size());
- Thread.sleep(waitForSeconds * 1000 + 2000);
- metricTrigger.run();
- assertEquals(1, events.size());
- }
- }
- }
-
- private Map<String, Object> createTriggerProps(long waitForSeconds, String metric, Double below, Double above, String collection, String shard, String node) {
- Map<String, Object> props = new HashMap<>();
- props.put("metric", metric);
- if (above != null) {
- props.put("above", above);
- }
- if (below != null) {
- props.put("below", below);
- }
- if (collection != null) {
- props.put("collection", collection);
- }
- if (shard != null) {
- props.put("shard", shard);
- }
- if (node != null) {
- props.put("node", node);
- }
- props.put("event", "metric");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
-
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
deleted file mode 100644
index b4ca413..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerIntegrationTest.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.Overseer;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.data.Stat;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-// TODO: this class shares duplicated code with NodeLostTriggerIntegrationTest ... merge?
-
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class NodeAddedTriggerIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static volatile CountDownLatch actionConstructorCalled;
- private static volatile CountDownLatch actionInitCalled;
- private static volatile CountDownLatch triggerFiredLatch;
- private static volatile int waitForSeconds = 1;
- private static volatile AtomicBoolean triggerFired;
- private static volatile Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private static volatile SolrCloudManager cloudManager;
-
- @After
- public void after() throws Exception {
- shutdownCluster();
- }
-
- @AfterClass
- public static void cleanUpAfterClass() throws Exception {
- cloudManager = null;
- }
-
- private static CountDownLatch getTriggerFiredLatch() {
- return triggerFiredLatch;
- }
-
- @Before
- public void setupTest() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- final Overseer overseer = cluster.getOpenOverseer();
- assertNotNull(overseer);
- cloudManager = overseer.getSolrCloudManager();
- assertNotNull(cloudManager);
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance");
-
- // aggressively remove all active scheduled triggers
- final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
- // TODO: is this really safe? is it possible overseer is still in process of adding some to schedule?
- scheduledTriggers.removeAll();
-
- // clear any persisted auto scaling configuration
- Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
- if (log.isInfoEnabled()) {
- log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion());
- }
-
- cluster.getSolrClient().setDefaultCollection(null);
-
- waitForSeconds = 1 + random().nextInt(3);
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
- triggerFiredLatch = new CountDownLatch(1);
- triggerFired = new AtomicBoolean(false);
- events.clear();
-
- // clear any events or markers
- // todo: consider the impact of such cleanup on regular cluster restarts
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
- }
-
- private void deleteChildrenRecursively(String path) throws Exception {
- cloudManager.getDistribStateManager().removeRecursively(path, true, false);
- }
-
- @Test
- @Ignore // nocommit debug
- public void testNodeAddedTriggerRestoreState() throws Exception {
-
- final String triggerName = "node_added_restore_trigger";
-
- // should be enough to ensure trigger doesn't fire any actions until we replace the trigger
- waitForSeconds = 500000;
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : '"+triggerName+"'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '"+waitForSeconds+"s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
- actionInitCalled.await(60, TimeUnit.SECONDS));
-
- // start a new node
- final JettySolrRunner newNode = cluster.startJettySolrRunner();
- final String nodeName = newNode.getNodeName();
-
- // poll the internal state of the trigger until it run()s at least once and updates
- // it's internal state to know the node we added is live
- //
- // (this should run roughly once a second)
- (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource()))
- .waitFor("initial trigger never ran to detect new live node", () ->
- (((Collection<String>) getTriggerState(triggerName).get("lastLiveNodes"))
- .contains(nodeName)));
-
- // since we know the nodeAdded event has been detected, we can recored the current timestamp
- // (relative to the cluster's time source) and later assert that (restored state) correctly
- // tracked that the event happened prior to "now"
- final long maxEventTimeNs = cloudManager.getTimeSource().getTimeNs();
-
- //
- // now replace the trigger with a new instance to test that the state gets copied over correctly
- //
-
- // reset the actionInitCalled counter so we can confirm the second instances is inited
- actionInitCalled = new CountDownLatch(1);
- // use a low waitTime to ensure it processes the event quickly.
- // (this updated property also ensures the set-trigger won't be treated as a No-Op)
- waitForSeconds = 0 + random().nextInt(3);
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : '"+triggerName+"'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '"+waitForSeconds+"s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
- actionInitCalled.await(60, TimeUnit.SECONDS));
-
- // the trigger actions should now (eventually) record that the node was added
- assertTrue("Second instance of our trigger never fired the action to process the event",
- triggerFiredLatch.await(30, TimeUnit.SECONDS));
-
- assertEquals("Wrong number of events recorded: " + events.toString(),
- 1, events.size());
-
- final TriggerEvent event = events.iterator().next();
- assertNotNull("null event???", event);
- assertTrue("Event should have been a nodeAdded event: " + event.getClass(),
- event instanceof NodeAddedTrigger.NodeAddedEvent);
-
- assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES));
- assertEquals("event has incorrect NODE_NAMES: " + event,
- Collections.singletonList(nodeName),
- event.getProperty(TriggerEvent.NODE_NAMES));
-
- assertTrue("event TS is too late, should be before (max) expected TS @ "
- + maxEventTimeNs + ": " + event,
- event.getEventTime() < maxEventTimeNs);
-
- assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES));
- assertEquals("event has unexpeted number of EVENT_TIMES: " + event,
- 1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size());
- assertEquals("event's TS doesn't match EVENT_TIMES: " + event,
- event.getEventTime(),
- ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next());
- }
-
- @Test
- public void testNodeAddedTrigger() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(15);
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode.getNodeName()));
-
- // reset
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
-
- // update the trigger with exactly the same data
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_trigger'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- // this should be a no-op so the action should have been created but init should not be called
- if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
- }
-
- public static class TestTriggerAction extends TriggerActionBase {
-
- public TestTriggerAction() {
- actionConstructorCalled.countDown();
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- try {
- if (triggerFired.compareAndSet(false, true)) {
- events.add(event);
- long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail(event.source + " was fired before the configured waitFor period");
- }
- getTriggerFiredLatch().countDown();
- } else {
- fail(event.source + " was fired more than once!");
- }
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
-
- @Override
- public void init() throws Exception {
- log.info("TestTriggerAction init");
- actionInitCalled.countDown();
- super.init();
- }
- }
-
- /**
- * Helper method for getting a copy of the current (internal) trigger state of a scheduled trigger.
- */
- private Map<String, Object> getTriggerState(final String name) {
- final Overseer overseer = cluster.getOpenOverseer();
- final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
- final AutoScaling.Trigger t = scheduledTriggers.getTrigger(name);
- assertNotNull(name + " is not a currently scheduled trigger", t);
- assertTrue(name + " is not a TriggerBase w/state: " + t.getClass(),
- t instanceof TriggerBase);
- return ((TriggerBase)t).deepCopyState();
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
deleted file mode 100644
index 36d5d5e..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeAddedTriggerTest.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrResourceLoader;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test for {@link NodeAddedTrigger}
- */
-@Ignore // nocommit this is removed in master
-public class NodeAddedTriggerTest extends SolrCloudTestCase {
- private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false);
- private static AtomicBoolean actionInitCalled = new AtomicBoolean(false);
- private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false);
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the processor to fire on first run! event=" + event);
- return true;
- };
-
- private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(2);
-
- @BeforeClass
- public static void setupCluster() throws Exception {
-
- }
-
- @Before
- public void beforeTest() throws Exception {
- actionConstructorCalled = new AtomicBoolean(false);
- actionInitCalled = new AtomicBoolean(false);
- actionCloseCalled = new AtomicBoolean(false);
- configureCluster(1)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @After
- public void afterTest() throws Exception {
- shutdownCluster();
- }
-
- @Test
- public void testTrigger() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- long waitForSeconds = 1 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
-
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger1")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- trigger.configure(container.getResourceLoader(), cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- JettySolrRunner newNode1 = cluster.startJettySolrRunner();
- JettySolrRunner newNode2 = cluster.startJettySolrRunner();
-
- cluster.waitForAllNodes(30);
-
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("NodeAddedTrigger was fired more than once!");
- }
- return true;
- });
- int counter = 0;
- do {
- trigger.run();
- Thread.sleep(1000);
- if (counter++ > 10) {
- fail("Newly added node was not discovered by trigger even after 10 seconds");
- }
- } while (!fired.get());
-
- TriggerEvent nodeAddedEvent = eventRef.get();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>)nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode1.getNodeName()));
- assertTrue(nodeNames.contains(newNode2.getNodeName()));
- }
-
- // clean nodeAdded markers - normally done by OverseerTriggerThread
- container.getZkController().getSolrCloudManager().getDistribStateManager()
- .removeRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH, true, false);
-
- // add a new node but remove it before the waitFor period expires
- // and assert that the trigger doesn't fire at all
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger2")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- trigger.configure(container.getResourceLoader(), cloudManager, props);
- trigger.init();
- final long waitTime = 2;
- props.put("waitFor", waitTime);
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- AtomicBoolean fired = new AtomicBoolean(false);
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("NodeAddedTrigger was fired more than once!");
- }
- return true;
- });
- trigger.run(); // first run should detect the new node
- newNode.stop(); // stop the new jetty
- int counter = 0;
- do {
- trigger.run();
- Thread.sleep(1000);
- if (counter++ > waitTime + 1) { // run it a little more than the wait time
- break;
- }
- } while (true);
-
- // ensure the event was not fired
- assertFalse(fired.get());
- }
- }
-
- public void testActionLifecycle() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- Map<String, Object> props = createTriggerProps(0);
- List<Map<String, String>> actions = (List<Map<String, String>>) props.get("actions");
- Map<String, String> action = new HashMap<>(2);
- action.put("name", "testActionInit");
- action.put("class", NodeAddedTriggerTest.AssertInitTriggerAction.class.getName());
- actions.add(action);
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) {
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- assertEquals(true, actionConstructorCalled.get());
- assertEquals(false, actionInitCalled.get());
- assertEquals(false, actionCloseCalled.get());
- trigger.init();
- assertEquals(true, actionInitCalled.get());
- assertEquals(false, actionCloseCalled.get());
- }
- assertEquals(true, actionCloseCalled.get());
- }
-
- public static class AssertInitTriggerAction implements TriggerAction {
- public AssertInitTriggerAction() {
- actionConstructorCalled.set(true);
- }
-
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
-
- }
-
- @Override
- public void init() {
- actionInitCalled.compareAndSet(false, true);
- }
-
- @Override
- public String getName() {
- return "";
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
-
- }
-
- @Override
- public void close() throws IOException {
- actionCloseCalled.compareAndSet(false, true);
- }
- }
-
- @Test
- public void testListenerAcceptance() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- Map<String, Object> props = createTriggerProps(0);
- try (NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger")) {
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run(); // starts tracking live nodes
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- AtomicInteger callCount = new AtomicInteger(0);
- AtomicBoolean fired = new AtomicBoolean(false);
-
- trigger.setProcessor(event -> {
- if (callCount.incrementAndGet() < 2) {
- return false;
- } else {
- fired.compareAndSet(false, true);
- return true;
- }
- });
-
- trigger.run(); // first run should detect the new node and fire immediately but listener isn't ready
- assertEquals(1, callCount.get());
- assertFalse(fired.get());
- trigger.run(); // second run should again fire
- assertEquals(2, callCount.get());
- assertTrue(fired.get());
- trigger.run(); // should not fire
- assertEquals(2, callCount.get());
- }
- }
-
- @Test
- public void testRestoreState() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- long waitForSeconds = 1 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
-
- // add a new node but update the trigger before the waitFor period expires
- // and assert that the new trigger still fires
- NodeAddedTrigger trigger = new NodeAddedTrigger("node_added_trigger");
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- trigger.setProcessor(null); // the processor may get called for old nodes
- trigger.run(); // this run should detect the new node
- trigger.close(); // close the old trigger
-
- try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("some_different_name")) {
- newTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- newTrigger.init();
- try {
- newTrigger.restoreState(trigger);
- fail("Trigger should only be able to restore state from an old trigger of the same name");
- } catch (AssertionError e) {
- // expected
- }
- }
-
- try (NodeAddedTrigger newTrigger = new NodeAddedTrigger("node_added_trigger")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- newTrigger.configure(container.getResourceLoader(), cloudManager, props);
- newTrigger.init();
- AtomicBoolean stop = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- newTrigger.setProcessor(event -> {
- //the processor may get called 2 times, for newly added node and initial nodes
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("NodeAddedListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- List<String> nodeNames = (List<String>) event.getProperty(NodeAddedTrigger.NodeAddedEvent.NODE_NAMES);
- if (nodeNames.contains(newNode.getNodeName())) {
- stop.set(true);
- eventRef.set(event);
- }
- return true;
- });
- newTrigger.restoreState(trigger); // restore state from the old trigger
- int counter = 0;
- do {
- newTrigger.run();
- Thread.sleep(1000);
- if (counter++ > 10) {
- fail("Newly added node was not discovered by trigger even after 10 seconds");
- }
- } while (!stop.get());
-
- // ensure the event was fired
- assertTrue(stop.get());
- TriggerEvent nodeAddedEvent = eventRef.get();
- assertNotNull(nodeAddedEvent);
- }
- }
-
- private Map<String, Object> createTriggerProps(long waitForSeconds) {
- Map<String, Object> props = new HashMap<>();
- props.put("event", "nodeLost");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
deleted file mode 100644
index 5b437a0..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerIntegrationTest.java
+++ /dev/null
@@ -1,359 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.Overseer;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.data.Stat;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-// TODO: this class shares duplicated code with NodeAddedTriggerIntegrationTest ... merge?
-
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class NodeLostTriggerIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static volatile CountDownLatch actionConstructorCalled;
- private static volatile CountDownLatch actionInitCalled;
- private static volatile CountDownLatch triggerFiredLatch;
- private static volatile int waitForSeconds = 1;
- private static volatile AtomicBoolean triggerFired;
- private static volatile Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private static volatile SolrCloudManager cloudManager;
-
- private static CountDownLatch getTriggerFiredLatch() {
- return triggerFiredLatch;
- }
-
- @Before
- public void setupTest() throws Exception {
-
- configureCluster(4)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- final Overseer overseer = cluster.getOpenOverseer();
- assertNotNull(overseer);
- cloudManager = overseer.getSolrCloudManager();
- assertNotNull(cloudManager);
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance");
-
- // aggressively remove all active scheduled triggers
- final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
- // TODO: is this really safe? is it possible overseer is still in process of adding some to schedule?
- scheduledTriggers.removeAll();
-
- // clear any persisted auto scaling configuration
- Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
- if (log.isInfoEnabled()) {
- log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion());
- }
-
- cluster.getSolrClient().setDefaultCollection(null);
-
- waitForSeconds = 1 + random().nextInt(3);
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
- triggerFiredLatch = new CountDownLatch(1);
- triggerFired = new AtomicBoolean(false);
- events.clear();
-
- // clear any events or markers
- // todo: consider the impact of such cleanup on regular cluster restarts
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
- }
-
- @After
- public void cleanUpTest() throws Exception {
- shutdownCluster();
- }
-
- @AfterClass
- public static void cleanUpAfterClass() throws Exception {
- cloudManager = null;
- }
-
- private void deleteChildrenRecursively(String path) throws Exception {
- cloudManager.getDistribStateManager().removeRecursively(path, true, false);
- }
-
- @Test
- @Ignore // nocommit debug
- public void testNodeLostTriggerRestoreState() throws Exception {
-
- final String triggerName = "node_lost_restore_trigger";
-
- // start a new node
- final JettySolrRunner newNode = cluster.startJettySolrRunner();
- final String nodeName = newNode.getNodeName();
-
- // should be enough to ensure trigger doesn't fire any actions until we replace the trigger
- waitForSeconds = 500000;
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : '"+triggerName+"'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '"+waitForSeconds+"s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
- actionInitCalled.await(60, TimeUnit.SECONDS));
-
- // poll the internal state of the trigger until it run()s at least once and updates
- // it's internal state to know the node we added is live
- //
- // (this should run roughly once a second)
- (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource()))
- .waitFor("initial trigger never ran to detect new live node", () ->
- (((Collection<String>) getTriggerState(triggerName).get("lastLiveNodes"))
- .contains(nodeName)));
-
- // kill our node
- cluster.stopJettySolrRunner(newNode);
- cluster.waitForJettyToStop(newNode);
-
- // poll the internal state of the trigger until it run()s at least once (more) and updates
- // it's internal state to know the node we killed is no longer alive
- //
- // (this should run roughly once a second of simulated time)
- (new TimeOut(30, TimeUnit.SECONDS, cloudManager.getTimeSource()))
- .waitFor("initial trigger never ran to detect lost node", () ->
- ! (((Collection<String>) getTriggerState(triggerName).get("lastLiveNodes"))
- .contains(nodeName)));
-
- // since we know the nodeLost event has been detected, we can recored the current timestamp
- // (relative to the cluster's time source) and later assert that (restored state) correctly
- // tracked that the event happened prior to "now"
- final long maxEventTimeNs = cloudManager.getTimeSource().getTimeNs();
-
- // even though our trigger has detected a lost node, the *action* we registered should not have
- // been run yet, due to the large waitFor configuration...
- assertEquals("initial trigger action should not have fired", false, triggerFired.get());
- assertEquals("initial trigger action latch should not have counted down",
- 1, triggerFiredLatch.getCount());
- assertEquals("initial trigger action should not have recorded any events: " + events.toString(),
- 0, events.size());
-
- //
- // now replace the trigger with a new instance to test that the state gets copied over correctly
- //
-
- // reset the actionInitCalled counter so we can confirm the second instances is inited
- actionInitCalled = new CountDownLatch(1);
- // use a low waitTime to ensure it processes the event quickly.
- // (this updated property also ensures the set-trigger won't be treated as a No-Op)
- waitForSeconds = 0 + random().nextInt(3);
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : '"+triggerName+"'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '"+waitForSeconds+"s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- assertTrue("Trigger was not init()ed even after await()ing an excessive amount of time",
- actionInitCalled.await(60, TimeUnit.SECONDS));
-
- // the trigger actions should now (eventually) record that the node is lost
- assertTrue("Second instance of our trigger never fired the action to process the event",
- triggerFiredLatch.await(30, TimeUnit.SECONDS));
-
- assertEquals("Wrong number of events recorded: " + events.toString(),
- 1, events.size());
-
- final TriggerEvent event = events.iterator().next();
- assertNotNull("null event???", event);
- assertTrue("Event should have been a nodeLost event: " + event.getClass(),
- event instanceof NodeLostTrigger.NodeLostEvent);
-
- assertNotNull("event is missing NODE_NAMES: " + event, event.getProperty(TriggerEvent.NODE_NAMES));
- assertEquals("event has incorrect NODE_NAMES: " + event,
- Collections.singletonList(nodeName),
- event.getProperty(TriggerEvent.NODE_NAMES));
-
- assertTrue("event TS is too late, should be before (max) expected TS @ "
- + maxEventTimeNs + ": " + event,
- event.getEventTime() < maxEventTimeNs);
-
- assertNotNull("event is missing EVENT_TIMES: " + event, event.getProperty(TriggerEvent.EVENT_TIMES));
- assertEquals("event has unexpeted number of EVENT_TIMES: " + event,
- 1, ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).size());
- assertEquals("event's TS doesn't match EVENT_TIMES: " + event,
- event.getEventTime(),
- ((Collection)event.getProperty(TriggerEvent.EVENT_TIMES)).iterator().next());
- }
-
- @Test
- @Ignore // nocommit debug
- public void testNodeLostTrigger() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int nonOverseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (!jetty.getNodeName().equals(overseerLeader)) {
- nonOverseerLeaderIndex = i;
- }
- }
- CloudTestUtils.assertAutoScalingRequest(cloudManager, setTriggerCommand);
-
- if (!actionInitCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- triggerFired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- String lostNodeName = cluster.getJettySolrRunner(nonOverseerLeaderIndex).getNodeName();
- JettySolrRunner j = cluster.stopJettySolrRunner(nonOverseerLeaderIndex);
- cluster.waitForJettyToStop(j);
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- NodeLostTrigger.NodeLostEvent nodeLostEvent = (NodeLostTrigger.NodeLostEvent) events.iterator().next();
- assertNotNull(nodeLostEvent);
- List<String> nodeNames = (List<String>) nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(lostNodeName));
-
- // reset
- actionConstructorCalled = new CountDownLatch(1);
- actionInitCalled = new CountDownLatch(1);
-
- // update the trigger with exactly the same data
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}");
-
- // this should be a no-op so the action should have been created but init should not be called
- if (!actionConstructorCalled.await(3, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- assertFalse(actionInitCalled.await(2, TimeUnit.SECONDS));
- }
-
- public static class TestTriggerAction extends TriggerActionBase {
-
- public TestTriggerAction() {
- actionConstructorCalled.countDown();
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- try {
- if (triggerFired.compareAndSet(false, true)) {
- events.add(event);
- long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail(event.source + " was fired before the configured waitFor period");
- }
- getTriggerFiredLatch().countDown();
- } else {
- fail(event.source + " was fired more than once!");
- }
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
-
- @Override
- public void init() throws Exception {
- log.info("TestTriggerAction init");
- actionInitCalled.countDown();
- super.init();
- }
- }
-
- /**
- * Helper method for getting a copy of the current (internal) trigger state of a scheduled trigger.
- */
- private Map<String, Object> getTriggerState(final String name) {
- final Overseer overseer = cluster.getOpenOverseer();
- final ScheduledTriggers scheduledTriggers = ((OverseerTriggerThread) overseer.getTriggerThread().getThread()).getScheduledTriggers();
- final AutoScaling.Trigger t = scheduledTriggers.getTrigger(name);
- assertNotNull(name + " is not a currently scheduled trigger", t);
- assertTrue(name + " is not a TriggerBase w/state: " + t.getClass(),
- t instanceof TriggerBase);
- return ((TriggerBase)t).deepCopyState();
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
deleted file mode 100644
index 203336d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeLostTriggerTest.java
+++ /dev/null
@@ -1,395 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test for {@link NodeLostTrigger}
- */
-@LuceneTestCase.Nightly // TODO speed up
-@Ignore // nocommit this is removed in master
-public class NodeLostTriggerTest extends SolrCloudTestCase {
- private static AtomicBoolean actionConstructorCalled = new AtomicBoolean(false);
- private static AtomicBoolean actionInitCalled = new AtomicBoolean(false);
- private static AtomicBoolean actionCloseCalled = new AtomicBoolean(false);
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the listener to fire on first run!");
- return true;
- };
-
- private static final long WAIT_FOR_DELTA_NANOS = TimeUnit.MILLISECONDS.toNanos(5);
-
- @After
- public void tearDownCluster() throws Exception {
- shutdownCluster();
- }
-
- @Before
- public void beforeTest() throws Exception {
- actionConstructorCalled = new AtomicBoolean(false);
- actionInitCalled = new AtomicBoolean(false);
- actionCloseCalled = new AtomicBoolean(false);
-
- configureCluster(3)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @Test
- public void testTrigger() throws Exception {
- cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
-
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- long waitForSeconds = 1 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
-
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger1")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- trigger.configure(container.getResourceLoader(), cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
- String lostNodeName1 = cluster.getJettySolrRunner(1).getNodeName();
- JettySolrRunner j = cluster.stopJettySolrRunner(1);
- cluster.waitForJettyToStop(j);
- String lostNodeName2 = cluster.getJettySolrRunner(1).getNodeName();
- j = cluster.stopJettySolrRunner(1);
- cluster.waitForJettyToStop(j);
- Thread.sleep(1000);
-
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("NodeLostListener was fired more than once!");
- }
- return true;
- });
- int counter = 0;
- do {
- trigger.run();
- Thread.sleep(1000);
- if (counter++ > 10) {
- fail("Lost node was not discovered by trigger even after 10 seconds");
- }
- } while (!fired.get());
-
- TriggerEvent nodeLostEvent = eventRef.get();
- assertNotNull(nodeLostEvent);
- List<String> nodeNames = (List<String>)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames + " doesn't contain " + lostNodeName1, nodeNames.contains(lostNodeName1));
- assertTrue(nodeNames + " doesn't contain " + lostNodeName2, nodeNames.contains(lostNodeName2));
-
- }
-
- // clean nodeLost markers - normally done by OverseerTriggerThread
- container.getZkController().getSolrCloudManager().getDistribStateManager()
- .removeRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH, true, false);
-
- // remove a node but add it back before the waitFor period expires
- // and assert that the trigger doesn't fire at all
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger2")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- trigger.configure(container.getResourceLoader(), cloudManager, props);
- final long waitTime = 2;
- props.put("waitFor", waitTime);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- JettySolrRunner lostNode = cluster.getJettySolrRunner(1);
- String lostNodeName = lostNode.getNodeName();
- lostNode.stop();
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- trigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitTime, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("NodeLostListener was fired more than once!");
- }
- return true;
- });
- trigger.run(); // first run should detect the lost node
- int counter = 0;
- do {
- if (!container.getZkController().getZkStateReader().getClusterState().getLiveNodes().contains(lostNodeName)) {
- break;
- }
- Thread.sleep(100);
- if (counter++ > 20) {
- fail("Live nodes not updated!");
- }
- } while (true);
- counter = 0;
- lostNode.start();
- do {
- trigger.run();
- Thread.sleep(1000);
- if (counter++ > waitTime + 1) { // run it a little more than the wait time
- break;
- }
- } while (true);
-
- // ensure the event was not fired
- assertFalse("event was fired: " + eventRef.get(), fired.get());
- }
- }
-
- public void testActionLifecycle() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- Map<String, Object> props = createTriggerProps(0);
- List<Map<String, String>> actions = (List<Map<String, String>>) props.get("actions");
- Map<String, String> action = new HashMap<>(2);
- action.put("name", "testActionInit");
- action.put("class", AssertInitTriggerAction.class.getName());
- actions.add(action);
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) {
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- assertEquals(true, actionConstructorCalled.get());
- assertEquals(false, actionInitCalled.get());
- assertEquals(false, actionCloseCalled.get());
- trigger.init();
- assertEquals(true, actionInitCalled.get());
- assertEquals(false, actionCloseCalled.get());
- }
- assertEquals(true, actionCloseCalled.get());
- }
-
- public static class AssertInitTriggerAction implements TriggerAction {
- public AssertInitTriggerAction() {
- actionConstructorCalled.set(true);
- }
-
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, Map<String, Object> properties) throws TriggerValidationException {
-
- }
-
- @Override
- public void init() {
- actionInitCalled.compareAndSet(false, true);
- }
-
- @Override
- public String getName() {
- return "";
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
-
- }
-
- @Override
- public void close() throws IOException {
- actionCloseCalled.compareAndSet(false, true);
- }
-
- }
-
- @Test
- //28-June-2018 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 16-Apr-2018
- public void testListenerAcceptance() throws Exception {
-
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- Map<String, Object> props = createTriggerProps(0);
-
- try (NodeLostTrigger trigger = new NodeLostTrigger("node_added_trigger")) {
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
-
- cluster.waitForAllNodes(30);
-
- trigger.run(); // starts tracking live nodes
-
- // stop the newly created node
- newNode.stop();
- cluster.waitForJettyToStop(newNode);
-
- AtomicInteger callCount = new AtomicInteger(0);
- AtomicBoolean fired = new AtomicBoolean(false);
-
- trigger.setProcessor(event -> {
- if (callCount.incrementAndGet() < 2) {
- return false;
- } else {
- fired.compareAndSet(false, true);
- return true;
- }
- });
-
- Thread.sleep(1000);
-
- trigger.run(); // first run should detect the lost node and fire immediately but listener isn't ready
-
- TimeOut timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- timeout.waitFor("Timeout waiting for callCount to hit at least 1", () -> callCount.get() >= 1);
- assertEquals(1, callCount.get());
- assertFalse(fired.get());
- trigger.run(); // second run should again fire
- timeout = new TimeOut(5, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- timeout.waitFor("Timeout waiting for callCount to hit at least 2", () -> callCount.get() >= 2);
- assertEquals(2, callCount.get());
- assertTrue(fired.get());
- trigger.run(); // should not fire
- assertEquals(2, callCount.get());
- }
- }
-
- @Test
- public void testRestoreState() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- long waitForSeconds = 1 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(waitForSeconds);
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- String lostNodeName = newNode.getNodeName();
-
- // remove a node but update the trigger before the waitFor period expires
- // and assert that the new trigger still fires
-
- NodeLostTrigger trigger = new NodeLostTrigger("node_lost_trigger");
- trigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
-
- // stop the newly created node
- List<JettySolrRunner> jettySolrRunners = cluster.getJettySolrRunners();
- for (int i = 0; i < jettySolrRunners.size(); i++) {
- JettySolrRunner jettySolrRunner = jettySolrRunners.get(i);
- if (newNode == jettySolrRunner) {
- JettySolrRunner j = cluster.stopJettySolrRunner(i);
- cluster.waitForJettyToStop(j);
- break;
- }
- }
-
- trigger.run(); // this run should detect the lost node
- trigger.close(); // close the old trigger
-
- try (NodeLostTrigger newTrigger = new NodeLostTrigger("some_different_name")) {
- newTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), props);
- newTrigger.init();
- try {
- newTrigger.restoreState(trigger);
- fail("Trigger should only be able to restore state from an old trigger of the same name");
- } catch (AssertionError e) {
- // expected
- }
- }
-
- try (NodeLostTrigger newTrigger = new NodeLostTrigger("node_lost_trigger")) {
- final SolrCloudManager cloudManager = container.getZkController().getSolrCloudManager();
- newTrigger.configure(container.getResourceLoader(), cloudManager, props);
- newTrigger.init();
- AtomicBoolean fired = new AtomicBoolean(false);
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- newTrigger.setProcessor(event -> {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- long currentTimeNanos = cloudManager.getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail("NodeLostListener was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
- }
- } else {
- fail("NodeLostListener was fired more than once!");
- }
- return true;
- });
- newTrigger.restoreState(trigger); // restore state from the old trigger
- int counter = 0;
- do {
- newTrigger.run();
- Thread.sleep(1000);
- if (counter++ > 10) {
- fail("Lost node was not discovered by trigger even after 10 seconds");
- }
- } while (!fired.get());
-
- TriggerEvent nodeLostEvent = eventRef.get();
- assertNotNull(nodeLostEvent);
- List<String> nodeNames = (List<String>)nodeLostEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(lostNodeName));
- }
- }
-
- private Map<String, Object> createTriggerProps(long waitForSeconds) {
- Map<String, Object> props = new HashMap<>();
- props.put("event", "nodeLost");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
deleted file mode 100644
index 80e9fa7..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/NodeMarkersRegistrationTest.java
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.SortedSet;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantLock;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.NoOpenOverseerFoundException;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.cloud.LiveNodesListener;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TimeOut;
-import org.apache.zookeeper.KeeperException;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_ACTIVE;
-import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_INACTIVE;
-import static org.apache.solr.cloud.autoscaling.OverseerTriggerThread.MARKER_STATE;
-
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class NodeMarkersRegistrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static volatile CountDownLatch triggerFiredLatch;
- private static volatile CountDownLatch listenerEventLatch;
- private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private volatile ZkStateReader zkStateReader;
- private static final ReentrantLock lock = new ReentrantLock();
-
- @Before
- public void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- zkStateReader = cluster.getSolrClient().getZkStateReader();
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
- }
-
- @After
- public void teardownCluster() throws Exception {
- shutdownCluster();
- }
-
- private static CountDownLatch getTriggerFiredLatch() {
- return triggerFiredLatch;
- }
-
- //@AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13376")
- @Test
- public void testNodeMarkersRegistration() throws Exception, NoOpenOverseerFoundException {
- triggerFiredLatch = new CountDownLatch(1);
- listenerEventLatch = new CountDownLatch(1);
- TestLiveNodesListener listener = registerLiveNodesListener();
-
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
- // add a nodes
- JettySolrRunner node = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.addedNodes.size());
- assertTrue(listener.addedNodes.toString(), listener.addedNodes.contains(node.getNodeName()));
- // verify that a znode doesn't exist (no trigger)
- String pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node.getNodeName();
- assertFalse("Path " + pathAdded + " was created but there are no nodeAdded triggers", zkClient().exists(pathAdded));
- listener.reset();
-
- // stop overseer
- log.info("====== KILL OVERSEER 1");
- JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex);
- cluster.waitForJettyToStop(j);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
-
- assertEquals(0, listener.addedNodes.size());
- // wait until the new overseer is up
- String newOverseerLeader;
- do {
- Thread.sleep(500);
- overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- newOverseerLeader = (String) overSeerStatus.get("leader");
- } while (newOverseerLeader == null || newOverseerLeader.equals(overseerLeader));
-
- assertEquals(1, listener.lostNodes.size());
- assertEquals(overseerLeader, listener.lostNodes.iterator().next());
-
-
- String pathLost = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + overseerLeader;
-
- TimeOut timeout = new TimeOut(15, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- AtomicBoolean markerInactive = new AtomicBoolean();
- try {
- timeout.waitFor("nodeLost marker to get inactive", () -> {
- try {
- if (!zkClient().exists(pathLost)) {
- throw new RuntimeException("marker " + pathLost + " should exist!");
- }
- Map<String, Object> markerData = Utils.getJson(zkClient(), pathLost);
- markerInactive.set(markerData.getOrDefault(MARKER_STATE, MARKER_ACTIVE).equals(MARKER_INACTIVE));
- return markerInactive.get();
- } catch (KeeperException e) {
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- return false;
- }
- });
- } catch (TimeoutException e) {
- // okay
- }
-
- // verify that the marker is inactive - the new overseer should deactivate markers once they are processed
- assertTrue("Marker " + pathLost + " still active!", markerInactive.get());
-
- listener.reset();
-
- // set up triggers
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- log.info("====== ADD TRIGGERS");
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_triggerMR'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_triggerMR'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestEventMarkerAction.class.getName() + "'}]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setListener = "{\n" +
- " \"set-listener\" : {\n" +
- " \"name\" : \"listener_node_added_triggerMR\",\n" +
- " \"trigger\" : \"node_added_triggerMR\",\n" +
- " \"stage\" : \"STARTED\",\n" +
- " \"class\" : \"" + AssertingListener.class.getName() + "\"\n" +
- " }\n" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListener);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- overseerLeader = (String) overSeerStatus.get("leader");
- overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
-
- // create another node
- log.info("====== ADD NODE 1");
- JettySolrRunner node1 = cluster.startJettySolrRunner();
- cluster.waitForNode(node1, 10);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.addedNodes.size());
- assertEquals(node1.getNodeName(), listener.addedNodes.iterator().next());
- // verify that a znode exists
- pathAdded = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node1.getNodeName();
- assertTrue("Path " + pathAdded + " wasn't created", zkClient().exists(pathAdded));
-
- listenerEventLatch.countDown(); // let the trigger thread continue
-
- assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS));
-
- // kill this node
- listener.reset();
- events.clear();
- triggerFiredLatch = new CountDownLatch(1);
-
- String node1Name = node1.getNodeName();
- cluster.stopJettySolrRunner(node1);
- cluster.waitForJettyToStop(node1);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
- assertEquals(1, listener.lostNodes.size());
- assertEquals(node1Name, listener.lostNodes.iterator().next());
- // verify that a znode exists
- String pathLost2 = ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH + "/" + node1Name;
- assertTrue("Path " + pathLost2 + " wasn't created", zkClient().exists(pathLost2));
-
- listenerEventLatch.countDown(); // let the trigger thread continue
-
- assertTrue(triggerFiredLatch.await(10, TimeUnit.SECONDS));
-
- // triggers don't remove markers
- assertTrue("Path " + pathLost2 + " should still exist", zkClient().exists(pathLost2));
-
- listener.reset();
- events.clear();
- triggerFiredLatch = new CountDownLatch(1);
- // kill overseer again
- log.info("====== KILL OVERSEER 2");
- JettySolrRunner jetty = cluster.getCurrentOverseerJetty();
- cluster.stopJettySolrRunner(jetty);
- cluster.waitForJettyToStop(jetty);
- if (!listener.onChangeLatch.await(10, TimeUnit.SECONDS)) {
- fail("onChange listener didn't execute on cluster change");
- }
-
-
- if (!triggerFiredLatch.await(20, TimeUnit.SECONDS)) {
- fail("Trigger should have fired by now");
- }
- assertEquals(1, events.size());
- TriggerEvent ev = events.iterator().next();
- List<String> nodeNames = (List<String>) ev.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(overseerLeader));
- assertEquals(TriggerEventType.NODELOST, ev.getEventType());
- }
-
- private TestLiveNodesListener registerLiveNodesListener() {
- TestLiveNodesListener listener = new TestLiveNodesListener();
- zkStateReader.registerLiveNodesListener(listener);
- return listener;
- }
-
- private static class TestLiveNodesListener implements LiveNodesListener {
- Set<String> lostNodes = ConcurrentHashMap.newKeySet();
- Set<String> addedNodes = ConcurrentHashMap.newKeySet();
- CountDownLatch onChangeLatch = new CountDownLatch(1);
-
- public void reset() {
- lostNodes.clear();
- addedNodes.clear();
- onChangeLatch = new CountDownLatch(1);
- }
-
- @Override
- public boolean onChange(SortedSet<String> oldLiveNodes, SortedSet<String> newLiveNodes) {
- onChangeLatch.countDown();
- Set<String> old = new HashSet<>(oldLiveNodes);
- old.removeAll(newLiveNodes);
- if (!old.isEmpty()) {
- lostNodes.addAll(old);
- }
- newLiveNodes.removeAll(oldLiveNodes);
- if (!newLiveNodes.isEmpty()) {
- addedNodes.addAll(newLiveNodes);
- }
- return false;
- }
- }
-
- public static class TestEventMarkerAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- boolean locked = lock.tryLock();
- if (!locked) {
- log.info("We should never have a tryLock fail because actions are never supposed to be executed concurrently");
- return;
- }
- try {
- events.add(event);
- getTriggerFiredLatch().countDown();
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- } finally {
- lock.unlock();
- }
- }
-
- @Override
- public void init() throws Exception {
- log.info("TestEventMarkerAction init");
- super.init();
- }
- }
-
- public static class AssertingListener extends TriggerListenerBase {
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- if (!Thread.currentThread().getName().startsWith("ScheduledTrigger")) {
- // for future safety
- throw new IllegalThreadStateException("AssertingListener should have been invoked by a thread from the scheduled trigger thread pool");
- }
- log.debug(" --- listener fired for event: {}, stage: {}", event, stage);
- listenerEventLatch.await();
- log.debug(" --- listener wait complete for event: {}, stage: {}", event, stage);
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
deleted file mode 100644
index 7dd4376..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/RestoreTriggerStateTest.java
+++ /dev/null
@@ -1,169 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.LogLevel;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
-
-/**
- * Integration test to ensure that triggers can restore state from ZooKeeper after overseer restart
- * so that events detected before restart are not lost.
- *
- * Added in SOLR-10515
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class RestoreTriggerStateTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static CountDownLatch actionInitCalled;
- private static CountDownLatch triggerFiredLatch;
- private static AtomicBoolean triggerFired;
- private static CountDownLatch actionConstructorCalled;
- private static Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private static int waitForSeconds = 1;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
-
- actionInitCalled = new CountDownLatch(1);
- triggerFiredLatch = new CountDownLatch(1);
- actionConstructorCalled = new CountDownLatch(1);
- triggerFired = new AtomicBoolean();
- }
-
- @Test
- public void testEventFromRestoredState() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_added_triggerEFRS'," +
- "'event' : 'nodeAdded'," +
- "'waitFor' : '10s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'test','class':'" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- if (!actionInitCalled.await(10, TimeUnit.SECONDS)) {
- fail("The TriggerAction should have been created by now");
- }
-
- NamedList<Object> overSeerStatus = cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus());
- String overseerLeader = (String) overSeerStatus.get("leader");
- int overseerLeaderIndex = 0;
- for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) {
- JettySolrRunner jetty = cluster.getJettySolrRunner(i);
- if (jetty.getNodeName().equals(overseerLeader)) {
- overseerLeaderIndex = i;
- break;
- }
- }
-
- events.clear();
-
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForNode(newNode, 10);
- boolean await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- // reset
- triggerFired.set(false);
- triggerFiredLatch = new CountDownLatch(1);
- NodeAddedTrigger.NodeAddedEvent nodeAddedEvent = (NodeAddedTrigger.NodeAddedEvent) events.iterator().next();
- assertNotNull(nodeAddedEvent);
- List<String> nodeNames = (List<String>) nodeAddedEvent.getProperty(TriggerEvent.NODE_NAMES);
- assertTrue(nodeNames.contains(newNode.getNodeName()));
- // add a second node - state of the trigger will change but it won't fire for waitFor sec.
- JettySolrRunner newNode2 = cluster.startJettySolrRunner();
- cluster.waitForNode(newNode, 10);
- // kill overseer leader
- JettySolrRunner j = cluster.stopJettySolrRunner(overseerLeaderIndex);
- cluster.waitForJettyToStop(j);
- await = triggerFiredLatch.await(20, TimeUnit.SECONDS);
- assertTrue("The trigger did not fire at all", await);
- assertTrue(triggerFired.get());
- }
-
- public static class TestTriggerAction extends TriggerActionBase {
-
- public TestTriggerAction() {
- actionConstructorCalled.countDown();
- }
-
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- try {
- if (triggerFired.compareAndSet(false, true)) {
- events.add(event);
- long currentTimeNanos = actionContext.getCloudManager().getTimeSource().getTimeNs();
- long eventTimeNanos = event.getEventTime();
- long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
- if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
- fail(event.source + " was fired before the configured waitFor period");
- }
- triggerFiredLatch.countDown();
- } else {
- fail(event.source + " was fired more than once!");
- }
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
-
- @Override
- public void init() throws Exception {
- log.info("TestTriggerAction init");
- actionInitCalled.countDown();
- super.init();
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
deleted file mode 100644
index 2052346..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledMaintenanceTriggerTest.java
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.DistribStateManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
-import org.apache.solr.common.cloud.ClusterState;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
-import org.apache.zookeeper.CreateMode;
-import org.junit.After;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- *
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class ScheduledMaintenanceTriggerTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static SolrCloudManager cloudManager;
- private static SolrClient solrClient;
- private static TimeSource timeSource;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(1)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- if (random().nextBoolean()) {
- cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
- solrClient = cluster.getSolrClient();
- } else {
- cloudManager = SimCloudManager.createCluster(1, TimeSource.get("simTime:50"));
- // wait for defaults to be applied - due to accelerated time sometimes we may miss this
- cloudManager.getTimeSource().sleep(10000);
- AutoScalingConfig cfg = cloudManager.getDistribStateManager().getAutoScalingConfig();
- assertFalse("autoscaling config is empty", cfg.isEmpty());
- solrClient = ((SimCloudManager)cloudManager).simGetSolrClient();
- }
- timeSource = cloudManager.getTimeSource();
- }
-
- @Before
- public void initTest() throws Exception {
- // disable .scheduled_maintenance
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {'name' : '.scheduled_maintenance'}" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, suspendTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- String setPropertiesCommand = "{" +
- "'set-properties' : {" +
- "'" + AutoScalingParams.TRIGGER_COOLDOWN_PERIOD_SECONDS + "': 1" +
- "}" +
- "}";
- response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, setPropertiesCommand));
- assertEquals(response.get("result").toString(), "success");
- triggerFired = new CountDownLatch(1);
- }
-
- private String addNode() throws Exception {
- if (cloudManager instanceof SimCloudManager) {
- return ((SimCloudManager) cloudManager).simAddNode();
- } else {
- return cluster.startJettySolrRunner().getNodeName();
- }
- }
-
- private void stopNode(String nodeName) throws Exception {
- if (cloudManager instanceof SimCloudManager) {
- ((SimCloudManager) cloudManager).simRemoveNode(nodeName, true);
- } else {
- for (JettySolrRunner jetty : cluster.getJettySolrRunners()) {
- if (jetty.getNodeName().equals(nodeName)) {
- cluster.stopJettySolrRunner(jetty);
- break;
- }
- }
- }
- }
-
- @After
- public void restoreDefaults() throws Exception {
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST,
- "{'set-trigger' : " + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_DSL + "}");
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- if (autoScalingConfig.getTriggerListenerConfigs().containsKey("foo")) {
- String cmd = "{" +
- "'remove-listener' : {'name' : 'foo'}" +
- "}";
- response = solrClient.request(AutoScalingRequest.create(SolrRequest.METHOD.POST, cmd));
- assertEquals(response.get("result").toString(), "success");
- }
- }
-
- @AfterClass
- public static void teardown() throws Exception {
- if (cloudManager instanceof SimCloudManager) {
- cloudManager.close();
- }
- solrClient = null;
- cloudManager = null;
- }
-
- @Test
- public void testTriggerDefaults() throws Exception {
- AutoScalingConfig autoScalingConfig = cloudManager.getDistribStateManager().getAutoScalingConfig();
- log.info("{}", autoScalingConfig);
- AutoScalingConfig.TriggerConfig triggerConfig = autoScalingConfig.getTriggerConfigs().get(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME);
- assertNotNull(triggerConfig);
- assertEquals(3, triggerConfig.actions.size());
- assertTrue(triggerConfig.actions.get(0).actionClass.endsWith(InactiveShardPlanAction.class.getSimpleName()));
- assertTrue(triggerConfig.actions.get(1).actionClass.endsWith(InactiveMarkersPlanAction.class.getSimpleName()));
- assertTrue(triggerConfig.actions.get(2).actionClass.endsWith(ExecutePlanAction.class.getSimpleName()));
- AutoScalingConfig.TriggerListenerConfig listenerConfig = autoScalingConfig.getTriggerListenerConfigs().get(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + ".system");
- assertNotNull(listenerConfig);
- assertEquals(AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME, listenerConfig.trigger);
- assertTrue(listenerConfig.listenerClass.endsWith(SystemLogListener.class.getSimpleName()));
- }
-
- static Map<String, List<CapturedEvent>> listenerEvents = new ConcurrentHashMap<>();
- static CountDownLatch listenerCreated = new CountDownLatch(1);
-
- public static class CapturingTriggerListener extends TriggerListenerBase {
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- listenerCreated.countDown();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message);
- log.info("=======> {}", ev);
- lst.add(ev);
- }
- }
-
- static CountDownLatch triggerFired;
-
- public static class TestTriggerAction extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- if (context.getProperties().containsKey("inactive_shard_plan") || context.getProperties().containsKey("inactive_markers_plan")) {
- triggerFired.countDown();
- }
- }
- }
-
- @Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 17-Mar-2018
- public void testInactiveShardCleanup() throws Exception {
- String collection1 = getClass().getSimpleName() + "_collection1";
- CollectionAdminRequest.Create create1 = CollectionAdminRequest.createCollection(collection1,
- "conf", 1, 1);
-
- create1.process(solrClient);
- CloudUtil.waitForState(cloudManager, "failed to create " + collection1, collection1,
- CloudUtil.clusterShape(1, 1));
-
- // also create a very stale lock
- Map<String, Object> lockData = new HashMap<>();
- lockData.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs() -
- TimeUnit.NANOSECONDS.convert(48, TimeUnit.HOURS)));
- String staleLockName = collection1 + "/staleShard-splitting";
- cloudManager.getDistribStateManager().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" +
- staleLockName, Utils.toJSON(lockData), CreateMode.EPHEMERAL, true);
-
- // expect two events - one for a very stale lock, one for the cleanup
- triggerFired = new CountDownLatch(2);
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'foo'," +
- "'trigger' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED','FAILED']," +
- "'beforeAction' : 'inactive_shard_plan'," +
- "'afterAction' : 'inactive_shard_plan'," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," +
- "'event' : 'scheduled'," +
- "'startTime' : 'NOW+10SECONDS'," +
- "'every' : '+2SECONDS'," + // must be longer than the cooldown period
- "'enabled' : true," +
- "'actions' : [{'name' : 'inactive_shard_plan', 'class' : 'solr.InactiveShardPlanAction', 'ttl' : '20'}," +
- "{'name' : 'execute_plan', 'class' : '" + ExecutePlanAction.class.getName() + "'}," +
- "{'name' : 'test', 'class' : '" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
-
- boolean await = listenerCreated.await(10, TimeUnit.SECONDS);
- assertTrue("listener not created in time", await);
-
- CollectionAdminRequest.SplitShard split1 = CollectionAdminRequest.splitShard(collection1)
- .setShardName("shard1");
- split1.process(solrClient);
- CloudUtil.waitForState(cloudManager, "failed to split " + collection1, collection1,
- CloudUtil.clusterShape(3, 1, true, true));
-
-
- await = triggerFired.await(90, TimeUnit.SECONDS);
- assertTrue("cleanup action didn't run", await);
-
- // cleanup should have occurred
- assertFalse("no events captured!", listenerEvents.isEmpty());
- List<CapturedEvent> events = new ArrayList<>(listenerEvents.get("foo"));
- listenerEvents.clear();
-
- assertFalse(events.isEmpty());
- CapturedEvent ce = null;
- CapturedEvent staleLock = null;
- for (CapturedEvent e : events) {
- if (e.stage != TriggerEventProcessorStage.AFTER_ACTION) {
- continue;
- }
- Map<String, Object> plan = (Map<String, Object>)e.context.get("properties.inactive_shard_plan");
- if (plan == null) {
- continue;
- }
- if (plan.containsKey("cleanup")) {
- ce = e;
- }
- // capture only the first
- if (plan.containsKey("staleLocks") && staleLock == null) {
- staleLock = e;
- }
- }
- assertNotNull("missing cleanup event: " + events, ce);
- assertNotNull("missing staleLocks event: " + events, staleLock);
-
- Map<String, Object> map = (Map<String, Object>)ce.context.get("properties.inactive_shard_plan");
- assertNotNull(map);
-
- Map<String, List<String>> inactive = (Map<String, List<String>>)map.get("inactive");
- assertEquals(1, inactive.size());
- assertNotNull(inactive.get(collection1));
- Map<String, List<String>> cleanup = (Map<String, List<String>>)map.get("cleanup");
- assertEquals(1, cleanup.size());
- assertNotNull(cleanup.get(collection1));
-
- map = (Map<String, Object>)staleLock.context.get("properties.inactive_shard_plan");
- assertNotNull(map);
- Map<String, Map<String, Object>> locks = (Map<String, Map<String, Object>>)map.get("staleLocks");
- assertNotNull(locks);
- assertTrue("missing stale lock data: " + locks + "\nevents: " + events, locks.containsKey(staleLockName));
-
- ClusterState state = cloudManager.getClusterStateProvider().getClusterState();
-
- CloudUtil.clusterShape(2, 1).matches(state.getLiveNodes(), state.getCollection(collection1));
- }
-
- public static CountDownLatch getTriggerFired() {
- return triggerFired;
- }
-
- public static class TestTriggerAction2 extends TriggerActionBase {
-
- @Override
- public void process(TriggerEvent event, ActionContext context) throws Exception {
- getTriggerFired().countDown();
- }
- }
-
-
- @Test
- public void testInactiveMarkersCleanup() throws Exception {
- triggerFired = new CountDownLatch(1);
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'trigger1'," +
- "'event' : 'nodeAdded'," +
- "'waitFor': '1s'" +
- "'enabled' : true," +
- "'actions' : [" +
- "{'name' : 'test', 'class' : '" + TestTriggerAction2.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : '" + AutoScaling.SCHEDULED_MAINTENANCE_TRIGGER_NAME + "'," +
- "'event' : 'scheduled'," +
- "'startTime' : 'NOW+20SECONDS'," +
- "'every' : '+2SECONDS'," + // must be longer than the cooldown period!!
- "'enabled' : true," +
- "'actions' : [{'name' : 'inactive_markers_plan', 'class' : 'solr.InactiveMarkersPlanAction', 'ttl' : '20'}," +
- "{'name' : 'test', 'class' : '" + TestTriggerAction.class.getName() + "'}]" +
- "}}";
-
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- cloudManager.getTimeSource().sleep(5000);
-
- triggerFired = new CountDownLatch(1);
- String node = addNode();
-
- boolean await = triggerFired.await(30, TimeUnit.SECONDS);
- assertTrue("trigger should have fired", await);
-
- triggerFired = new CountDownLatch(1);
-
- // should have a marker
- DistribStateManager stateManager = cloudManager.getDistribStateManager();
- String nodeAddedPath = ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH + "/" + node;
- assertTrue("marker for nodeAdded doesn't exist", stateManager.hasData(nodeAddedPath));
-
- // wait for the cleanup to fire
- await = triggerFired.await(90, TimeUnit.SECONDS);
- assertTrue("cleanup trigger should have fired", await);
- assertFalse("marker for nodeAdded still exists", stateManager.hasData(nodeAddedPath));
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
deleted file mode 100644
index 1f1bbb1..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerIntegrationTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.Date;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.util.LogLevel;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Integration test for {@link ScheduledTrigger}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-// 12-Jun-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 26-Mar-2018
-@Ignore // nocommit this is removed in master
-public class ScheduledTriggerIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static CountDownLatch triggerFiredLatch;
- private static final Set<TriggerEvent> events = ConcurrentHashMap.newKeySet();
- private static final AtomicReference<Map<String, Object>> actionContextPropertiesRef = new AtomicReference<>();
-
- @Before
- public void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cluster.getOpenOverseer().getSolrCloudManager(), ".scheduled_maintenance");
-
- triggerFiredLatch = new CountDownLatch(1);
- }
-
- @After
- public void afterTest() throws Exception {
- shutdownCluster();
- events.clear();
- actionContextPropertiesRef.set(null);
- }
-
- @Test
- // commented 15-Sep-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 2-Aug-2018
- // commented out on: 17-Feb-2019 @BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 14-Oct-2018
- public void testScheduledTrigger() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
-
- // this collection will place 2 cores on 1st node and 1 core on 2nd node
- String collectionName = "testScheduledTrigger";
- CollectionAdminRequest.createCollection(collectionName, 1, 3)
- .setMaxShardsPerNode(5).process(solrClient);
-
- cluster.waitForActiveCollection(collectionName, 1, 3);
-
- // create a policy which allows only 1 core per node thereby creating a violation for the above collection
- String setClusterPolicy = "{\n" +
- " \"set-cluster-policy\" : [\n" +
- " {\"cores\" : \"<2\", \"node\" : \"#EACH\"}\n" +
- " ]\n" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicy);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // start a new node which can be used to balance the cluster as per policy
- JettySolrRunner newNode = cluster.startJettySolrRunner();
- cluster.waitForAllNodes(30);
-
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'sched_trigger_integration1'," +
- "'event' : 'scheduled'," +
- "'startTime' : '" + new Date().toInstant().toString() + "'" +
- "'every' : '+3SECONDS'" +
- "'actions' : [" +
- "{'name' : 'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name' : 'execute','class':'" + ExecutePlanAction.class.getName() + "'}," +
- "{'name' : 'recorder', 'class': '" + ContextPropertiesRecorderAction.class.getName() + "'}" +
- "]}}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- assertTrue("ScheduledTrigger did not fire in time", triggerFiredLatch.await(10, TimeUnit.SECONDS));
- assertEquals(1, events.size());
- Map<String, Object> actionContextProps = actionContextPropertiesRef.get();
- assertNotNull(actionContextProps);
- TriggerEvent event = events.iterator().next();
- List<SolrRequest> operations = (List<SolrRequest>) actionContextProps.get("operations");
- assertNotNull(operations);
- assertEquals(1, operations.size());
- for (SolrRequest operation : operations) {
- SolrParams params = operation.getParams();
- assertEquals(newNode.getNodeName(), params.get("targetNode"));
- }
- }
-
- public static class ContextPropertiesRecorderAction extends TriggerActionBase {
- @Override
- public void process(TriggerEvent event, ActionContext actionContext) {
- actionContextPropertiesRef.set(actionContext.getProperties());
- try {
- events.add(event);
- triggerFiredLatch.countDown();
- } catch (Throwable t) {
- log.debug("--throwable", t);
- throw t;
- }
- }
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java
deleted file mode 100644
index 9b37d01..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/ScheduledTriggerTest.java
+++ /dev/null
@@ -1,143 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.time.Instant;
-import java.time.format.DateTimeFormatter;
-import java.time.format.DateTimeFormatterBuilder;
-import java.time.temporal.ChronoField;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.TimeZone;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicReference;
-
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.util.LogLevel;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- * Test for {@link ScheduledTrigger}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class ScheduledTriggerTest extends SolrCloudTestCase {
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the listener to fire on first run!");
- return true;
- };
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(1)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // added 20-Sep-2018
- // this does not appear to be a good way to test this
- public void testTrigger() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
-
- Map<String, Object> properties = createTriggerProperties(new Date().toInstant().toString(), TimeZone.getDefault().getID());
-
- scheduledTriggerTest(container, properties);
-
- TimeZone timeZone = TimeZone.getDefault();
- DateTimeFormatter dateTimeFormatter = new DateTimeFormatterBuilder()
- .append(DateTimeFormatter.ISO_LOCAL_DATE).appendPattern("['T'[HH[:mm[:ss]]]]") //brackets mean optional
- .parseDefaulting(ChronoField.HOUR_OF_DAY, 0)
- .parseDefaulting(ChronoField.MINUTE_OF_HOUR, 0)
- .parseDefaulting(ChronoField.SECOND_OF_MINUTE, 0)
- .toFormatter(Locale.ROOT).withZone(timeZone.toZoneId());
- properties = createTriggerProperties(dateTimeFormatter.format(Instant.now()), timeZone.getID());
- scheduledTriggerTest(container, properties);
- }
-
- @Test
- public void testIgnoredEvent() throws Exception {
- CoreContainer container = cluster.getJettySolrRunners().get(0).getCoreContainer();
- long threeDaysAgo = new Date().getTime() - TimeUnit.DAYS.toMillis(3);
- Map<String, Object> properties = createTriggerProperties(new Date(threeDaysAgo).toInstant().toString(),
- TimeZone.getDefault().getID(),
- "+2DAYS", "+1HOUR");
- try (ScheduledTrigger scheduledTrigger = new ScheduledTrigger("sched1")) {
- scheduledTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), properties);
- scheduledTrigger.init();
- AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
- scheduledTrigger.setProcessor(event -> {
- eventRef.set(event);
- return true;
- });
- scheduledTrigger.run();
- assertTrue(eventRef.get().isIgnored());
- }
- }
-
- private void scheduledTriggerTest(CoreContainer container, Map<String, Object> properties) throws Exception {
- try (ScheduledTrigger scheduledTrigger = new ScheduledTrigger("sched1")) {
- scheduledTrigger.configure(container.getResourceLoader(), container.getZkController().getSolrCloudManager(), properties);
- scheduledTrigger.init();
- scheduledTrigger.setProcessor(noFirstRunProcessor);
- scheduledTrigger.run();
- final List<Long> eventTimes = Collections.synchronizedList(new ArrayList<>());
- scheduledTrigger.setProcessor(event -> {
- eventTimes.add(event.getEventTime());
- return true;
- });
- for (int i = 0; i < 3; i++) {
- Thread.sleep(3000);
- scheduledTrigger.run();
- }
- assertEquals(3, eventTimes.size());
- }
- }
-
- private Map<String, Object> createTriggerProperties(String startTime, String timeZone) {
- return createTriggerProperties(startTime, timeZone, "+3SECOND", "+2SECOND");
- }
-
- private Map<String, Object> createTriggerProperties(String startTime, String timeZone, String every, String graceTime) {
- Map<String, Object> properties = new HashMap<>();
- properties.put("graceDuration", graceTime);
- properties.put("startTime", startTime);
- properties.put("timeZone", timeZone);
- properties.put("every", every);
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- properties.put("actions", actions);
- return properties;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
deleted file mode 100644
index 138daf7..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerIntegrationTest.java
+++ /dev/null
@@ -1,747 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.carrotsearch.randomizedtesting.annotations.Nightly;
-import com.google.common.util.concurrent.AtomicDouble;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.ZkNodeProps;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.util.LogLevel;
-import org.apache.zookeeper.data.Stat;
-import org.junit.AfterClass;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.cloud.autoscaling.TriggerIntegrationTest.WAIT_FOR_DELTA_NANOS;
-import static org.apache.solr.common.cloud.ZkStateReader.SOLR_AUTOSCALING_CONF_PATH;
-
-/**
- * Integration test for {@link SearchRateTrigger}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG;org.apache.solr.client.solrj.cloud.autoscaling=DEBUG")
-@LuceneTestCase.Slow
-@Nightly // this test is too long for non nightly right now
-@Ignore // nocommit this is removed in master
-public class SearchRateTriggerIntegrationTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final TimeSource timeSource = TimeSource.NANO_TIME;
- private static volatile CountDownLatch listenerCreated = new CountDownLatch(1);
- private static volatile CountDownLatch listenerEventLatch = new CountDownLatch(0);
- private static volatile Map<String, List<CapturedEvent>> listenerEvents = new HashMap<>();
- private static volatile CountDownLatch finished = new CountDownLatch(1);
- private static volatile CountDownLatch started = new CountDownLatch(1);
- private static SolrCloudManager cloudManager;
-
- private int waitForSeconds;
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(5)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
-
- cloudManager = cluster.getOpenOverseer().getSolrCloudManager();
-
- // disable .scheduled_maintenance (once it exists)
- CloudTestUtils.waitForTriggerToBeScheduled(cloudManager, ".scheduled_maintenance");
- CloudTestUtils.suspendTrigger(cloudManager, ".scheduled_maintenance");
-
- }
-
- @AfterClass
- public static void cleanUpAfterClass() throws Exception {
- cloudManager = null;
- }
-
- @Before
- public void beforeTest() throws Exception {
- cluster.deleteAllCollections();
- // clear any persisted auto scaling configuration
- Stat stat = zkClient().setData(SOLR_AUTOSCALING_CONF_PATH, Utils.toJSON(new ZkNodeProps()), true);
- if (log.isInfoEnabled()) {
- log.info("{} reset, new znode version {}", SOLR_AUTOSCALING_CONF_PATH, stat.getVersion());
- }
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_EVENTS_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_TRIGGER_STATE_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_LOST_PATH);
- deleteChildrenRecursively(ZkStateReader.SOLR_AUTOSCALING_NODE_ADDED_PATH);
-
- finished = new CountDownLatch(1);
- started = new CountDownLatch(1);
- listenerCreated = new CountDownLatch(1);
- listenerEvents = new HashMap<>();
- listenerEventLatch = new CountDownLatch(0);
-
- waitForSeconds = 3 + random().nextInt(5);
- }
-
- private void deleteChildrenRecursively(String path) throws Exception {
- cloudManager.getDistribStateManager().removeRecursively(path, true, false);
- }
-
- @Test
- public void testAboveSearchRate() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String COLL1 = "aboveRate_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 1, 2);
- create.process(solrClient);
-
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 2));
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- // the trigger is initially disabled so that we have the time to set up listeners
- // and generate the traffic
- "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger1'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : false," +
- "'collections' : '" + COLL1 + "'," +
- "'aboveRate' : 1.0," +
- "'belowRate' : 0.1," +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
- "]" +
- "}}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'started'," +
- "'trigger' : 'search_rate_trigger1'," +
- "'stage' : ['STARTED']," +
- "'class' : '" + StartedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'search_rate_trigger1'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}");
- listenerEventLatch = new CountDownLatch(3);
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'search_rate_trigger1'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- SolrParams query = params(CommonParams.Q, "*:*");
- for (int i = 0; i < 500; i++) {
- solrClient.query(COLL1, query);
- }
-
- // enable the trigger
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'resume-trigger' : {" +
- "'name' : 'search_rate_trigger1'" +
- "}" +
- "}");
-
- assertTrue("The trigger did not start in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("The trigger did not finish in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("the listener should have recorded all events w/in a reasonable amount of time",
- listenerEventLatch.await(60, TimeUnit.SECONDS));
-
- // suspend the trigger
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'suspend-trigger' : {" +
- "'name' : 'search_rate_trigger1'" +
- "}" +
- "}");
-
- List<CapturedEvent> events = listenerEvents.get("srt");
- assertEquals(listenerEvents.toString(), 3, events.size());
- assertEquals("AFTER_ACTION", events.get(0).stage.toString());
- assertEquals("compute", events.get(0).actionName);
- assertEquals("AFTER_ACTION", events.get(1).stage.toString());
- assertEquals("execute", events.get(1).actionName);
- assertEquals("SUCCEEDED", events.get(2).stage.toString());
- assertNull(events.get(2).actionName);
-
- CapturedEvent ev = events.get(0);
- long now = timeSource.getTimeNs();
- // verify waitFor
- assertTrue(TimeUnit.SECONDS.convert(waitForSeconds, TimeUnit.NANOSECONDS) - WAIT_FOR_DELTA_NANOS <= now - ev.event.getEventTime());
- Map<String, Double> nodeRates = (Map<String, Double>) ev.event.getProperties().get(SearchRateTrigger.HOT_NODES);
- assertNotNull("nodeRates", nodeRates);
- // no node violations because node rates weren't set in the config
- assertTrue(nodeRates.toString(), nodeRates.isEmpty());
- List<ReplicaInfo> replicaRates = (List<ReplicaInfo>) ev.event.getProperties().get(SearchRateTrigger.HOT_REPLICAS);
- assertNotNull("replicaRates", replicaRates);
- assertTrue(replicaRates.toString(), replicaRates.size() > 0);
- AtomicDouble totalReplicaRate = new AtomicDouble();
- replicaRates.forEach(r -> {
- assertTrue(r.toString(), r.getVariable("rate") != null);
- totalReplicaRate.addAndGet((Double) r.getVariable("rate"));
- });
- Map<String, Object> shardRates = (Map<String, Object>) ev.event.getProperties().get(SearchRateTrigger.HOT_SHARDS);
- assertNotNull("shardRates", shardRates);
- assertEquals(shardRates.toString(), 1, shardRates.size());
- shardRates = (Map<String, Object>) shardRates.get(COLL1);
- assertNotNull("shardRates", shardRates);
- assertEquals(shardRates.toString(), 1, shardRates.size());
- AtomicDouble totalShardRate = new AtomicDouble();
- shardRates.forEach((s, r) -> totalShardRate.addAndGet((Double) r));
- Map<String, Double> collectionRates = (Map<String, Double>) ev.event.getProperties().get(SearchRateTrigger.HOT_COLLECTIONS);
- assertNotNull("collectionRates", collectionRates);
- assertEquals(collectionRates.toString(), 1, collectionRates.size());
- Double collectionRate = collectionRates.get(COLL1);
- assertNotNull(collectionRate);
- assertTrue(collectionRate > 5.0);
- // two replicas - the trigger calculates average over all searchable replicas
- assertEquals(collectionRate / 2, totalShardRate.get(), 5.0);
- assertEquals(collectionRate, totalReplicaRate.get(), 5.0);
-
- // check operations
- List<MapWriter> ops = (List<MapWriter>) ev.context.get("properties.operations");
- assertNotNull(ops);
- assertTrue(ops.size() > 1);
- for (MapWriter m : ops) {
- assertEquals("ADDREPLICA", m._get("params.action",null));
- }
- }
-
- @Test
- public void testBelowSearchRate() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String COLL1 = "belowRate_collection";
- // replicationFactor == 2
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 1, 2);
- create.process(solrClient);
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 2));
-
- // add a couple of spare replicas above RF. Use different types.
- // these additional replicas will be placed on other nodes in the cluster
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT));
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG));
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL));
-
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 5));
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger2'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : false," +
- "'collections' : '" + COLL1 + "'," +
- "'aboveRate' : 1.0," +
- "'aboveNodeRate' : 1.0," +
- // RecoveryStrategy calls /admin/ping, which calls /select so the rate may not be zero
- // even when no external requests were made .. but it's hard to predict exactly
- // what it will be. use an insanely high rate so all shards/nodes are suspect
- // and produce an Op regardless of how much internal traffic is produced...
- "'belowRate' : 1.0," +
- "'belowNodeRate' : 1.0," +
- // ...but do absolutely nothing to nodes except generate an 'NONE' Op
- "'belowNodeOp' : 'none'," +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
- "]" +
- "}}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'started'," +
- "'trigger' : 'search_rate_trigger2'," +
- "'stage' : ['STARTED']," +
- "'class' : '" + StartedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'search_rate_trigger2'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}");
- listenerEventLatch = new CountDownLatch(3);
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'search_rate_trigger2'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- // Explicitly Do Nothing Here
-
- // enable the trigger
- final String resumeTriggerCommand = "{ 'resume-trigger' : { 'name' : 'search_rate_trigger2' } }";
- CloudTestUtils.assertAutoScalingRequest(cloudManager, resumeTriggerCommand);
-
- assertTrue("The trigger did not start in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("The trigger did not finish in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("the listener should have recorded all events w/in a reasonable amount of time",
- listenerEventLatch.await(60, TimeUnit.SECONDS));
-
- // suspend the trigger
- final String suspendTriggerCommand = "{ 'suspend-trigger' : { 'name' : 'search_rate_trigger2' } }";
- CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand);
-
- List<CapturedEvent> events = listenerEvents.get("srt");
- assertEquals(events.toString(), 3, events.size());
- CapturedEvent ev = events.get(0);
- assertEquals(ev.toString(), "compute", ev.actionName);
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
- // 5 cold nodes, 3 cold replicas
- assertEquals(ops.toString(), 5 + 3, ops.size());
- AtomicInteger coldNodes = new AtomicInteger();
- AtomicInteger coldReplicas = new AtomicInteger();
- ops.forEach(op -> {
- if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) {
- coldNodes.incrementAndGet();
- } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
- coldReplicas.incrementAndGet();
- } else {
- fail("unexpected op: " + op);
- }
- });
- assertEquals("cold nodes", 5, coldNodes.get());
- assertEquals("cold replicas", 3, coldReplicas.get());
-
- // now the collection should be down to RF = 2
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 2));
-
- listenerEvents.clear();
- listenerEventLatch = new CountDownLatch(3);
- finished = new CountDownLatch(1);
- started = new CountDownLatch(1);
-
- // resume trigger
- CloudTestUtils.assertAutoScalingRequest(cloudManager, resumeTriggerCommand);
-
- assertTrue("The trigger did not start in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("The trigger did not finish in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("the listener should have recorded all events w/in a reasonable amount of time",
- listenerEventLatch.await(60, TimeUnit.SECONDS));
-
- // suspend the trigger
- CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand);
-
- // there should be only coldNode ops now, and no coldReplica ops since searchable RF == collection RF
-
- events = listenerEvents.get("srt");
- assertEquals(events.toString(), 3, events.size());
-
- ev = events.get(0);
- assertEquals(ev.toString(), "compute", ev.actionName);
- ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
- assertEquals(ops.toString(), 2, ops.size());
- assertEquals(ops.toString(), CollectionParams.CollectionAction.NONE, ops.get(0).getAction());
- assertEquals(ops.toString(), CollectionParams.CollectionAction.NONE, ops.get(1).getAction());
-
-
- listenerEvents.clear();
- listenerEventLatch = new CountDownLatch(3);
- finished = new CountDownLatch(1);
- started = new CountDownLatch(1);
-
- log.info("## test single replicas.");
-
- // now allow single replicas
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger2'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : true," +
- "'collections' : '" + COLL1 + "'," +
- "'aboveRate' : 1.0," +
- "'aboveNodeRate' : 1.0," +
- "'belowRate' : 1.0," + // same excessively high values
- "'belowNodeRate' : 1.0," +
- "'minReplicas' : 1," + // NEW: force lower replicas
- "'belowNodeOp' : 'none'," + // still do nothing to nodes
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
- "]" +
- "}}");
-
- assertTrue("The trigger did not start in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("The trigger did not finish in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("the listener should have recorded all events w/in a reasonable amount of time",
- listenerEventLatch.await(60, TimeUnit.SECONDS));
-
- // suspend the trigger
- CloudTestUtils.assertAutoScalingRequest(cloudManager, suspendTriggerCommand);
-
- events = listenerEvents.get("srt");
- assertEquals(events.toString(), 3, events.size());
-
- ev = events.get(0);
- assertEquals(ev.toString(), "compute", ev.actionName);
- ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
-
- assertTrue(ops.toString(), ops.size() > 0);
- AtomicInteger coldNodes2 = new AtomicInteger();
- ops.forEach(op -> {
- if (op.getAction().equals(CollectionParams.CollectionAction.NONE)) {
- coldNodes2.incrementAndGet();
- } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
- // ignore
- } else {
- fail("unexpected op: " + op);
- }
- });
-
- assertEquals("coldNodes: " +ops.toString(), 2, coldNodes2.get());
-
- // now the collection should be at RF == 1, with one additional PULL replica
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 1));
- }
-
- @Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-13163")
- public void testDeleteNode() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String COLL1 = "deleteNode_collection";
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 1, 2);
-
- create.process(solrClient);
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 2));
-
- // add a couple of spare replicas above RF. Use different types to verify that only
- // searchable replicas are considered
- // these additional replicas will be placed on other nodes in the cluster
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.NRT));
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.TLOG));
- solrClient.request(CollectionAdminRequest.addReplicaToShard(COLL1, "shard1", Replica.Type.PULL));
-
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 5));
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-trigger' : {" +
- "'name' : 'search_rate_trigger3'," +
- "'event' : 'searchRate'," +
- "'waitFor' : '" + waitForSeconds + "s'," +
- "'enabled' : false," +
- "'collections' : '" + COLL1 + "'," +
- "'aboveRate' : 1.0," +
- "'aboveNodeRate' : 1.0," +
- // RecoveryStrategy calls /admin/ping, which calls /select so the rate may not be zero
- // even when no external requests were made .. but it's hard to predict exactly
- // what it will be. use an insanely high rate so all shards/nodes are suspect
- // and produce an Op regardless of how much internal traffic is produced...
- "'belowRate' : 1.0," +
- "'belowNodeRate' : 1.0," +
- // ...our Ops should be to delete underutilised nodes...
- "'belowNodeOp' : 'DELETENODE'," +
- // ...allow deleting all spare replicas...
- "'minReplicas' : 1," +
- // ...and allow requesting all deletions in one event.
- "'maxOps' : 10," +
- "'actions' : [" +
- "{'name':'compute','class':'" + ComputePlanAction.class.getName() + "'}," +
- "{'name':'execute','class':'" + ExecutePlanAction.class.getName() + "'}" +
- "]" +
- "}}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'started'," +
- "'trigger' : 'search_rate_trigger3'," +
- "'stage' : ['STARTED']," +
- "'class' : '" + StartedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'srt'," +
- "'trigger' : 'search_rate_trigger3'," +
- "'stage' : ['FAILED','SUCCEEDED']," +
- "'afterAction': ['compute', 'execute']," +
- "'class' : '" + CapturingTriggerListener.class.getName() + "'" +
- "}" +
- "}");
- listenerEventLatch = new CountDownLatch(3);
-
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'finished'," +
- "'trigger' : 'search_rate_trigger3'," +
- "'stage' : ['SUCCEEDED']," +
- "'class' : '" + FinishedProcessingListener.class.getName() + "'" +
- "}" +
- "}");
-
- // Explicitly Do Nothing Here
-
- // enable the trigger
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'resume-trigger' : {" +
- "'name' : 'search_rate_trigger3'" +
- "}" +
- "}");
-
- assertTrue("The trigger did not start in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("The trigger did not finish in a reasonable amount of time",
- started.await(60, TimeUnit.SECONDS));
-
- assertTrue("the listener should have recorded all events w/in a reasonable amount of time",
- listenerEventLatch.await(60, TimeUnit.SECONDS));
-
- // suspend the trigger
- CloudTestUtils.assertAutoScalingRequest
- (cloudManager,
- "{" +
- "'suspend-trigger' : {" +
- "'name' : 'search_rate_trigger3'" +
- "}" +
- "}");
-
- List<CapturedEvent> events = listenerEvents.get("srt");
- assertEquals(events.toString(), 3, events.size());
-
- CapturedEvent ev = events.get(0);
- assertEquals(ev.toString(), "compute", ev.actionName);
- List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>)ev.event.getProperty(TriggerEvent.REQUESTED_OPS);
- assertNotNull("there should be some requestedOps: " + ev.toString(), ops);
- // 4 DELETEREPLICA, 4 DELETENODE (minReplicas==1 & leader should be protected)
- assertEquals(ops.toString(), 4 + 4, ops.size());
- // The above assert can fail with actual==9 because all 5 nodes are resulting in a DELETENODE
- // Which is problemtatic for 2 reasons:
- // 1) it means that the leader node has not been protected from the 'belowNodeOp':'DELETENODE'
- // - definitely a bug that needs fixed
- // 2) it suggests that minReplicas isn't being respected by 'belowNodeOp':'DELETENODE'
- // - something that needs more rigerous testing
- // - ie: if belowRate==0 && belowNodeRate==1 && minReplicas==2, will leader + 1 be protected?
- //
- // In general, to adequately trust testing of 'belowNodeOp':'DELETENODE' we should also test:
- // - some nodes with multiple replicas of the shard to ensure best nodes are picked
- // - node nodes hosting replicas of multiple shards/collection, only some of which are belowNodeRate
-
-
-
- AtomicInteger replicas = new AtomicInteger();
- AtomicInteger nodes = new AtomicInteger();
- ops.forEach(op -> {
- if (op.getAction().equals(CollectionParams.CollectionAction.DELETEREPLICA)) {
- replicas.incrementAndGet();
- } else if (op.getAction().equals(CollectionParams.CollectionAction.DELETENODE)) {
- nodes.incrementAndGet();
- } else {
- fail("unexpected op: " + op);
- }
- });
- assertEquals(ops.toString(), 4, replicas.get());
- assertEquals(ops.toString(), 4, nodes.get());
- // check status
- ev = events.get(1);
- assertEquals(ev.toString(), "execute", ev.actionName);
- List<NamedList<Object>> responses = (List<NamedList<Object>>)ev.context.get("properties.responses");
- assertNotNull(ev.toString(), responses);
- assertEquals(responses.toString(), 8, responses.size());
- replicas.set(0);
- nodes.set(0);
- responses.forEach(m -> {
- if (m.get("success") != null) {
- replicas.incrementAndGet();
- } else if (m.get("status") != null) {
- Object status = m.get("status");
- String state;
- if (status instanceof Map) {
- state = (String)((Map)status).get("state");
- } else if (status instanceof NamedList) {
- state = (String)((NamedList)status).get("state");
- } else {
- throw new IllegalArgumentException("unsupported status format: " + status.getClass().getName() + ", " + status);
- }
- if ("completed".equals(state)) {
- nodes.incrementAndGet();
- } else {
- fail("unexpected DELETENODE status: " + m);
- }
- } else {
- fail("unexpected status: " + m);
- }
- });
-
- assertEquals(responses.toString(), 4, replicas.get());
- assertEquals(responses.toString(), 4, nodes.get());
-
- // we are left with one searchable replica
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS,
- CloudUtil.clusterShape(1, 1));
- }
-
- public static class CapturingTriggerListener extends TriggerListenerBase {
- @Override
- public void configure(SolrResourceLoader loader, SolrCloudManager cloudManager, AutoScalingConfig.TriggerListenerConfig config) throws TriggerValidationException {
- super.configure(loader, cloudManager, config);
- listenerCreated.countDown();
- }
-
- @Override
- public synchronized void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName,
- ActionContext context, Throwable error, String message) {
- CapturedEvent ev = new CapturedEvent(timeSource.getTimeNs(), context, config, stage, actionName, event, message);
- final CountDownLatch latch = listenerEventLatch;
- synchronized (latch) {
- if (0 == latch.getCount()) {
- log.warn("Ignoring captured event since latch is 'full': {}", ev);
- } else {
- List<CapturedEvent> lst = listenerEvents.computeIfAbsent(config.name, s -> new ArrayList<>());
- log.info("=======> {}", ev);
- lst.add(ev);
- latch.countDown();
- }
- }
- }
- }
-
- public static class StartedProcessingListener extends TriggerListenerBase {
-
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- started.countDown();
- }
- }
-
- public static class FinishedProcessingListener extends TriggerListenerBase {
-
- @Override
- public void onEvent(TriggerEvent event, TriggerEventProcessorStage stage, String actionName, ActionContext context, Throwable error, String message) throws Exception {
- finished.countDown();
- }
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
deleted file mode 100644
index d04f88d..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SearchRateTriggerTest.java
+++ /dev/null
@@ -1,360 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import com.codahale.metrics.MetricRegistry;
-import com.google.common.util.concurrent.AtomicDouble;
-
-import org.apache.solr.client.solrj.cloud.NodeStateProvider;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventType;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
-import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudUtil;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.ZkDistributedQueueFactory;
-import org.apache.solr.common.cloud.SolrZkClient;
-import org.apache.solr.common.params.AutoScalingParams;
-import org.apache.solr.common.params.CollectionParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.SolrParams;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.core.CoreContainer;
-import org.apache.solr.core.SolrResourceLoader;
-import org.apache.solr.metrics.SolrMetricManager;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.Test;
-
-/**
- *
- */
-@Ignore // nocommit this is removed in master
-public class SearchRateTriggerTest extends SolrCloudTestCase {
- private static final String PREFIX = SearchRateTriggerTest.class.getSimpleName() + "-";
- private static final String COLL1 = PREFIX + "collection1";
- private static final String COLL2 = PREFIX + "collection2";
-
- private AutoScaling.TriggerEventProcessor noFirstRunProcessor = event -> {
- fail("Did not expect the listener to fire on first run!");
- return true;
- };
-
- @BeforeClass
- public static void setupCluster() throws Exception {
-
- }
-
- @Before
- public void removeCollections() throws Exception {
- configureCluster(4)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @After
- public void after() throws Exception {
- shutdownCluster();
- }
-
- @Test
- @AwaitsFix(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028")
- public void testTrigger() throws Exception {
- JettySolrRunner targetNode = cluster.getJettySolrRunner(0);
- SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
- SolrResourceLoader loader = targetNode.getCoreContainer().getResourceLoader();
- CoreContainer container = targetNode.getCoreContainer();
- SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient());
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 2, 2);
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
- create = CollectionAdminRequest.createCollection(COLL2,
- "conf", 2, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
-
- CloudUtil.waitForState(cloudManager, COLL1, 60, TimeUnit.SECONDS, clusterShape(2, 2));
- CloudUtil.waitForState(cloudManager, COLL2, 60, TimeUnit.SECONDS, clusterShape(2, 2));
-
- double rate = 1.0;
- String baseUrl = targetNode.getBaseUrl();
- long waitForSeconds = 5 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(Arrays.asList(COLL1, COLL2), waitForSeconds, rate, -1);
- final List<TriggerEvent> events = new ArrayList<>();
-
- try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger")) {
- trigger.configure(loader, cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
- trigger.setProcessor(event -> events.add(event));
-
- // generate replica traffic
- String coreName = container.getLoadedCoreNames().iterator().next();
- String url = baseUrl.toString() + "/" + coreName;
- try (HttpSolrClient simpleClient = new HttpSolrClient.Builder(url).build()) {
- SolrParams query = params(CommonParams.Q, "*:*", CommonParams.DISTRIB, "false");
- for (int i = 0; i < 130; i++) {
- simpleClient.query(query);
- }
- String registryCoreName = coreName.replaceFirst("_", ".").replaceFirst("_", ".");
- SolrMetricManager manager = targetNode.getCoreContainer().getMetricManager();
- MetricRegistry registry = manager.registry("solr.core."+registryCoreName);
- TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME);
- // If we getting the rate too early, it will return 0
- timeOut.waitFor("Timeout waiting for rate is not zero",
- () -> registry.timer("QUERY./select.requestTimes").getOneMinuteRate()!=0.0);
- trigger.run();
- // waitFor delay
- assertEquals(0, events.size());
- Thread.sleep(waitForSeconds * 1000);
- // should generate replica event
- trigger.run();
- assertEquals(1, events.size());
- TriggerEvent event = events.get(0);
- assertEquals(TriggerEventType.SEARCHRATE, event.eventType);
- List<ReplicaInfo> infos = (List<ReplicaInfo>)event.getProperty(SearchRateTrigger.HOT_REPLICAS);
- assertEquals(1, infos.size());
- ReplicaInfo info = infos.get(0);
- assertEquals(coreName, info.getCore());
- assertTrue((Double)info.getVariable(AutoScalingParams.RATE) > rate);
- }
- // close that jetty to remove the violation - alternatively wait for 1 min...
- JettySolrRunner j = cluster.stopJettySolrRunner(1);
- cluster.waitForJettyToStop(j);
- events.clear();
- SolrParams query = params(CommonParams.Q, "*:*");
- for (int i = 0; i < 130; i++) {
- solrClient.query(COLL1, query);
- }
- Thread.sleep(waitForSeconds * 1000);
- trigger.run();
- // should generate collection event
- assertEquals(1, events.size());
- TriggerEvent event = events.get(0);
- Map<String, Double> hotCollections = (Map<String, Double>)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS);
- assertEquals(1, hotCollections.size());
- Double Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
- assertTrue(Rate > rate);
- events.clear();
-
- for (int i = 0; i < 150; i++) {
- solrClient.query(COLL2, query);
- solrClient.query(COLL1, query);
- }
- Thread.sleep(waitForSeconds * 1000);
- trigger.run();
- // should generate collection event but not for COLL2 because of waitFor
- assertEquals(1, events.size());
- event = events.get(0);
- Map<String, Double> hotNodes = (Map<String, Double>)event.getProperty(SearchRateTrigger.HOT_NODES);
- assertTrue("hotNodes", hotNodes.isEmpty());
- hotNodes.forEach((n, r) -> assertTrue(n, r > rate));
- hotCollections = (Map<String, Double>)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS);
- assertEquals(1, hotCollections.size());
- Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
-
- events.clear();
- // assert that waitFor prevents new events from being generated
- trigger.run();
- // should not generate any events
- assertEquals(0, events.size());
-
- Thread.sleep(waitForSeconds * 1000 * 2);
- trigger.run();
- // should generate collection event
- assertEquals(1, events.size());
- event = events.get(0);
- hotCollections = (Map<String, Double>)event.getProperty(SearchRateTrigger.HOT_COLLECTIONS);
- assertEquals(2, hotCollections.size());
- Rate = hotCollections.get(COLL1);
- assertNotNull(Rate);
- Rate = hotCollections.get(COLL2);
- assertNotNull(Rate);
- hotNodes = (Map<String, Double>)event.getProperty(SearchRateTrigger.HOT_NODES);
- assertTrue("hotNodes", hotNodes.isEmpty());
- }
- }
-
- private static final AtomicDouble mockRate = new AtomicDouble();
-
- @Test
- public void testWaitForElapsed() throws Exception {
- SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- SolrZkClient zkClient = solrClient.getZkStateReader().getZkClient();
- SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), solrClient) {
- @Override
- public NodeStateProvider getNodeStateProvider() {
- return new SolrClientNodeStateProvider(solrClient, null) {
- @Override
- public Map<String, Object> getNodeValues(String node, Collection<String> tags) {
- Map<String, Object> values = super.getNodeValues(node, tags);
- values.keySet().forEach(k -> {
- values.replace(k, mockRate.get());
- });
- return values;
- }
- };
- }
- };
- TimeSource timeSource = cloudManager.getTimeSource();
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(COLL1,
- "conf", 2, 2);
- create.setMaxShardsPerNode(1);
- create.process(solrClient);
- CloudUtil.waitForState(cloudManager, COLL1, 10, TimeUnit.SECONDS, clusterShape(2, 4));
-
- long waitForSeconds = 3 + random().nextInt(5);
- Map<String, Object> props = createTriggerProps(Arrays.asList(COLL1, COLL2), waitForSeconds, 1.0, 0.1);
- final List<TriggerEvent> events = new ArrayList<>();
-
- try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger1")) {
- trigger.configure(loader, cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
- trigger.setProcessor(event -> events.add(event));
-
- // set mock rates
- mockRate.set(2.0);
- TimeOut timeOut = new TimeOut(waitForSeconds + 2, TimeUnit.SECONDS, timeSource);
- // simulate ScheduledTriggers
- while (!timeOut.hasTimedOut()) {
- trigger.run();
- timeSource.sleep(1000);
- }
- // violation persisted longer than waitFor - there should be events
- assertTrue(events.toString(), events.size() > 0);
- TriggerEvent event = events.get(0);
- assertEquals(event.toString(), TriggerEventType.SEARCHRATE, event.eventType);
- Map<String, Object> hotNodes, hotCollections, hotShards;
- List<ReplicaInfo> hotReplicas;
- hotNodes = (Map<String, Object>)event.properties.get(SearchRateTrigger.HOT_NODES);
- hotCollections = (Map<String, Object>)event.properties.get(SearchRateTrigger.HOT_COLLECTIONS);
- hotShards = (Map<String, Object>)event.properties.get(SearchRateTrigger.HOT_SHARDS);
- hotReplicas = (List<ReplicaInfo>)event.properties.get(SearchRateTrigger.HOT_REPLICAS);
- assertTrue("no hot nodes?", hotNodes.isEmpty());
- assertFalse("no hot collections?", hotCollections.isEmpty());
- assertFalse("no hot shards?", hotShards.isEmpty());
- assertFalse("no hot replicas?", hotReplicas.isEmpty());
- }
-
- mockRate.set(0.0);
- events.clear();
-
- try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger2")) {
- trigger.configure(loader, cloudManager, props);
- trigger.init();
- trigger.setProcessor(noFirstRunProcessor);
- trigger.run();
- trigger.setProcessor(event -> events.add(event));
-
- mockRate.set(2.0);
- trigger.run();
- // waitFor not elapsed
- assertTrue(events.toString(), events.isEmpty());
- Thread.sleep(1000);
- trigger.run();
- assertTrue(events.toString(), events.isEmpty());
- Thread.sleep(1000);
- mockRate.set(0.0);
- trigger.run();
- Thread.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds - 2, TimeUnit.SECONDS));
- trigger.run();
-
- // violations persisted shorter than waitFor - there should be no events
- assertTrue(events.toString(), events.isEmpty());
-
- }
- }
-
- @Test
- public void testDefaultsAndBackcompat() throws Exception {
- Map<String, Object> props = new HashMap<>();
- props.put("rate", 1.0);
- props.put("collection", "test");
- SolrResourceLoader loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
- SolrZkClient zkClient = cluster.getSolrClient().getZkStateReader().getZkClient();
- SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(zkClient), cluster.getSolrClient());
- try (SearchRateTrigger trigger = new SearchRateTrigger("search_rate_trigger2")) {
- trigger.configure(loader, cloudManager, props);
- Map<String, Object> config = trigger.getConfig();
- Set<String> collections = (Set<String>)config.get(SearchRateTrigger.COLLECTIONS_PROP);
- assertEquals(collections.toString(), 1, collections.size());
- assertEquals("test", collections.iterator().next());
- assertEquals("#ANY", config.get(AutoScalingParams.SHARD));
- assertEquals("#ANY", config.get(AutoScalingParams.NODE));
- assertEquals(1.0, config.get(SearchRateTrigger.ABOVE_RATE_PROP));
- assertEquals(-1.0, config.get(SearchRateTrigger.BELOW_RATE_PROP));
- assertEquals(SearchRateTrigger.DEFAULT_METRIC, config.get(SearchRateTrigger.METRIC_PROP));
- assertEquals(SearchRateTrigger.DEFAULT_MAX_OPS, config.get(SearchRateTrigger.MAX_OPS_PROP));
- assertNull(config.get(SearchRateTrigger.MIN_REPLICAS_PROP));
- assertEquals(CollectionParams.CollectionAction.ADDREPLICA, config.get(SearchRateTrigger.ABOVE_OP_PROP));
- assertEquals(CollectionParams.CollectionAction.MOVEREPLICA, config.get(SearchRateTrigger.ABOVE_NODE_OP_PROP));
- assertEquals(CollectionParams.CollectionAction.DELETEREPLICA, config.get(SearchRateTrigger.BELOW_OP_PROP));
- assertNull(config.get(SearchRateTrigger.BELOW_NODE_OP_PROP));
- }
- }
-
- private Map<String, Object> createTriggerProps(List<String> collections, long waitForSeconds, double aboveRate, double belowRate) {
- Map<String, Object> props = new HashMap<>();
- props.put("aboveRate", aboveRate);
- props.put("belowRate", belowRate);
- props.put("event", "searchRate");
- props.put("waitFor", waitForSeconds);
- props.put("enabled", true);
- if (collections != null && !collections.isEmpty()) {
- props.put("collections", String.join(",", collections));
- }
- List<Map<String, String>> actions = new ArrayList<>(3);
- Map<String, String> map = new HashMap<>(2);
- map.put("name", "compute_plan");
- map.put("class", "solr.ComputePlanAction");
- actions.add(map);
- map = new HashMap<>(2);
- map.put("name", "execute_plan");
- map.put("class", "solr.ExecutePlanAction");
- actions.add(map);
- props.put("actions", actions);
- return props;
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
deleted file mode 100644
index b288fef..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/SystemLogListenerTest.java
+++ /dev/null
@@ -1,294 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import java.io.IOException;
-import java.lang.invoke.MethodHandles;
-import java.util.Collection;
-import java.util.Map;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Supplier;
-
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.SolrServerException;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.client.solrj.response.QueryResponse;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.common.SolrDocument;
-import org.apache.solr.common.SolrDocumentList;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.params.CollectionAdminParams;
-import org.apache.solr.common.params.CommonParams;
-import org.apache.solr.common.params.ModifiableSolrParams;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.util.LogLevel;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Test for {@link SystemLogListener}
- */
-@LogLevel("org.apache.solr.cloud.autoscaling=DEBUG")
-@Ignore // nocommit this is removed in master
-public class SystemLogListenerTest extends SolrCloudTestCase {
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
-
- private static final AtomicBoolean fired = new AtomicBoolean(false);
- private static final int NODE_COUNT = 3;
- private static CountDownLatch triggerFiredLatch = new CountDownLatch(1);
- private static final AtomicReference<Map> actionContextPropsRef = new AtomicReference<>();
- private static final AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
-
- public static class AssertingTriggerAction extends TriggerActionBase {
- @Override
- public void process(TriggerEvent event, ActionContext context) {
- if (fired.compareAndSet(false, true)) {
- eventRef.set(event);
- actionContextPropsRef.set(context.getProperties());
- triggerFiredLatch.countDown();
- }
- }
- }
-
- public static class ErrorTriggerAction extends TriggerActionBase {
- @Override
- public void process(TriggerEvent event, ActionContext context) {
- throw new RuntimeException("failure from ErrorTriggerAction");
- }
- }
-
- @Before
- public void setupCluster() throws Exception {
- configureCluster(NODE_COUNT)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- CollectionAdminRequest.createCollection(CollectionAdminParams.SYSTEM_COLL, null, 1, 3)
- .process(cluster.getSolrClient());
- cluster.waitForActiveCollection(CollectionAdminParams.SYSTEM_COLL, 1, 3);
- }
-
- @After
- public void teardownCluster() throws Exception {
- shutdownCluster();
- }
-
- @Test
- public void test() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setTriggerCommand = "{" +
- "'set-trigger' : {" +
- "'name' : 'node_lost_trigger'," +
- "'event' : 'nodeLost'," +
- "'waitFor' : '1s'," +
- "'enabled' : true," +
- "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
- "{'name':'execute_plan','class':'solr.ExecutePlanAction'}," +
- "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}," +
- "{'name':'error','class':'" + ErrorTriggerAction.class.getName() + "'}]" +
- "}}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setTriggerCommand);
- NamedList<Object> response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // remove default listener
- String removeListenerCommand = "{\n" +
- "\t\"remove-listener\" : {\n" +
- "\t\t\"name\" : \"node_lost_trigger.system\"\n" +
- "\t}\n" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, removeListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection("test",
- "conf",3, 2);
- create.setMaxShardsPerNode(3);
- create.process(solrClient);
-
- waitForState("Timed out waiting for replicas of new collection to be active",
- "test", clusterShape(3, 6));
-
- String setListenerCommand = "{" +
- "'set-listener' : " +
- "{" +
- "'name' : 'foo'," +
- "'trigger' : 'node_lost_trigger'," +
- "'stage' : ['STARTED','ABORTED','SUCCEEDED', 'FAILED']," +
- "'beforeAction' : ['compute_plan','execute_plan','test','error']," +
- "'afterAction' : ['compute_plan','execute_plan','test','error']," +
- "'class' : '" + SystemLogListener.class.getName() + "'" +
- "}" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setListenerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
-
- // Stop a node (that's safe to stop for the purposes of this test)
- final JettySolrRunner stoppedJetty = pickNodeToStop();
- if (log.isInfoEnabled()) {
- log.info("Stopping node {}", stoppedJetty.getNodeName());
- }
- cluster.stopJettySolrRunner(stoppedJetty);
- cluster.waitForJettyToStop(stoppedJetty);
-
- assertTrue("Trigger was not fired ", triggerFiredLatch.await(60, TimeUnit.SECONDS));
- assertTrue(fired.get());
- Map context = actionContextPropsRef.get();
- assertNotNull(context);
-
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, TimeSource.NANO_TIME);
-
- ModifiableSolrParams query = new ModifiableSolrParams();
- query.add(CommonParams.Q, "type:" + SystemLogListener.DOC_TYPE);
- query.add(CommonParams.SORT, "id asc");
-
- try {
- timeout.waitFor("", new Supplier<Boolean>() {
-
- @Override
- public Boolean get() {
- try {
- cluster.getSolrClient().commit(CollectionAdminParams.SYSTEM_COLL, true, true);
-
- return cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL, query).getResults().size() == 9;
- } catch (SolrServerException | IOException e) {
- throw new RuntimeException(e);
- }
- }
- });
- } catch (TimeoutException e) {
- // fine
- }
- // make sure the event docs are replicated and committed
- Thread.sleep(5000);
- cluster.getSolrClient().commit(CollectionAdminParams.SYSTEM_COLL, true, true);
-
-
- QueryResponse resp = cluster.getSolrClient().query(CollectionAdminParams.SYSTEM_COLL, query);
- SolrDocumentList docs = resp.getResults();
- assertNotNull(docs);
- assertEquals("wrong number of events added to .system: " + docs.toString(),
- 9, docs.size());
- docs.forEach(doc -> assertCommonFields(doc));
-
- // STARTED
- SolrDocument doc = docs.get(0);
- assertEquals("STARTED", doc.getFieldValue("stage_s"));
-
- // BEFORE_ACTION compute_plan
- doc = docs.get(1);
- assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("compute_plan", doc.getFieldValue("action_s"));
-
- // AFTER_ACTION compute_plan
- doc = docs.get(2);
- assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("compute_plan", doc.getFieldValue("action_s"));
- Collection<Object> vals = doc.getFieldValues("operations.params_ts");
- assertEquals(3, vals.size());
- for (Object val : vals) {
- assertTrue(val.toString(), String.valueOf(val).contains("action=MOVEREPLICA"));
- }
-
- // BEFORE_ACTION execute_plan
- doc = docs.get(3);
- assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("execute_plan", doc.getFieldValue("action_s"));
- vals = doc.getFieldValues("operations.params_ts");
- assertEquals(3, vals.size());
-
- // AFTER_ACTION execute_plan
- doc = docs.get(4);
- assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("execute_plan", doc.getFieldValue("action_s"));
- vals = doc.getFieldValues("operations.params_ts");
- assertNotNull(vals);
- assertEquals(3, vals.size());
- vals = doc.getFieldValues("responses_ts");
- assertNotNull(vals);
- assertEquals(3, vals.size());
- vals.forEach(s -> assertTrue(s.toString(), s.toString().startsWith("success MOVEREPLICA action completed successfully")));
-
- // BEFORE_ACTION test
- doc = docs.get(5);
- assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("test", doc.getFieldValue("action_s"));
-
- // AFTER_ACTION test
- doc = docs.get(6);
- assertEquals("AFTER_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("test", doc.getFieldValue("action_s"));
-
- // BEFORE_ACTION error
- doc = docs.get(7);
- assertEquals("BEFORE_ACTION", doc.getFieldValue("stage_s"));
- assertEquals("error", doc.getFieldValue("action_s"));
-
- // FAILED error
- doc = docs.get(8);
- assertEquals("FAILED", doc.getFieldValue("stage_s"));
- assertEquals("error", doc.getFieldValue("action_s"));
- assertEquals("failure from ErrorTriggerAction", doc.getFieldValue("error.message_t"));
- assertTrue(doc.getFieldValue("error.details_t").toString().contains("RuntimeException"));
- }
-
- private void assertCommonFields(SolrDocument doc) {
- assertEquals(SystemLogListener.class.getSimpleName(), doc.getFieldValue(SystemLogListener.SOURCE_FIELD));
- assertEquals(SystemLogListener.DOC_TYPE, doc.getFieldValue(CommonParams.TYPE));
- assertEquals("node_lost_trigger", doc.getFieldValue("event.source_s"));
- assertNotNull(doc.getFieldValue("event.time_l"));
- assertNotNull(doc.getFieldValue("timestamp"));
- assertNotNull(doc.getFieldValue("event.property.nodeNames_ss"));
- assertNotNull(doc.getFieldValue("event_str"));
- assertEquals("NODELOST", doc.getFieldValue("event.type_s"));
- }
-
- /**
- * Helper method for picking a node that can safely be stoped
- * @see <a href="https://issues.apache.org/jira/browse/SOLR-13050">SOLR-13050</a>
- */
- private JettySolrRunner pickNodeToStop() throws Exception {
- // first get the nodeName of the overser.
- // stopping the overseer is not something we want to hassle with in this test
- final String overseerNodeName = (String) cluster.getSolrClient().request
- (CollectionAdminRequest.getOverseerStatus()).get("leader");
-
- // now find a node that is *NOT* the overseer or the leader of a .system collection shard
- for (Replica r : getCollectionState(CollectionAdminParams.SYSTEM_COLL).getReplicas()) {
- if ( ! (r.getBool("leader", false) || r.getNodeName().equals(overseerNodeName) ) ) {
- return cluster.getReplicaJetty(r);
- }
- }
- fail("Couldn't find non-leader, non-overseer, replica of .system collection to kill");
- return null;
- }
-
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
deleted file mode 100644
index 232fd46..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TestPolicyCloud.java
+++ /dev/null
@@ -1,545 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.nio.charset.StandardCharsets;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.function.BiConsumer;
-
-import com.google.common.collect.ImmutableSet;
-import org.apache.lucene.util.Constants;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.DistributedQueueFactory;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.Policy;
-import org.apache.solr.client.solrj.cloud.autoscaling.ReplicaInfo;
-import org.apache.solr.client.solrj.cloud.autoscaling.Row;
-import org.apache.solr.client.solrj.cloud.autoscaling.Variable.Type;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.BaseHttpSolrClient;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
-import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.SolrClientCloudManager;
-import org.apache.solr.client.solrj.impl.SolrClientNodeStateProvider;
-import org.apache.solr.client.solrj.request.CollectionAdminRequest;
-import org.apache.solr.cloud.CloudTestUtils.AutoScalingRequest;
-import org.apache.solr.cloud.OverseerTaskProcessor;
-import org.apache.solr.cloud.SolrCloudTestCase;
-import org.apache.solr.cloud.ZkDistributedQueueFactory;
-import org.apache.solr.common.cloud.CollectionStatePredicate;
-import org.apache.solr.common.cloud.DocCollection;
-import org.apache.solr.common.cloud.Replica;
-import org.apache.solr.common.cloud.Slice;
-import org.apache.solr.common.cloud.ZkStateReader;
-import org.apache.solr.common.util.NamedList;
-import org.apache.solr.common.util.TimeSource;
-import org.apache.solr.common.util.Utils;
-import org.apache.solr.util.TimeOut;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Ignore;
-import org.junit.rules.ExpectedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import static org.apache.solr.common.util.Utils.getObjectByPath;
-
-@LuceneTestCase.Slow
-@Ignore // nocommit this is removed in master
-public class TestPolicyCloud extends SolrCloudTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- @org.junit.Rule
- public ExpectedException expectedException = ExpectedException.none();
-
- @BeforeClass
- public static void setupCluster() throws Exception {
- configureCluster(5)
- .addConfig("conf", configset("cloud-minimal"))
- .configure();
- }
-
- @Before
- public void before() throws Exception {
- // remove default policy
- String commands = "{set-cluster-policy : []}";
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- }
-
- @After
- public void after() throws Exception {
- cluster.deleteAllCollections();
- cluster.getSolrClient().getZkStateReader().getZkClient().setData(ZkStateReader.SOLR_AUTOSCALING_CONF_PATH,
- "{}".getBytes(StandardCharsets.UTF_8), true);
- }
-
- public void testCreateCollection() throws Exception {
- String commands = "{ set-cluster-policy: [ {cores: '0', node: '#ANY'} ] }"; // disallow replica placement anywhere
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- String collectionName = "testCreateCollection";
- BaseHttpSolrClient.RemoteSolrException exp = expectThrows(BaseHttpSolrClient.RemoteSolrException.class,
- () -> CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1).process(cluster.getSolrClient()));
-
- assertTrue(exp.getMessage().contains("No node can satisfy the rules"));
- assertTrue(exp.getMessage().contains("AutoScaling.error.diagnostics"));
-
- // wait for a while until we don't see the collection
- TimeOut timeout = new TimeOut(30, TimeUnit.SECONDS, new TimeSource.NanoTimeSource());
- boolean removed = false;
- while (! timeout.hasTimedOut()) {
- timeout.sleep(100);
- removed = !cluster.getSolrClient().getZkStateReader().getClusterState().hasCollection(collectionName);
- if (removed) {
- timeout.sleep(500); // just a bit of time so it's more likely other
- // readers see on return
- break;
- }
- }
- if (!removed) {
- fail("Collection should have been deleted from cluster state but still exists: " + collectionName);
- }
-
- commands = "{ set-cluster-policy: [ {cores: '<2', node: '#ANY'} ] }";
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1).process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 2, 2);
-
- }
-
- public void testDataProviderPerReplicaDetails() throws Exception {
- CollectionAdminRequest.createCollection("perReplicaDataColl", "conf", 1, 5)
- .process(cluster.getSolrClient());
- cluster.waitForActiveCollection("perReplicaDataColl", 1, 5);
- DocCollection coll = getCollectionState("perReplicaDataColl");
- String autoScaleJson = "{" +
- " 'cluster-preferences': [" +
- " { maximize : freedisk , precision: 50}," +
- " { minimize : cores, precision: 2}" +
- " ]," +
- " 'cluster-policy': [" +
- " { replica : '0' , 'nodeRole': 'overseer'}," +
- " { 'replica': '<2', 'shard': '#ANY', 'node': '#ANY'" +
- " }" +
- " ]," +
- " 'policies': {" +
- " 'policy1': [" +
- " { 'replica': '<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " { 'replica': '<2', 'shard': '#EACH', 'sysprop.rack': 'rack1'}" +
- " ]" +
- " }" +
- "}";
- AutoScalingConfig config = new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson));
- AtomicInteger count = new AtomicInteger(0);
- try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(cluster.getZkClient()), cluster.getSolrClient())) {
- String nodeName = cloudManager.getClusterStateProvider().getLiveNodes().iterator().next();
- SolrClientNodeStateProvider nodeStateProvider = (SolrClientNodeStateProvider) cloudManager.getNodeStateProvider();
- Map<String, Map<String, List<ReplicaInfo>>> result = nodeStateProvider.getReplicaInfo(nodeName, Collections.singleton("UPDATE./update.requests"));
- nodeStateProvider.forEachReplica(nodeName, replicaInfo -> {
- if (replicaInfo.getVariables().containsKey("UPDATE./update.requests")) count.incrementAndGet();
- });
- assertTrue(count.get() > 0);
-
- Policy.Session session = config.getPolicy().createSession(cloudManager);
-
- for (Row row : session.getSortedNodes()) {
- Object val = row.getVal(Type.TOTALDISK.tagName, null);
- if (log.isInfoEnabled()) {
- log.info("node: {} , totaldisk : {}, freedisk : {}", row.node, val, row.getVal("freedisk", null));
- }
- assertTrue(val != null);
-
- }
-
- count .set(0);
- for (Row row : session.getSortedNodes()) {
- row.collectionVsShardVsReplicas.forEach((c, shardVsReplicas) -> shardVsReplicas.forEach((s, replicaInfos) -> {
- for (ReplicaInfo replicaInfo : replicaInfos) {
- if (replicaInfo.getVariables().containsKey(Type.CORE_IDX.tagName)) count.incrementAndGet();
- }
- }));
- }
- assertTrue(count.get() > 0);
- }
- }
-
- private static CollectionStatePredicate expectAllReplicasOnSpecificNode
- (final String expectedNodeName,
- final int expectedSliceCount,
- final int expectedReplicaCount) {
-
- return (liveNodes, collection) -> {
- if (null == collection || expectedSliceCount != collection.getSlices().size()) {
- return false;
- }
- int actualReplicaCount = 0;
- for (Slice slice : collection) {
- for (Replica replica : slice) {
- if ( ! (replica.isActive(liveNodes)
- && expectedNodeName.equals(replica.getNodeName())) ) {
- return false;
- }
- actualReplicaCount++;
- }
- }
- return expectedReplicaCount == actualReplicaCount;
- };
- }
-
- public void testCreateCollectionAddReplica() throws Exception {
- final JettySolrRunner jetty = cluster.getRandomJetty(random());
- final String jettyNodeName = jetty.getNodeName();
- final int port = jetty.getLocalPort();
-
- final String commands = "{set-policy :{c1 : [{replica:0 , shard:'#EACH', port: '!" + port + "'}]}}";
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
-
- final String collectionName = "testCreateCollectionAddReplica";
- log.info("Creating collection {}", collectionName);
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1)
- .setPolicy("c1")
- .process(cluster.getSolrClient());
-
- waitForState("Should have found exactly one replica, only on expected jetty: " +
- jettyNodeName + "/" + port,
- collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 1),
- 120, TimeUnit.SECONDS);
-
- log.info("Adding replica to {}", collectionName);
- CollectionAdminRequest.addReplicaToShard(collectionName, "shard1")
- .process(cluster.getSolrClient());
-
- waitForState("Should have found exactly two replicas, only on expected jetty: " +
- jettyNodeName + "/" + port,
- collectionName, expectAllReplicasOnSpecificNode(jettyNodeName, 1, 2),
- 120, TimeUnit.SECONDS);
-
- }
-
- public void testCreateCollectionSplitShard() throws Exception {
-
- final List<JettySolrRunner> shuffledJetties = new ArrayList<>(cluster.getJettySolrRunners());
- Collections.shuffle(shuffledJetties, random());
- assertTrue(2 < shuffledJetties.size()); // sanity check test setup
-
- final JettySolrRunner firstNode = shuffledJetties.get(0);
- final JettySolrRunner secondNode = shuffledJetties.get(1);
-
- final int firstNodePort = firstNode.getLocalPort();
- final int secondNodePort = secondNode.getLocalPort();
- assertNotEquals(firstNodePort, secondNodePort);
-
- final String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" +
- firstNodePort + "'}, {replica:1, shard:'#EACH', port:'" + secondNodePort + "'}]}}";
-
- final String firstNodeName = firstNode.getNodeName();
- final String secondNodeName = secondNode.getNodeName();
- assertNotEquals(firstNodeName, secondNodeName);
-
- final NamedList<Object> response = cluster.getSolrClient()
- .request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- assertEquals("success", response.get("result"));
-
- // through out the test, every shard shuld have 2 replicas, one on each of these two nodes
- final Set<String> expectedNodeNames = ImmutableSet.of(firstNodeName, secondNodeName);
-
- final String collectionName = "testCreateCollectionSplitShard";
- log.info("Creating collection {}", collectionName);
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2)
- .setPolicy("c1")
- .process(cluster.getSolrClient());
-
- waitForState("Should have found exactly 1 slice w/2 live Replicas, one on each expected jetty: " +
- firstNodeName + "/" + firstNodePort + " & " + secondNodeName + "/" + secondNodePort,
- collectionName, (liveNodes, collection) -> {
- // short circut if collection is deleted
- // or we some how have the wrong number of slices
- if (null == collection || 1 != collection.getSlices().size()) {
- return false;
- }
- // Note: only 1 slices, but simpler to loop then extract...
- for (Slice slice : collection.getSlices()) {
- // short circut if our slice isn't active, or has wrong # replicas
- if (Slice.State.ACTIVE != slice.getState()
- || 2 != slice.getReplicas().size()) {
- return false;
- }
- // make sure our replicas are fully live...
- final List<Replica> liveReplicas = slice.getReplicas
- ((r) -> r.isActive(liveNodes));
- if (2 != liveReplicas.size()) {
- return false;
- }
- // now the main check we care about: were the replicas split up on
- // the expected nodes...
- if (! expectedNodeNames.equals(ImmutableSet.of
- (liveReplicas.get(0).getNodeName(),
- liveReplicas.get(1).getNodeName()))) {
- return false;
- }
- }
- return true;
- });
-
- log.info("Splitting (single) Shard on collection {}", collectionName);
- CollectionAdminRequest.splitShard(collectionName).setShardName("shard1")
- .process(cluster.getSolrClient());
-
- waitForState("Should have found exactly 3 shards (1 inactive) each w/two live Replicas, " +
- "one on each expected jetty: " +
- firstNodeName + "/" + firstNodePort + " & " + secondNodeName + "/" + secondNodePort,
- collectionName, (liveNodes, collection) -> {
- // short circut if collection is deleted
- // or we some how have the wrong number of (active) slices
- if (null == collection
- || 3 != collection.getSlices().size()
- || 2 != collection.getActiveSlices().size()) {
- return false;
- }
- // Note: we're checking all slices, even the inactive (split) slice...
- for (Slice slice : collection.getSlices()) {
- // short circut if our slice has wrong # replicas
- if (2 != slice.getReplicas().size()) {
- return false;
- }
- // make sure our replicas are fully live...
- final List<Replica> liveReplicas = slice.getReplicas
- ((r) -> r.isActive(liveNodes));
- if (2 != liveReplicas.size()) {
- return false;
- }
- // now the main check we care about: were the replicas split up on
- // the expected nodes...
- if (! expectedNodeNames.equals(ImmutableSet.of
- (liveReplicas.get(0).getNodeName(),
- liveReplicas.get(1).getNodeName()))) {
- return false;
- }
- }
- return true;
- });
- }
-
- public void testMetricsTag() throws Exception {
- CloudHttp2SolrClient solrClient = cluster.getSolrClient();
- String setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'metrics:abc':'overseer', 'replica':0}" +
- " ]" +
- "}";
- SolrRequest req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- try {
- solrClient.request(req);
- fail("expected exception");
- } catch (BaseHttpSolrClient.RemoteExecutionException e) {
- // expected
- assertTrue(String.valueOf(getObjectByPath(e.getMetaData(),
- false, "error/details[0]/errorMessages[0]")).contains("Invalid metrics: param in"));
- }
- setClusterPolicyCommand = "{" +
- " 'set-cluster-policy': [" +
- " {'cores':'<10', 'node':'#ANY'}," +
- " {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
- " {'metrics:solr.node:ADMIN./admin/authorization.clientErrors:count':'>58768765', 'replica':0}" +
- " ]" +
- "}";
- req = AutoScalingRequest.create(SolrRequest.METHOD.POST, setClusterPolicyCommand);
- solrClient.request(req);
-
- final String collectionName = "metrics_tags";
- CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1)
- .process(cluster.getSolrClient());
- cluster.waitForActiveCollection(collectionName, 1, 1);
- DocCollection collection = getCollectionState(collectionName);
- DistributedQueueFactory queueFactory = new ZkDistributedQueueFactory(cluster.getZkClient());
- try (SolrCloudManager provider = new SolrClientCloudManager(queueFactory, solrClient)) {
- List<String> tags = Arrays.asList("metrics:solr.node:ADMIN./admin/authorization.clientErrors:count",
- "metrics:solr.jvm:buffers.direct.Count");
- Map<String, Object> val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), tags);
- for (String tag : tags) {
- assertNotNull("missing : " + tag, val.get(tag));
- }
- val = provider.getNodeStateProvider().getNodeValues(collection.getReplicas().get(0).getNodeName(), Collections.singleton("diskType"));
-
- Set<String> diskTypes = ImmutableSet.of("rotational", "ssd");
- assertTrue(diskTypes.contains(val.get("diskType")));
- }
- }
-
- public void testCreateCollectionAddShardWithReplicaTypeUsingPolicy() throws Exception {
- JettySolrRunner jetty = cluster.getJettySolrRunners().get(0);
- String nrtNodeName = jetty.getNodeName();
- int nrtPort = jetty.getLocalPort();
-
- jetty = cluster.getJettySolrRunners().get(1);
- String pullNodeName = jetty.getNodeName();
- int pullPort = jetty.getLocalPort();
-
- jetty = cluster.getJettySolrRunners().get(2);
- String tlogNodeName = jetty.getNodeName();
- int tlogPort = jetty.getLocalPort();
- log.info("NRT {} PULL {} , TLOG {} ", nrtNodeName, pullNodeName, tlogNodeName);
-
- String commands = "{set-cluster-policy :[" +
- "{replica:0 , shard:'#EACH', type: NRT, port: '!" + nrtPort + "'}" +
- "{replica:0 , shard:'#EACH', type: PULL, port: '!" + pullPort + "'}" +
- "{replica:0 , shard:'#EACH', type: TLOG, port: '!" + tlogPort + "'}" +
- "]}";
-
-
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- Map<String, Object> json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH);
- assertEquals("full json:" + Utils.toJSONString(json), "!" + nrtPort,
- Utils.getObjectByPath(json, true, "cluster-policy[0]/port"));
- assertEquals("full json:" + Utils.toJSONString(json), "!" + pullPort,
- Utils.getObjectByPath(json, true, "cluster-policy[1]/port"));
- assertEquals("full json:" + Utils.toJSONString(json), "!" + tlogPort,
- Utils.getObjectByPath(json, true, "cluster-policy[2]/port"));
-
- final String collectionName = "addshard_with_reptype_using_policy";
- CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1", 1, 1, 1)
- .setMaxShardsPerNode(-1)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 1, 3);
-
- DocCollection coll = getCollectionState(collectionName);
-
-
- BiConsumer<String, Replica> verifyReplicas = (s, replica) -> {
- switch (replica.getType()) {
- case NRT: {
- assertTrue("NRT replica should be in " + nrtNodeName, replica.getNodeName().equals(nrtNodeName));
- break;
- }
- case TLOG: {
- assertTrue("TLOG replica should be in " + tlogNodeName, replica.getNodeName().equals(tlogNodeName));
- break;
- }
- case PULL: {
- assertTrue("PULL replica should be in " + pullNodeName, replica.getNodeName().equals(pullNodeName));
- break;
- }
- }
-
- };
- coll.forEachReplica(verifyReplicas);
-
- CollectionAdminRequest.createShard(collectionName, "s3").
- process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 2, 6);
-
- coll = getCollectionState(collectionName);
- assertEquals(3, coll.getSlice("s3").getReplicas().size());
- coll.forEachReplica(verifyReplicas);
- }
-
- public void testCreateCollectionAddShardUsingPolicy() throws Exception {
- JettySolrRunner jetty = cluster.getRandomJetty(random());
- int port = jetty.getLocalPort();
-
- String commands = "{set-policy :{c1 : [{replica:1 , shard:'#EACH', port: '" + port + "'}]}}";
- cluster.getSolrClient().request(AutoScalingRequest.create(SolrRequest.METHOD.POST, commands));
- Map<String, Object> json = Utils.getJson(cluster.getZkClient(), ZkStateReader.SOLR_AUTOSCALING_CONF_PATH);
- assertEquals("full json:"+ Utils.toJSONString(json) , "#EACH",
- Utils.getObjectByPath(json, true, "/policies/c1[0]/shard"));
-
- final String collectionName = "addshard_using_policy";
- CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "s1,s2", 1)
- .setPolicy("c1")
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 2, 2);
- DocCollection coll = getCollectionState(collectionName);
- assertEquals("c1", coll.getPolicyName());
- assertEquals(2,coll.getReplicas().size());
- coll.forEachReplica((s, replica) -> assertEquals(jetty.getNodeName(), replica.getNodeName()));
-
- CollectionAdminRequest.createShard(collectionName, "s3").process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 3, 3);
-
- coll = getCollectionState(collectionName);
- assertEquals(1, coll.getSlice("s3").getReplicas().size());
- coll.getSlice("s3").forEach(replica -> assertEquals(jetty.getNodeName(), replica.getNodeName()));
- }
-
- public void testDataProvider() throws Exception {
- final String collectionName = "data_provider";
- CollectionAdminRequest.createCollectionWithImplicitRouter(collectionName, "conf", "shard1", 2)
- .process(cluster.getSolrClient());
-
- cluster.waitForActiveCollection(collectionName, 1, 2);
-
- DocCollection rulesCollection = getCollectionState(collectionName);
-
- try (SolrCloudManager cloudManager = new SolrClientCloudManager(new ZkDistributedQueueFactory(cluster.getZkClient()), cluster.getSolrClient())) {
- Map<String, Object> val = cloudManager.getNodeStateProvider().getNodeValues(rulesCollection.getReplicas().get(0).getNodeName(), Arrays.asList(
- "freedisk",
- "cores",
- "host",
- "heapUsage",
- "sysLoadAvg"));
- assertNotNull(val.get("freedisk"));
- assertNotNull(val.get("host"));
- assertNotNull(val.get("heapUsage"));
- assertNotNull(val.get("sysLoadAvg"));
- assertTrue(((Number) val.get("cores")).intValue() > 0);
- assertTrue("freedisk value is " + ((Number) val.get("freedisk")).doubleValue(), Double.compare(((Number) val.get("freedisk")).doubleValue(), 0.0d) > 0);
- assertTrue("heapUsage value is " + ((Number) val.get("heapUsage")).doubleValue(), Double.compare(((Number) val.get("heapUsage")).doubleValue(), 0.0d) > 0);
- if (!Constants.WINDOWS) {
- // the system load average metrics is not available on windows platform
- assertTrue("sysLoadAvg value is " + ((Number) val.get("sysLoadAvg")).doubleValue(), Double.compare(((Number) val.get("sysLoadAvg")).doubleValue(), 0.0d) > 0);
- }
- String overseerNode = OverseerTaskProcessor.getLeaderNode(cluster.getZkClient());
- cluster.getSolrClient().request(CollectionAdminRequest.addRole(overseerNode, "overseer"));
- for (int i = 0; i < 10; i++) {
- Map<String, Object> data = Utils.getJson(cluster.getZkClient(), ZkStateReader.ROLES);
- if (i >= 9 && data.isEmpty()) {
- throw new RuntimeException("NO overseer node created");
- }
- Thread.sleep(100);
- }
- val = cloudManager.getNodeStateProvider().getNodeValues(overseerNode, Arrays.asList(
- "nodeRole",
- "ip_1", "ip_2", "ip_3", "ip_4",
- "sysprop.java.version",
- "sysprop.java.vendor"));
- assertEquals("overseer", val.get("nodeRole"));
- assertNotNull(val.get("ip_1"));
- assertNotNull(val.get("ip_2"));
- assertNotNull(val.get("ip_3"));
- assertNotNull(val.get("ip_4"));
- assertNotNull(val.get("sysprop.java.version"));
- assertNotNull(val.get("sysprop.java.vendor"));
- }
- }
-}
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
deleted file mode 100644
index 5b81aae..0000000
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/TriggerCooldownIntegrationTest.java
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.solr.cloud.autoscaling;
-
-import java.lang.invoke.MethodHandles;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.solr.client.solrj.SolrRequest;
-import org.apache.solr.client.solrj.cloud.SolrCloudManager;
-import org.apache.solr.client.solrj.cloud.autoscaling.AutoScalingConfig;
-import org.apache.solr.client.solrj.cloud.autoscaling.TriggerEventProcessorStage;
-import org.apache.solr.client.solrj.embedded.JettySolrRunner;
-import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
... 7356 lines suppressed ...