You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/09 15:04:45 UTC
[04/50] [abbrv] git commit: SLIDER-270 test to replicate the failure
condition in the slider-hbase-provider
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 18:10:59 2014 +0100
----------------------------------------------------------------------
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ++++++++++++-------
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 ++++++++
3 files changed, 151 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
- extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
[:])
}
+ public List<String> hbaseArgs(List<String> extraArgs) {
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
- [
- ARG_PROVIDER, PROVIDER_HBASE
- ],
+ hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
- createMiniCluster(clustername, configuration,
- 1,
- true);
+ SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
+
+ //now flex
+ return flexCluster(
+ sliderClient,
+ clustername,
+ masterFlexTarget,
+ workerFlexTarget,
+ testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+ clustername = createMiniCluster(clustername, configuration,
+ 1,
+ true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher<SliderClient> launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true,
- [:]);
+ [
+ (ROLE_MASTER): masters,
+ (ROLE_WORKER): workers,
+ ],
+ hbaseArgs([]),
+ true,
+ true,
+ [:]);
sliderClient = launcher.service;
- try {
- basicHBaseClusterStartupSequence(sliderClient);
-
- describe("Waiting for initial worker count of $workers");
-
- //verify the #of roles is as expected
- //get the hbase status
- waitForWorkerInstanceCount(sliderClient, workers, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masters, hbaseClusterStartupToLiveTime);
-
- log.info("Slider worker count at $workers, waiting for region servers to match");
- waitForHBaseRegionServerCount(sliderClient, clustername, workers, hbaseClusterStartupToLiveTime);
-
- //now flex
- describe("Flexing masters:$masters -> $masterFlexTarget ; workers $workers -> $flexTarget");
- boolean flexed;
- flexed = 0 == sliderClient.flex(clustername,
- [
- (ROLE_WORKER): flexTarget,
- (ROLE_MASTER): masterFlexTarget
- ]
- );
- waitForWorkerInstanceCount(sliderClient, flexTarget, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masterFlexTarget,
- hbaseClusterStartupToLiveTime);
-
- if (testHBaseAfter) {
- waitForHBaseRegionServerCount(sliderClient, clustername, flexTarget,
- hbaseClusterStartupToLiveTime);
- }
- return flexed;
- } finally {
- maybeStopCluster(sliderClient, null, "end of flex test run");
- }
+ basicHBaseClusterStartupSequence(sliderClient);
+
+ describe("Waiting for initial worker count of $workers");
+
+ //verify the #of roles is as expected
+ //get the hbase status
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(
+ sliderClient,
+ masters,
+ hbaseClusterStartupToLiveTime);
+
+ log.info(
+ "Slider worker count at $workers, waiting for region servers to match");
+ waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ sliderClient
+ }
+
+ public boolean flexCluster(
+ SliderClient sliderClient,
+ String clustername,
+ int masterFlexTarget,
+ int workerFlexTarget,
+ boolean testHBaseAfter) {
+ int flexTarget
+ describe(
+ "Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
+ boolean flexed;
+ flexed = 0 == sliderClient.flex(clustername,
+ [
+ (ROLE_WORKER): workerFlexTarget,
+ (ROLE_MASTER): masterFlexTarget
+ ]
+ );
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(sliderClient, masterFlexTarget,
+ hbaseClusterStartupToLiveTime);
+
+ if (testHBaseAfter) {
+ waitForHBaseRegionServerCount(sliderClient, clustername, workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ }
+ flexed
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
index 680286c..683c02a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
@@ -31,7 +31,7 @@ import org.junit.Test
class TestClusterFlex2To5 extends HBaseMiniClusterTestBase {
@Test
- public void testClusterFlex1To5() throws Throwable {
+ public void testClusterFlex2To5() throws Throwable {
assert flexHBaseClusterTestRun(
"",
1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
new file mode 100644
index 0000000..43ade73
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlexDownMultiple() throws Throwable {
+ def clusterName = createClusterName();
+ SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 3,
+ true)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 1,
+ true)
+
+
+
+ }
+
+
+}