You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/09 15:04:45 UTC

[04/50] [abbrv] git commit: SLIDER-270 test to replicate the failure condition in the slider-hbase-provider

SLIDER-270 test to replicate the failure condition in the slider-hbase-provider


Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86

Branch: refs/heads/feature/SLIDER-151_REST_API
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 18:10:59 2014 +0100

----------------------------------------------------------------------
 .../minicluster/HBaseMiniClusterTestBase.groovy | 149 ++++++++++++-------
 .../flexing/TestClusterFlex2To5.groovy          |   2 +-
 .../flexing/TestClusterFlexDownMultiple.groovy  |  58 ++++++++
 3 files changed, 151 insertions(+), 58 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
 import org.apache.slider.api.ClusterDescription
 import org.apache.slider.api.ClusterNode
 import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
 import org.apache.slider.client.SliderClient
 import org.apache.slider.core.main.ServiceLauncher
 import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
         (ROLE_MASTER): masters,
         (ROLE_WORKER): workers,
     ];
-    extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM
-    extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY << YRAM
-    extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+    hbaseArgs(extraArgs)
     
     return createCluster(clustername,
         roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
         [:])
   }
 
+  public List<String> hbaseArgs(List<String> extraArgs) {
+    extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+    YRAM
+    extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+    YRAM
+    extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+    return extraArgs;
+  }
+
   /**
    * Create an AM without a master
    * @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
     ];
     return createCluster(clustername,
         roles,
-        [
-            ARG_PROVIDER, PROVIDER_HBASE
-        ],
+        hbaseArgs([]),
         deleteExistingData,
         blockUntilRunning,
         [:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
       int masters,
       int masterFlexTarget,
       int workers,
-      int flexTarget,
+      int workerFlexTarget,
       boolean testHBaseAfter) {
-    createMiniCluster(clustername, configuration,
-                      1,
-                      true);
+    SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
+
+    //now flex
+    return flexCluster(
+        sliderClient,
+        clustername,
+        masterFlexTarget,
+        workerFlexTarget,
+        testHBaseAfter)
+
+  }
+
+  public SliderClient startHBaseCluster(
+      String clustername,
+      int masters,
+      int workers) {
+    clustername = createMiniCluster(clustername, configuration,
+        1,
+        true);
     //now launch the cluster
     SliderClient sliderClient;
     ServiceLauncher<SliderClient> launcher = createCluster(clustername,
-           [
-               (ROLE_MASTER): masters,
-               (ROLE_WORKER): workers,
-           ],
-           [
-               ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
-               ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
-               ARG_PROVIDER , PROVIDER_HBASE
-           ],
-           true,
-           true,
-           [:]);
+        [
+            (ROLE_MASTER): masters,
+            (ROLE_WORKER): workers,
+        ],
+        hbaseArgs([]),
+        true,
+        true,
+        [:]);
     sliderClient = launcher.service;
-    try {
-      basicHBaseClusterStartupSequence(sliderClient);
-
-      describe("Waiting for initial worker count of $workers");
-
-      //verify the #of roles is as expected
-      //get the hbase status
-      waitForWorkerInstanceCount(sliderClient, workers, hbaseClusterStartupToLiveTime);
-      waitForSliderMasterCount(sliderClient, masters, hbaseClusterStartupToLiveTime);
-
-      log.info("Slider worker count at $workers, waiting for region servers to match");
-      waitForHBaseRegionServerCount(sliderClient, clustername, workers, hbaseClusterStartupToLiveTime);
-
-      //now flex
-      describe("Flexing  masters:$masters -> $masterFlexTarget ; workers $workers -> $flexTarget");
-      boolean flexed;
-      flexed = 0 == sliderClient.flex(clustername,
-          [
-              (ROLE_WORKER): flexTarget,
-              (ROLE_MASTER): masterFlexTarget
-          ]
-      );
-      waitForWorkerInstanceCount(sliderClient, flexTarget, hbaseClusterStartupToLiveTime);
-      waitForSliderMasterCount(sliderClient, masterFlexTarget,
-                             hbaseClusterStartupToLiveTime);
-
-      if (testHBaseAfter) {
-        waitForHBaseRegionServerCount(sliderClient, clustername, flexTarget,
-                                      hbaseClusterStartupToLiveTime);
-      }
-      return flexed;
-    } finally {
-      maybeStopCluster(sliderClient, null, "end of flex test run");
-    }
 
+    basicHBaseClusterStartupSequence(sliderClient);
+
+    describe("Waiting for initial worker count of $workers");
+
+    //verify the #of roles is as expected
+    //get the hbase status
+    waitForWorkerInstanceCount(
+        sliderClient,
+        workers,
+        hbaseClusterStartupToLiveTime);
+    waitForSliderMasterCount(
+        sliderClient,
+        masters,
+        hbaseClusterStartupToLiveTime);
+
+    log.info(
+        "Slider worker count at $workers, waiting for region servers to match");
+    waitForHBaseRegionServerCount(
+        sliderClient,
+        clustername,
+        workers,
+        hbaseClusterStartupToLiveTime);
+    sliderClient
+  }
+
+  public boolean flexCluster(
+      SliderClient sliderClient,
+      String clustername,
+      int masterFlexTarget,
+      int workerFlexTarget,
+      boolean testHBaseAfter) {
+    int flexTarget
+    describe(
+        "Flexing  masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
+    boolean flexed;
+    flexed = 0 == sliderClient.flex(clustername,
+        [
+            (ROLE_WORKER): workerFlexTarget,
+            (ROLE_MASTER): masterFlexTarget
+        ]
+    );
+    waitForWorkerInstanceCount(
+        sliderClient,
+        workerFlexTarget,
+        hbaseClusterStartupToLiveTime);
+    waitForSliderMasterCount(sliderClient, masterFlexTarget,
+        hbaseClusterStartupToLiveTime);
+
+    if (testHBaseAfter) {
+      waitForHBaseRegionServerCount(sliderClient, clustername, workerFlexTarget,
+          hbaseClusterStartupToLiveTime);
+    }
+    flexed
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
index 680286c..683c02a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
@@ -31,7 +31,7 @@ import org.junit.Test
 class TestClusterFlex2To5 extends HBaseMiniClusterTestBase {
 
   @Test
-  public void testClusterFlex1To5() throws Throwable {
+  public void testClusterFlex2To5() throws Throwable {
     assert flexHBaseClusterTestRun(
         "",
         1,

http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
new file mode 100644
index 0000000..43ade73
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
+
+  @Test
+  public void testClusterFlexDownMultiple() throws Throwable {
+    def clusterName = createClusterName();
+    SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+    
+    assert flexCluster(
+        sliderClient,
+        clusterName,
+        1,
+        3,
+        true)
+
+    assert flexCluster(
+        sliderClient,
+        clusterName,
+        1,
+        1,
+        true)
+    
+
+    
+  }
+
+
+}