You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2014/07/17 02:23:45 UTC
[2/7] SLIDER-241 rename Tests to ITs
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloClusterIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloClusterIT.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloClusterIT.groovy
new file mode 100644
index 0000000..79dd611
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloClusterIT.groovy
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.accumulo.funtest
+
+import static org.apache.slider.providers.accumulo.AccumuloConfigFileOptions.*
+import static org.apache.slider.providers.accumulo.AccumuloKeys.*
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.client.SliderClient
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+/**
+ *
+ */
+@CompileStatic
+@Slf4j
+class FunctionalAccumuloClusterIT extends AccumuloCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ public String getClusterName() {
+ return "test_functional_accumulo_cluster"
+ }
+
+ public String getPassword() {
+ return "password";
+ }
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(getClusterName())
+ }
+
+ @After
+ public void destroyCluster() {
+ teardown(getClusterName())
+ }
+
+ public int getNumMasters() {
+ return 1
+ }
+
+ public int getNumTservers() {
+ return 1
+ }
+
+ public int getNumMonitors() {
+ return 1
+ }
+
+ public int getNumGarbageCollectors() {
+ return 1
+ }
+
+ public int getNumTracers() {
+ return 0
+ }
+
+ public int getMonitorPort() {
+ return 0
+ }
+
+ @Test
+ public void testAccumuloClusterCreate() throws Throwable {
+
+ describe "Create a working Accumulo cluster"
+
+ def path = buildClusterPath(getClusterName())
+ assert !clusterFS.exists(path)
+
+ Map<String, Integer> roleMap = [
+ (ROLE_MASTER) : getNumMasters(),
+ (ROLE_TABLET) : getNumTservers(),
+ (ROLE_MONITOR): getNumMonitors(),
+ (ROLE_GARBAGE_COLLECTOR): getNumGarbageCollectors(),
+ (ROLE_TRACER) : getNumTracers()
+ ];
+
+ Map<String, String> clusterOps = [:]
+ clusterOps["site." + MONITOR_PORT_CLIENT] = Integer.toString(getMonitorPort())
+
+ List<String> extraArgs = []
+
+ createAccumuloCluster(
+ getClusterName(),
+ roleMap,
+ extraArgs,
+ true,
+ clusterOps,
+ "256",
+ getPassword()
+ )
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, getClusterName())
+ ClusterDescription cd = sliderClient.clusterDescription
+ assert getClusterName() == cd.name
+
+ log.info("Connected via Client {}", sliderClient.toString())
+
+ //wait for the role counts to be reached
+ waitForRoleCount(sliderClient, roleMap, ACCUMULO_LAUNCH_WAIT_TIME)
+
+ clusterLoadOperations(clusterName, roleMap, cd)
+ }
+
+
+ public String getDescription() {
+ return "Create a working Accumulo cluster $clusterName"
+ }
+
+ /**
+ * Override point for any cluster load operations
+ * @param clientConf
+ * @param numWorkers
+ */
+ public void clusterLoadOperations(
+ String clustername,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ log.info("Client Description = " + cd.toJsonString())
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloM1T1GC1Mon1IT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloM1T1GC1Mon1IT.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloM1T1GC1Mon1IT.groovy
new file mode 100644
index 0000000..865eacc
--- /dev/null
+++ b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/FunctionalAccumuloM1T1GC1Mon1IT.groovy
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.accumulo.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+
+@CompileStatic
+@Slf4j
+public class FunctionalAccumuloM1T1GC1Mon1IT extends FunctionalAccumuloClusterIT {
+
+ @Override
+ public String getClusterName() {
+ return "test_functional_accumulo_m1t1gc1mon1";
+ }
+
+ /**
+ * Override point for any cluster load operations
+ * @param clientConf
+ * @param numWorkers
+ */
+ @Override
+ public void clusterLoadOperations(
+ String clustername,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ slider(0, [
+ SliderActions.ACTION_FREEZE,
+ getClusterName(),
+ Arguments.ARG_WAIT,
+ Integer.toString(FREEZE_WAIT_TIME),
+ Arguments.ARG_MESSAGE,
+ "freeze-in-test-AccumuloCluster"
+ ])
+
+ //destroy the cluster. This only works if the permissions allow it
+ destroy(0, getClusterName())
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy
deleted file mode 100644
index 61366da..0000000
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloBuildSetup.groovy
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.accumulo.funtest
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
-import org.junit.Test
-
-class TestAccumuloBuildSetup extends AbstractTestBuildSetup {
-
- @Test
- public void testAccumuloBuildsHavePathsDefined() throws Throwable {
- Configuration conf = loadSliderConf();
- assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
-
- assumeBoolOption(conf, KEY_TEST_ACCUMULO_ENABLED, true)
-
- assertStringOptionSet(conf, KEY_TEST_ACCUMULO_APPCONF)
- assertStringOptionSet(conf, KEY_TEST_ACCUMULO_TAR)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy
deleted file mode 100644
index 5573dd4..0000000
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestAccumuloCI.groovy
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.accumulo.funtest
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-
-import org.apache.accumulo.core.client.Connector
-import org.apache.accumulo.core.client.ZooKeeperInstance
-import org.apache.accumulo.core.client.security.tokens.PasswordToken
-import org.apache.accumulo.test.continuous.ContinuousIngest
-import org.apache.accumulo.test.continuous.ContinuousVerify
-import org.apache.hadoop.fs.Path
-import org.apache.hadoop.io.Text
-import org.apache.hadoop.util.ToolRunner
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.funtest.framework.CommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.funtest.framework.PortAssignments
-
-/**
- *
- */
-@CompileStatic
-@Slf4j
-class TestAccumuloCI extends TestFunctionalAccumuloCluster {
-
- @Override
- String getClusterName() {
- return "test_accumulo_ci"
- }
-
- @Override
- public int getNumTservers() {
- return 2;
- }
-
- @Override
- public int getMonitorPort() {
- return PortAssignments._testAccumuloCI;
- }
-
- @Override
- void clusterLoadOperations(
- String clustername,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
- assert clustername
-
- String currentUser = System.getProperty("user.name");
- String zookeepers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM,
- FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
- ZooKeeperInstance inst = new ZooKeeperInstance(currentUser + "-" + clustername, zookeepers)
- PasswordToken passwd = new PasswordToken(getPassword())
- Connector conn = inst.getConnector("root", new PasswordToken(getPassword()))
-
- // Create the test table with some split points
- String tableName = "testAccumuloCi";
- conn.tableOperations().create(tableName)
- TreeSet<Text> splits = new TreeSet<Text>()
- splits.add(new Text("2"))
- splits.add(new Text("5"))
- splits.add(new Text("7"))
- conn.tableOperations().addSplits(tableName, splits)
-
- // Write 15M records per tserver -- should take a few minutes
- String[] ciOpts = ["-i", inst.getInstanceName(),
- "-z", zookeepers, "-u", "root",
- "-p", getPassword(), "--table", tableName,
- "--num", Integer.toString(1000 * 1000 * 15 * getNumTservers()),
- "--batchMemory", "100000000",
- "--batchLatency", "600000",
- "--batchThreads", "1"]
-
- ContinuousIngest ci = new ContinuousIngest();
- ci.main(ciOpts);
-
- // Create a directory for the verify to write its output to
- Path verifyOutput = new Path("/user/" + currentUser + "/.slider/cluster/" + clustername + "/verify-output")
- assert !clusterFS.exists(verifyOutput)
-
- YarnConfiguration verifyConf = new YarnConfiguration(CommandTestBase.SLIDER_CONFIG);
-
- // Try to load the necessary classes for the Mappers to find them
- if (loadClassesForMapReduce(verifyConf)) {
- // If we found those classes, try to run in distributed mode.
- tryToLoadMapredSite(verifyConf)
- }
-
- // Run ContinuousVerify and ensure that no holes exist
- ContinuousVerify verify = new ContinuousVerify();
- String[] verifyOpts = ["-i", inst.getInstanceName(),
- "-z", zookeepers, "-u", "root",
- "-p", getPassword(), "--table", tableName,
- "--output", verifyOutput.toString(), "--maxMappers", Integer.toString(2 * getNumTservers()),
- "--reducers", Integer.toString(getNumTservers())]
- assert 0 == ToolRunner.run(verifyConf, verify, verifyOpts)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy
deleted file mode 100644
index 06fe21c..0000000
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloCluster.groovy
+++ /dev/null
@@ -1,146 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.accumulo.funtest
-
-import static org.apache.slider.providers.accumulo.AccumuloConfigFileOptions.*
-import static org.apache.slider.providers.accumulo.AccumuloKeys.*
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.client.SliderClient
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
-/**
- *
- */
-@CompileStatic
-@Slf4j
-class TestFunctionalAccumuloCluster extends AccumuloCommandTestBase
- implements FuntestProperties, Arguments, SliderExitCodes {
-
-
- public String getClusterName() {
- return "test_functional_accumulo_cluster"
- }
-
- public String getPassword() {
- return "password";
- }
-
- @Before
- public void prepareCluster() {
- setupCluster(getClusterName())
- }
-
- @After
- public void destroyCluster() {
- teardown(getClusterName())
- }
-
- public int getNumMasters() {
- return 1
- }
-
- public int getNumTservers() {
- return 1
- }
-
- public int getNumMonitors() {
- return 1
- }
-
- public int getNumGarbageCollectors() {
- return 1
- }
-
- public int getNumTracers() {
- return 0
- }
-
- public int getMonitorPort() {
- return 0
- }
-
- @Test
- public void testAccumuloClusterCreate() throws Throwable {
-
- describe "Create a working Accumulo cluster"
-
- def path = buildClusterPath(getClusterName())
- assert !clusterFS.exists(path)
-
- Map<String, Integer> roleMap = [
- (ROLE_MASTER) : getNumMasters(),
- (ROLE_TABLET) : getNumTservers(),
- (ROLE_MONITOR): getNumMonitors(),
- (ROLE_GARBAGE_COLLECTOR): getNumGarbageCollectors(),
- (ROLE_TRACER) : getNumTracers()
- ];
-
- Map<String, String> clusterOps = [:]
- clusterOps["site." + MONITOR_PORT_CLIENT] = Integer.toString(getMonitorPort())
-
- List<String> extraArgs = []
-
- createAccumuloCluster(
- getClusterName(),
- roleMap,
- extraArgs,
- true,
- clusterOps,
- "256",
- getPassword()
- )
-
- //get a slider client against the cluster
- SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, getClusterName())
- ClusterDescription cd = sliderClient.clusterDescription
- assert getClusterName() == cd.name
-
- log.info("Connected via Client {}", sliderClient.toString())
-
- //wait for the role counts to be reached
- waitForRoleCount(sliderClient, roleMap, ACCUMULO_LAUNCH_WAIT_TIME)
-
- clusterLoadOperations(clusterName, roleMap, cd)
- }
-
-
- public String getDescription() {
- return "Create a working Accumulo cluster $clusterName"
- }
-
- /**
- * Override point for any cluster load operations
- * @param clientConf
- * @param numWorkers
- */
- public void clusterLoadOperations(
- String clustername,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
-
- log.info("Client Description = " + cd.toJsonString())
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy b/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy
deleted file mode 100644
index 4f07b40..0000000
--- a/slider-providers/accumulo/accumulo-funtests/src/test/groovy/org/apache/slider/providers/accumulo/funtest/TestFunctionalAccumuloM1T1GC1Mon1.groovy
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.accumulo.funtest
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-
-@CompileStatic
-@Slf4j
-public class TestFunctionalAccumuloM1T1GC1Mon1 extends TestFunctionalAccumuloCluster {
-
- @Override
- public String getClusterName() {
- return "test_functional_accumulo_m1t1gc1mon1";
- }
-
- /**
- * Override point for any cluster load operations
- * @param clientConf
- * @param numWorkers
- */
- @Override
- public void clusterLoadOperations(
- String clustername,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
-
- slider(0, [
- SliderActions.ACTION_FREEZE,
- getClusterName(),
- Arguments.ARG_WAIT,
- Integer.toString(FREEZE_WAIT_TIME),
- Arguments.ARG_MESSAGE,
- "freeze-in-test-AccumuloCluster"
- ])
-
- //destroy the cluster. This only works if the permissions allow it
- destroy(0, getClusterName())
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
new file mode 100644
index 0000000..84e55f7
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/FunctionalHBaseClusterIT.groovy
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.tools.ConfigHelper
+import org.apache.slider.core.registry.info.RegistryNaming
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+import org.apache.slider.providers.hbase.HBaseTestUtils
+import org.apache.slider.server.appmaster.PublishedArtifacts
+import org.apache.zookeeper.KeeperException
+import org.apache.zookeeper.WatchedEvent
+import org.apache.zookeeper.Watcher
+import org.apache.zookeeper.ZKUtil
+import org.apache.zookeeper.ZooKeeper
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_MASTER
+import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
+
+@CompileStatic
+@Slf4j
+public class FunctionalHBaseClusterIT extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ public static final String HBASE_HEAP = "96m"
+
+ public String getClusterName() {
+ return "test_functional_hbase_cluster"
+ }
+
+ public String getClusterZNode() {
+ return "/yarnapps_slider_yarn_" + clusterName;
+ }
+
+ @Before
+ public void prepareCluster() {
+
+ String quorumServers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
+
+ ZooKeeper monitor = new ZooKeeper(quorumServers,
+ 1000, new Watcher(){
+ @Override
+ public void process(WatchedEvent watchedEvent) {
+ }
+ }, false)
+ try {
+ ZKUtil.deleteRecursive(monitor, clusterZNode)
+ } catch (KeeperException.NoNodeException ignored) {
+ log.info(clusterZNode + " not there")
+ }
+ setupCluster(clusterName)
+ }
+
+ @After
+ public void teardownCluster() {
+ teardown(clusterName)
+ }
+
+ @Test
+ public void testHBaseCreateCluster() throws Throwable {
+
+ describe description
+
+ int numWorkers = desiredWorkerCount;
+
+ def clusterpath = buildClusterPath(clusterName)
+ assert !clusterFS.exists(clusterpath)
+ Map<String, Integer> roleMap = createHBaseCluster(
+ clusterName,
+ 1, numWorkers,
+ [
+ ARG_OPTION,
+ HBaseConfigFileOptions.KEY_HBASE_MASTER_INFO_PORT,
+ Integer.toString(masterPortAssignment),
+ ARG_COMP_OPT, ROLE_MASTER, RoleKeys.JVM_HEAP, HBASE_HEAP,
+ ARG_OPTION,
+ HBaseConfigFileOptions.KEY_REGIONSERVER_PORT,
+ Integer.toString(workerPortAssignment),
+ ARG_COMP_OPT, ROLE_WORKER, RoleKeys.JVM_HEAP, HBASE_HEAP,
+ ],
+ [:]
+ )
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, clusterName)
+ ClusterDescription cd2 = sliderClient.clusterDescription
+ assert clusterName == cd2.name
+
+ log.info("Connected via Client {} with {} workers", sliderClient.toString(),
+ numWorkers)
+
+ //wait for the role counts to be reached
+ waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+
+ Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(sliderClient)
+ HBaseTestUtils.assertHBaseMasterFound(clientConf)
+ HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+
+ clusterOperations(
+ clusterName,
+ sliderClient,
+ clientConf,
+ numWorkers,
+ roleMap,
+ cd2)
+ }
+
+ /**
+ * Override to change policy of the deired no of workers
+ * @return
+ */
+ def int getDesiredWorkerCount() {
+ return SLIDER_CONFIG.getInt(KEY_SLIDER_TEST_NUM_WORKERS,
+ DEFAULT_SLIDER_NUM_WORKERS)
+ }
+
+
+ public String getDescription() {
+ return "Create a working HBase cluster $clusterName"
+ }
+
+ /**
+ * Override point for any cluster operations
+ * @param clustername name of cluster
+ * @param sliderClient bonded low level client
+ * @param clientConf config
+ * @param numWorkers no. of workers created
+ * @param roleMap role map
+ * @param cd current cluster
+ */
+ public void clusterOperations(
+ String clustername,
+ SliderClient sliderClient,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+ log.info("Client Configuration = " + ConfigHelper.dumpConfigToString(clientConf))
+
+ //grab some registry bits
+ registry([ARG_LIST])
+ registry([ARG_LIST, ARG_SERVICETYPE, SliderKeys.APP_TYPE , ARG_VERBOSE])
+
+ //unknown service type
+ registry(EXIT_NOT_FOUND,
+ [ARG_LIST, ARG_SERVICETYPE, "org.apache.something"])
+
+ registry(EXIT_NOT_FOUND,
+ [ARG_LIST, ARG_NAME, "cluster-with-no-name"])
+
+ // how to work out the current service name?
+ def name = RegistryNaming.createRegistryName(clustername,
+ System.getProperty("user.name"),
+ SliderKeys.APP_TYPE,
+ 1)
+ registry([ARG_LIST, ARG_VERBOSE, ARG_NAME, name])
+
+ registry([ARG_LISTCONF, ARG_NAME, name])
+ registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, name, ARG_INTERNAL])
+ registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, "unknown"])
+ registry([ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
+ ARG_NAME, name])
+ registry(EXIT_NOT_FOUND, [ARG_GETCONF, "no-such-config",
+ ARG_NAME, name])
+
+ registry(EXIT_NOT_FOUND, [ARG_GETCONF, "illegal/config/name!",
+ ARG_NAME, name])
+
+ registry(EXIT_NOT_FOUND,[ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
+ ARG_NAME, name, ARG_INTERNAL])
+
+
+ def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
+ registry([ARG_GETCONF, yarn_site_config,
+ ARG_NAME, name])
+
+ File getConfDir = new File("target/$clusterName/getconf")
+ getConfDir.mkdirs();
+ registry([ARG_GETCONF, yarn_site_config,
+ ARG_NAME, name,
+ ARG_DEST, getConfDir.absolutePath])
+ File retrieved = new File(getConfDir, yarn_site_config +".xml")
+ def confFromFile = ConfigHelper.loadConfFromFile(retrieved)
+ assert confFromFile.get(YarnConfiguration.RM_ADDRESS)
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseBuildSetupIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseBuildSetupIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseBuildSetupIT.groovy
new file mode 100644
index 0000000..0dcffde
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseBuildSetupIT.groovy
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
+import org.junit.Test
+
+class HBaseBuildSetupIT extends AbstractTestBuildSetup {
+
+ @Test
+ public void testHBaseBuildsHavePathsDefined() throws Throwable {
+ Configuration conf = loadSliderConf();
+
+ assumeBoolOption(conf, KEY_TEST_HBASE_ENABLED, true)
+
+ assertStringOptionSet(conf, KEY_TEST_HBASE_APPCONF)
+ assertStringOptionSet(conf, KEY_TEST_HBASE_TAR)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
new file mode 100644
index 0000000..3a44e30
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterBuildDestroyIT.groovy
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.junit.AfterClass
+import org.junit.BeforeClass
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class HBaseClusterBuildDestroyIT extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments {
+
+
+ static String CLUSTER = "test_cluster_build_destroy"
+
+
+ @BeforeClass
+ public static void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @AfterClass
+ public static void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testBuildAndDestroyCluster() throws Throwable {
+ def clusterDir = SliderKeys.SLIDER_BASE_DIRECTORY + "/cluster/$CLUSTER"
+ def clusterDirPath = new Path(clusterFS.homeDirectory, clusterDir)
+ clusterFS.delete(clusterDirPath, true)
+ slider(0,
+ [
+ SliderActions.ACTION_BUILD,
+ CLUSTER,
+ ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
+ ARG_ZKHOSTS,
+ SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
+ ARG_IMAGE,
+ SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
+ ARG_CONFDIR,
+ SLIDER_CONFIG.get(KEY_TEST_HBASE_APPCONF),
+ ARG_COMPONENT, HBaseKeys.ROLE_MASTER, "1",
+ ARG_COMPONENT, HBaseKeys.ROLE_WORKER, "1",
+ ARG_OPTION, "site.hbase.master.info.port", "8180",
+ ])
+
+
+
+ assert clusterFS.exists(clusterDirPath)
+ //cluster exists if you don't want it to be live
+ exists(0, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(LauncherExitCodes.EXIT_FALSE, CLUSTER, true)
+ destroy(CLUSTER)
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
new file mode 100644
index 0000000..aca12ef
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseClusterLifecycleIT.groovy
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class HBaseClusterLifecycleIT extends HBaseCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes {
+
+
+ static String CLUSTER = "test_cluster_lifecycle"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ teardown(CLUSTER)
+ }
+
+ @Test
+ public void testClusterLifecycle() throws Throwable {
+
+ describe "Walk a 0-role cluster through its lifecycle"
+
+
+ def clusterpath = buildClusterPath(CLUSTER)
+ assert !clusterFS.exists(clusterpath)
+
+
+ Map<String, Integer> roleMap = createHBaseCluster(CLUSTER,
+ 0,
+ 0,
+ [],
+ [:])
+
+
+ //at this point the cluster should exist.
+ assertPathExists(clusterFS,"Cluster parent directory does not exist", clusterpath.parent)
+
+ assertPathExists(clusterFS,"Cluster directory does not exist", clusterpath)
+
+ // assert it exists on the command line
+ exists(0, CLUSTER)
+
+ //destroy will fail in use
+
+ destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //thaw will fail as cluster is in use
+ thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
+
+ //it's still there
+ exists(0, CLUSTER)
+
+ //listing the cluster will succeed
+ list(0, CLUSTER)
+
+ //simple status
+ status(0, CLUSTER)
+
+ //now status to a temp file
+ File jsonStatus = File.createTempFile("tempfile", ".json")
+ try {
+ slider(0,
+ [
+ SliderActions.ACTION_STATUS, CLUSTER,
+ ARG_OUTPUT, jsonStatus.canonicalPath
+ ])
+
+ assert jsonStatus.exists()
+ ClusterDescription cd = ClusterDescription.fromFile(jsonStatus)
+
+ assert CLUSTER == cd.name
+
+ log.info(cd.toJsonString())
+
+ getConf(0, CLUSTER)
+
+ //get a slider client against the cluster
+ SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
+ ClusterDescription cd2 = sliderClient.clusterDescription
+ assert CLUSTER == cd2.name
+
+ log.info("Connected via Client {}", sliderClient.toString())
+
+ //freeze
+ freeze(CLUSTER, [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "freeze-in-test cluster lifecycle"
+ ])
+
+ //cluster exists if you don't want it to be live
+ exists(0, CLUSTER, false)
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+
+ // thaw then freeze the cluster
+
+ thaw(CLUSTER,
+ [
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ])
+ exists(0, CLUSTER)
+ freeze(CLUSTER,
+ [
+ ARG_FORCE,
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "forced-freeze-in-test"
+ ])
+
+ //cluster is no longer live
+ exists(0, CLUSTER, false)
+
+ // condition returns false if it is required to be live
+ exists(EXIT_FALSE, CLUSTER, true)
+
+ // thaw with a restart count set to enable restart
+
+ describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
+ thaw(CLUSTER,
+ [
+ ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
+ ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
+ ])
+
+
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
+
+ def restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 0
+ freeze(CLUSTER)
+
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+
+ } finally {
+ jsonStatus.delete()
+ }
+
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
new file mode 100644
index 0000000..69da02a
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseIntegrationIT.groovy
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.IntegrationTestIngest
+import org.apache.hadoop.hbase.IntegrationTestingUtility
+import org.apache.hadoop.util.ToolRunner
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions;
+
+/* Runs IntegrationTestIngest on cluster
+ *
+ * Note: this test runs for about 20 minutes
+ * please set slider.test.timeout.seconds accordingly
+ */
+class HBaseIntegrationIT extends FunctionalHBaseClusterIT {
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_integration"
+ }
+
+ @Override
+ void clusterOperations(
+ String clustername,
+ SliderClient sliderClient,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ String parent = "/yarnapps_slider_yarn_" + clustername
+ clientConf.set(HBaseConfigFileOptions.KEY_ZNODE_PARENT, parent)
+
+ clientConf.set(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "true")
+
+ String[] args = []
+ IntegrationTestIngest test = new IntegrationTestIngest();
+ test.setConf(clientConf)
+ int ret = ToolRunner.run(clientConf, test, args);
+ assert ret == 0;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
new file mode 100644
index 0000000..7393186
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseLoadIT.groovy
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.hbase.util.LoadTestTool
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.HBaseConfigFileOptions
+
+class HBaseLoadIT extends FunctionalHBaseClusterIT {
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_load"
+ }
+
+ @Override
+ void clusterOperations(
+ String clustername,
+ SliderClient sliderClient,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+ assert clustername
+ int numKeys = 4000 * numWorkers
+ String[] args = ["-tn", "test", "-write", "4:100",
+ "-num_keys", numKeys,
+ "-zk", clientConf.get(HBaseConfigFileOptions.KEY_ZOOKEEPER_QUORUM),
+ "-zk_root", clientConf.get(HBaseConfigFileOptions.KEY_ZNODE_PARENT),
+
+ ]
+ LoadTestTool loadTool = new LoadTestTool();
+ loadTool.setConf(clientConf)
+ int ret = loadTool.run(args);
+ assert ret == 0;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy
new file mode 100644
index 0000000..cd87fab
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/HBaseNodeFailureIT.groovy
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.params.ActionKillContainerArgs
+import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.providers.hbase.HBaseTestUtils
+
+class HBaseNodeFailureIT extends FunctionalHBaseClusterIT {
+
+
+ public static final int RESTART_SLEEP_TIME = 5000
+
+ @Override
+ String getClusterName() {
+ return "test_hbase_node_failure"
+ }
+
+ @Override
+ String getDescription() {
+ "Fail containers and verify that the cluster recovers"
+ }
+
+ @Override
+ void clusterOperations(
+ String clustername,
+ SliderClient sliderClient,
+ Configuration clientConf,
+ int numWorkers,
+ Map<String, Integer> roleMap,
+ ClusterDescription cd) {
+
+
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+
+ //wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+ // then expect a restart
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+ // then expect a restart
+
+ //wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "2"
+
+ killInstanceOfRole(sliderClient, HBaseKeys.ROLE_MASTER)
+ // let it take
+ sleep(RESTART_SLEEP_TIME)
+
+ // wait for the role counts to be reached
+ cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
+ HBaseTestUtils.waitForHBaseRegionServerCount(
+ sliderClient,
+ clusterName,
+ numWorkers,
+ HBASE_LAUNCH_WAIT_TIME)
+ assert cd.roles[HBaseKeys.ROLE_MASTER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
+
+ // now trigger AM failure
+ ClusterDescription status = killAmAndWaitForRestart(sliderClient, clusterName)
+
+ def restarted = status.getInfo(
+ StatusKeys.INFO_CONTAINERS_AM_RESTART)
+ assert restarted != null
+ assert Integer.parseInt(restarted) == 1 + numWorkers
+
+ }
+
+ /**
+ * Kill a random in instance of a role in the cluster
+ * @param sliderClient client
+ * @param role
+ * @return ID of container killed
+ */
+ public String killInstanceOfRole(
+ SliderClient sliderClient, String role) {
+ ClusterDescription cd = sliderClient.getClusterDescription()
+ def instances = cd.instances[role]
+ if (instances == null || instances.size() == 0) {
+ log.info("No instances of role $role to kill")
+ return null;
+ }
+ String id = instances[new Random().nextInt(instances.size())]
+ ActionKillContainerArgs args = new ActionKillContainerArgs()
+ args.id = id
+ sliderClient.actionKillContainer(clusterName, args)
+ return id;
+ }
+
+
+ public int getWorkerPortAssignment() {
+ return 0
+ }
+
+ public int getMasterPortAssignment() {
+ return 0
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy
new file mode 100644
index 0000000..6611664
--- /dev/null
+++ b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/ImagesIT.groovy
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.funtest
+
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileSystem as HadoopFS
+import org.apache.hadoop.fs.Path
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.Before
+import org.junit.Test
+
+class ImagesIT extends CommandTestBase implements FuntestProperties {
+
+
+ @Before
+ public void verifyPreconditions() {
+ assumeBoolOption(SLIDER_CONFIG, KEY_SLIDER_FUNTESTS_ENABLED, true)
+ assumeBoolOption(SLIDER_CONFIG, KEY_TEST_HBASE_ENABLED, true)
+ }
+
+ @Test
+ public void testImageExists() throws Throwable {
+
+ Configuration conf = loadSliderConf()
+ String testImage = conf.get(KEY_TEST_HBASE_TAR)
+ assert testImage
+ Path path = new Path(testImage)
+ HadoopFS fs = HadoopFS.get(
+ path.toUri(),
+ conf)
+ assert fs.exists(path)
+ }
+
+ @Test
+ public void testAppConfExists() throws Throwable {
+ Configuration conf = loadSliderConf()
+ String dir = conf.get(KEY_TEST_HBASE_APPCONF)
+
+ assert conf.get(KEY_TEST_HBASE_APPCONF)
+ Path path = new Path(dir)
+ HadoopFS fs = HadoopFS.get(
+ path.toUri(),
+ conf)
+ assert fs.exists(path)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
deleted file mode 100644
index 3415b35..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestFunctionalHBaseCluster.groovy
+++ /dev/null
@@ -1,223 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.RoleKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.tools.ConfigHelper
-import org.apache.slider.core.registry.info.RegistryNaming
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.providers.hbase.HBaseConfigFileOptions
-import org.apache.slider.providers.hbase.HBaseTestUtils
-import org.apache.slider.server.appmaster.PublishedArtifacts
-import org.apache.zookeeper.KeeperException
-import org.apache.zookeeper.WatchedEvent
-import org.apache.zookeeper.Watcher
-import org.apache.zookeeper.ZKUtil
-import org.apache.zookeeper.ZooKeeper
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
-import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_MASTER
-import static org.apache.slider.providers.hbase.HBaseKeys.ROLE_WORKER
-
-@CompileStatic
-@Slf4j
-public class TestFunctionalHBaseCluster extends HBaseCommandTestBase
- implements FuntestProperties, Arguments, SliderExitCodes {
-
-
- public static final String HBASE_HEAP = "96m"
-
- public String getClusterName() {
- return "test_functional_hbase_cluster"
- }
-
- public String getClusterZNode() {
- return "/yarnapps_slider_yarn_" + clusterName;
- }
-
- @Before
- public void prepareCluster() {
-
- String quorumServers = SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS)
-
- ZooKeeper monitor = new ZooKeeper(quorumServers,
- 1000, new Watcher(){
- @Override
- public void process(WatchedEvent watchedEvent) {
- }
- }, false)
- try {
- ZKUtil.deleteRecursive(monitor, clusterZNode)
- } catch (KeeperException.NoNodeException ignored) {
- log.info(clusterZNode + " not there")
- }
- setupCluster(clusterName)
- }
-
- @After
- public void teardownCluster() {
- teardown(clusterName)
- }
-
- @Test
- public void testHBaseCreateCluster() throws Throwable {
-
- describe description
-
- int numWorkers = desiredWorkerCount;
-
- def clusterpath = buildClusterPath(clusterName)
- assert !clusterFS.exists(clusterpath)
- Map<String, Integer> roleMap = createHBaseCluster(
- clusterName,
- 1, numWorkers,
- [
- ARG_OPTION,
- HBaseConfigFileOptions.KEY_HBASE_MASTER_INFO_PORT,
- Integer.toString(masterPortAssignment),
- ARG_COMP_OPT, ROLE_MASTER, RoleKeys.JVM_HEAP, HBASE_HEAP,
- ARG_OPTION,
- HBaseConfigFileOptions.KEY_REGIONSERVER_PORT,
- Integer.toString(workerPortAssignment),
- ARG_COMP_OPT, ROLE_WORKER, RoleKeys.JVM_HEAP, HBASE_HEAP,
- ],
- [:]
- )
-
- //get a slider client against the cluster
- SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, clusterName)
- ClusterDescription cd2 = sliderClient.clusterDescription
- assert clusterName == cd2.name
-
- log.info("Connected via Client {} with {} workers", sliderClient.toString(),
- numWorkers)
-
- //wait for the role counts to be reached
- waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
-
- Configuration clientConf = HBaseTestUtils.createHBaseConfiguration(sliderClient)
- HBaseTestUtils.assertHBaseMasterFound(clientConf)
- HBaseTestUtils.waitForHBaseRegionServerCount(sliderClient,
- clusterName,
- numWorkers,
- HBASE_LAUNCH_WAIT_TIME)
-
- clusterOperations(
- clusterName,
- sliderClient,
- clientConf,
- numWorkers,
- roleMap,
- cd2)
- }
-
- /**
- * Override to change policy of the deired no of workers
- * @return
- */
- def int getDesiredWorkerCount() {
- return SLIDER_CONFIG.getInt(KEY_SLIDER_TEST_NUM_WORKERS,
- DEFAULT_SLIDER_NUM_WORKERS)
- }
-
-
- public String getDescription() {
- return "Create a working HBase cluster $clusterName"
- }
-
- /**
- * Override point for any cluster operations
- * @param clustername name of cluster
- * @param sliderClient bonded low level client
- * @param clientConf config
- * @param numWorkers no. of workers created
- * @param roleMap role map
- * @param cd current cluster
- */
- public void clusterOperations(
- String clustername,
- SliderClient sliderClient,
- Configuration clientConf,
- int numWorkers,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
-
- log.info("Client Configuration = " + ConfigHelper.dumpConfigToString(clientConf))
-
- //grab some registry bits
- registry([ARG_LIST])
- registry([ARG_LIST, ARG_SERVICETYPE, SliderKeys.APP_TYPE , ARG_VERBOSE])
-
- //unknown service type
- registry(EXIT_NOT_FOUND,
- [ARG_LIST, ARG_SERVICETYPE, "org.apache.something"])
-
- registry(EXIT_NOT_FOUND,
- [ARG_LIST, ARG_NAME, "cluster-with-no-name"])
-
- // how to work out the current service name?
- def name = RegistryNaming.createRegistryName(clustername,
- System.getProperty("user.name"),
- SliderKeys.APP_TYPE,
- 1)
- registry([ARG_LIST, ARG_VERBOSE, ARG_NAME, name])
-
- registry([ARG_LISTCONF, ARG_NAME, name])
- registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, name, ARG_INTERNAL])
- registry(EXIT_NOT_FOUND, [ARG_LISTCONF, ARG_NAME, "unknown"])
- registry([ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
- ARG_NAME, name])
- registry(EXIT_NOT_FOUND, [ARG_GETCONF, "no-such-config",
- ARG_NAME, name])
-
- registry(EXIT_NOT_FOUND, [ARG_GETCONF, "illegal/config/name!",
- ARG_NAME, name])
-
- registry(EXIT_NOT_FOUND,[ARG_GETCONF, PublishedArtifacts.COMPLETE_CONFIG,
- ARG_NAME, name, ARG_INTERNAL])
-
-
- def yarn_site_config = PublishedArtifacts.YARN_SITE_CONFIG
- registry([ARG_GETCONF, yarn_site_config,
- ARG_NAME, name])
-
- File getConfDir = new File("target/$clusterName/getconf")
- getConfDir.mkdirs();
- registry([ARG_GETCONF, yarn_site_config,
- ARG_NAME, name,
- ARG_DEST, getConfDir.absolutePath])
- File retrieved = new File(getConfDir, yarn_site_config +".xml")
- def confFromFile = ConfigHelper.loadConfFromFile(retrieved)
- assert confFromFile.get(YarnConfiguration.RM_ADDRESS)
-
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy
deleted file mode 100644
index c8f3be3..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseBuildSetup.groovy
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.slider.funtest.abstracttests.AbstractTestBuildSetup
-import org.junit.Test
-
-class TestHBaseBuildSetup extends AbstractTestBuildSetup {
-
- @Test
- public void testHBaseBuildsHavePathsDefined() throws Throwable {
- Configuration conf = loadSliderConf();
- assumeBoolOption(conf, KEY_SLIDER_FUNTESTS_ENABLED, true)
-
- assumeBoolOption(conf, KEY_TEST_HBASE_ENABLED, true)
-
- assertStringOptionSet(conf, KEY_TEST_HBASE_APPCONF)
- assertStringOptionSet(conf, KEY_TEST_HBASE_TAR)
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
deleted file mode 100644
index 3c9b8ed..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterBuildDestroy.groovy
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.hadoop.fs.Path
-import org.apache.slider.common.SliderKeys
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.core.main.LauncherExitCodes
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.apache.slider.providers.hbase.HBaseKeys
-import org.junit.AfterClass
-import org.junit.BeforeClass
-import org.junit.Test
-
-@CompileStatic
-@Slf4j
-public class TestHBaseClusterBuildDestroy extends HBaseCommandTestBase
- implements FuntestProperties, Arguments {
-
-
- static String CLUSTER = "test_cluster_build_destroy"
-
-
- @BeforeClass
- public static void prepareCluster() {
- assumeFunctionalTestsEnabled();
- setupCluster(CLUSTER)
- }
-
- @AfterClass
- public static void destroyCluster() {
- teardown(CLUSTER)
- }
-
- @Test
- public void testBuildAndDestroyCluster() throws Throwable {
- def clusterDir = SliderKeys.SLIDER_BASE_DIRECTORY + "/cluster/$CLUSTER"
- def clusterDirPath = new Path(clusterFS.homeDirectory, clusterDir)
- clusterFS.delete(clusterDirPath, true)
- slider(0,
- [
- SliderActions.ACTION_BUILD,
- CLUSTER,
- ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
- ARG_ZKHOSTS,
- SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
- ARG_IMAGE,
- SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
- ARG_CONFDIR,
- SLIDER_CONFIG.get(KEY_TEST_HBASE_APPCONF),
- ARG_COMPONENT, HBaseKeys.ROLE_MASTER, "1",
- ARG_COMPONENT, HBaseKeys.ROLE_WORKER, "1",
- ARG_OPTION, "site.hbase.master.info.port", "8180",
- ])
-
-
-
- assert clusterFS.exists(clusterDirPath)
- //cluster exists if you don't want it to be live
- exists(0, CLUSTER, false)
- // condition returns false if it is required to be live
- exists(LauncherExitCodes.EXIT_FALSE, CLUSTER, true)
- destroy(CLUSTER)
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
deleted file mode 100644
index 01c7131..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseClusterLifecycle.groovy
+++ /dev/null
@@ -1,183 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.funtest.framework.FuntestProperties
-import org.junit.After
-import org.junit.Before
-import org.junit.Test
-
-@CompileStatic
-@Slf4j
-public class TestHBaseClusterLifecycle extends HBaseCommandTestBase
- implements FuntestProperties, Arguments, SliderExitCodes {
-
-
- static String CLUSTER = "test_cluster_lifecycle"
-
-
- @Before
- public void prepareCluster() {
- setupCluster(CLUSTER)
- }
-
- @After
- public void destroyCluster() {
- teardown(CLUSTER)
- }
-
- @Test
- public void testClusterLifecycle() throws Throwable {
-
- describe "Walk a 0-role cluster through its lifecycle"
-
-
- def clusterpath = buildClusterPath(CLUSTER)
- assert !clusterFS.exists(clusterpath)
-
-
- Map<String, Integer> roleMap = createHBaseCluster(CLUSTER,
- 0,
- 0,
- [],
- [:])
-
-
- //at this point the cluster should exist.
- assertPathExists(clusterFS,"Cluster parent directory does not exist", clusterpath.parent)
-
- assertPathExists(clusterFS,"Cluster directory does not exist", clusterpath)
-
- // assert it exists on the command line
- exists(0, CLUSTER)
-
- //destroy will fail in use
-
- destroy(EXIT_APPLICATION_IN_USE, CLUSTER)
-
- //thaw will fail as cluster is in use
- thaw(EXIT_APPLICATION_IN_USE, CLUSTER)
-
- //it's still there
- exists(0, CLUSTER)
-
- //listing the cluster will succeed
- list(0, CLUSTER)
-
- //simple status
- status(0, CLUSTER)
-
- //now status to a temp file
- File jsonStatus = File.createTempFile("tempfile", ".json")
- try {
- slider(0,
- [
- SliderActions.ACTION_STATUS, CLUSTER,
- ARG_OUTPUT, jsonStatus.canonicalPath
- ])
-
- assert jsonStatus.exists()
- ClusterDescription cd = ClusterDescription.fromFile(jsonStatus)
-
- assert CLUSTER == cd.name
-
- log.info(cd.toJsonString())
-
- getConf(0, CLUSTER)
-
- //get a slider client against the cluster
- SliderClient sliderClient = bondToCluster(SLIDER_CONFIG, CLUSTER)
- ClusterDescription cd2 = sliderClient.clusterDescription
- assert CLUSTER == cd2.name
-
- log.info("Connected via Client {}", sliderClient.toString())
-
- //freeze
- freeze(CLUSTER, [
- ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
- ARG_MESSAGE, "freeze-in-test cluster lifecycle"
- ])
-
- //cluster exists if you don't want it to be live
- exists(0, CLUSTER, false)
- // condition returns false if it is required to be live
- exists(EXIT_FALSE, CLUSTER, true)
-
-
- // thaw then freeze the cluster
-
- thaw(CLUSTER,
- [
- ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
- ])
- exists(0, CLUSTER)
- freeze(CLUSTER,
- [
- ARG_FORCE,
- ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
- ARG_MESSAGE, "forced-freeze-in-test"
- ])
-
- //cluster is no longer live
- exists(0, CLUSTER, false)
-
- // condition returns false if it is required to be live
- exists(EXIT_FALSE, CLUSTER, true)
-
- // thaw with a restart count set to enable restart
-
- describe "the kill/restart phase may fail if yarn.resourcemanager.am.max-attempts is too low"
- thaw(CLUSTER,
- [
- ARG_WAIT, Integer.toString(THAW_WAIT_TIME),
- ARG_DEFINE, SliderXmlConfKeys.KEY_AM_RESTART_LIMIT + "=3"
- ])
-
-
- ClusterDescription status = killAmAndWaitForRestart(sliderClient, CLUSTER)
-
- def restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
- assert restarted != null
- assert Integer.parseInt(restarted) == 0
- freeze(CLUSTER)
-
- destroy(0, CLUSTER)
-
- //cluster now missing
- exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
-
- } finally {
- jsonStatus.delete()
- }
-
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy
deleted file mode 100644
index 4bf18f6..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseIntegration.groovy
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.IntegrationTestIngest
-import org.apache.hadoop.hbase.IntegrationTestingUtility
-import org.apache.hadoop.util.ToolRunner
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.HBaseConfigFileOptions;
-
-/* Runs IntegrationTestIngest on cluster
- *
- * Note: this test runs for about 20 minutes
- * please set slider.test.timeout.seconds accordingly
- */
-class TestHBaseIntegration extends TestFunctionalHBaseCluster {
-
- @Override
- String getClusterName() {
- return "test_hbase_integration"
- }
-
- @Override
- void clusterOperations(
- String clustername,
- SliderClient sliderClient,
- Configuration clientConf,
- int numWorkers,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
- String parent = "/yarnapps_slider_yarn_" + clustername
- clientConf.set(HBaseConfigFileOptions.KEY_ZNODE_PARENT, parent)
-
- clientConf.set(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER, "true")
-
- String[] args = []
- IntegrationTestIngest test = new IntegrationTestIngest();
- test.setConf(clientConf)
- int ret = ToolRunner.run(clientConf, test, args);
- assert ret == 0;
- }
-
-
- public int getWorkerPortAssignment() {
- return 0
- }
-
- public int getMasterPortAssignment() {
- return 0
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy
deleted file mode 100644
index 3d6c46c..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseLoad.groovy
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.hadoop.hbase.util.LoadTestTool
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.client.SliderClient
-import org.apache.slider.providers.hbase.HBaseConfigFileOptions
-
-class TestHBaseLoad extends TestFunctionalHBaseCluster {
-
- @Override
- String getClusterName() {
- return "test_hbase_load"
- }
-
- @Override
- void clusterOperations(
- String clustername,
- SliderClient sliderClient,
- Configuration clientConf,
- int numWorkers,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
- assert clustername
- int numKeys = 4000 * numWorkers
- String[] args = ["-tn", "test", "-write", "4:100",
- "-num_keys", numKeys,
- "-zk", clientConf.get(HBaseConfigFileOptions.KEY_ZOOKEEPER_QUORUM),
- "-zk_root", clientConf.get(HBaseConfigFileOptions.KEY_ZNODE_PARENT),
-
- ]
- LoadTestTool loadTool = new LoadTestTool();
- loadTool.setConf(clientConf)
- int ret = loadTool.run(args);
- assert ret == 0;
- }
-
-
- public int getWorkerPortAssignment() {
- return 0
- }
-
- public int getMasterPortAssignment() {
- return 0
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e1258c8a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy b/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy
deleted file mode 100644
index fab73b3..0000000
--- a/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestHBaseNodeFailure.groovy
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.funtest
-
-import org.apache.hadoop.conf.Configuration
-import org.apache.slider.api.ClusterDescription
-import org.apache.slider.api.RoleKeys
-import org.apache.slider.api.StatusKeys
-import org.apache.slider.client.SliderClient
-import org.apache.slider.common.params.ActionKillContainerArgs
-import org.apache.slider.providers.hbase.HBaseKeys
-import org.apache.slider.providers.hbase.HBaseTestUtils
-
-class TestHBaseNodeFailure extends TestFunctionalHBaseCluster {
-
-
- public static final int RESTART_SLEEP_TIME = 5000
-
- @Override
- String getClusterName() {
- return "test_hbase_node_failure"
- }
-
- @Override
- String getDescription() {
- "Fail containers and verify that the cluster recovers"
- }
-
- @Override
- void clusterOperations(
- String clustername,
- SliderClient sliderClient,
- Configuration clientConf,
- int numWorkers,
- Map<String, Integer> roleMap,
- ClusterDescription cd) {
-
-
- killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
- // let it take
- sleep(RESTART_SLEEP_TIME)
-
- //wait for the role counts to be reached
- cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
- // then expect a restart
- HBaseTestUtils.waitForHBaseRegionServerCount(
- sliderClient,
- clusterName,
- numWorkers,
- HBASE_LAUNCH_WAIT_TIME)
- assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
- killInstanceOfRole(sliderClient, HBaseKeys.ROLE_WORKER)
- // let it take
- sleep(RESTART_SLEEP_TIME)
- // then expect a restart
-
- //wait for the role counts to be reached
- cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
-
- HBaseTestUtils.waitForHBaseRegionServerCount(
- sliderClient,
- clusterName,
- numWorkers,
- HBASE_LAUNCH_WAIT_TIME)
- assert cd.roles[HBaseKeys.ROLE_WORKER][RoleKeys.ROLE_FAILED_INSTANCES] == "2"
-
- killInstanceOfRole(sliderClient, HBaseKeys.ROLE_MASTER)
- // let it take
- sleep(RESTART_SLEEP_TIME)
-
- // wait for the role counts to be reached
- cd = waitForRoleCount(sliderClient, roleMap, HBASE_LAUNCH_WAIT_TIME)
- HBaseTestUtils.waitForHBaseRegionServerCount(
- sliderClient,
- clusterName,
- numWorkers,
- HBASE_LAUNCH_WAIT_TIME)
- assert cd.roles[HBaseKeys.ROLE_MASTER][RoleKeys.ROLE_FAILED_INSTANCES] == "1"
-
- // now trigger AM failure
- ClusterDescription status = killAmAndWaitForRestart(sliderClient, clusterName)
-
- def restarted = status.getInfo(
- StatusKeys.INFO_CONTAINERS_AM_RESTART)
- assert restarted != null
- assert Integer.parseInt(restarted) == 1 + numWorkers
-
- }
-
- /**
- * Kill a random in instance of a role in the cluster
- * @param sliderClient client
- * @param role
- * @return ID of container killed
- */
- public String killInstanceOfRole(
- SliderClient sliderClient, String role) {
- ClusterDescription cd = sliderClient.getClusterDescription()
- def instances = cd.instances[role]
- if (instances == null || instances.size() == 0) {
- log.info("No instances of role $role to kill")
- return null;
- }
- String id = instances[new Random().nextInt(instances.size())]
- ActionKillContainerArgs args = new ActionKillContainerArgs()
- args.id = id
- sliderClient.actionKillContainer(clusterName, args)
- return id;
- }
-
-
- public int getWorkerPortAssignment() {
- return 0
- }
-
- public int getMasterPortAssignment() {
- return 0
- }
-}