You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/05/07 18:26:16 UTC
svn commit: r1593056 - in /incubator/slider/trunk: ./ slider-agent/
slider-assembly/ slider-assembly/src/assembly/
slider-core/src/main/java/org/apache/slider/common/tools/
slider-core/src/test/groovy/org/apache/slider/common/tools/ slider-funtest/
sli...
Author: stevel
Date: Wed May 7 16:26:15 2014
New Revision: 1593056
URL: http://svn.apache.org/r1593056
Log:
SLIDER-5 CLI to list and retrieve service bindings & configs
this patch sets up the docs and tests ready for a deployment
Added:
incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy
- copied, changed from r1592947, incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy
incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy
- copied, changed from r1592947, incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy
incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/registry/
incubator/slider/trunk/slider-funtest/src/test/resources/
incubator/slider/trunk/slider-funtest/src/test/resources/log4j.properties
- copied unchanged from r1592947, incubator/slider/trunk/slider-core/src/test/resources/log4j.properties
incubator/slider/trunk/src/site/markdown/developing/functional_tests.md
- copied, changed from r1592947, incubator/slider/trunk/src/site/markdown/developing/testing.md
incubator/slider/trunk/src/site/markdown/developing/manual_testing.md
- copied, changed from r1592947, incubator/slider/trunk/slider-funtest/src/test/manual/testing.md
Removed:
incubator/slider/trunk/slider-funtest/src/test/manual/testing.md
Modified:
incubator/slider/trunk/pom.xml
incubator/slider/trunk/slider-agent/pom.xml
incubator/slider/trunk/slider-assembly/pom.xml
incubator/slider/trunk/slider-assembly/src/assembly/slider-bin.xml
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
incubator/slider/trunk/slider-funtest/pom.xml
incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
incubator/slider/trunk/src/site/markdown/developing/agent_test_setup.md
incubator/slider/trunk/src/site/markdown/developing/building.md
incubator/slider/trunk/src/site/markdown/developing/index.md
incubator/slider/trunk/src/site/markdown/developing/testing.md
incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml
Modified: incubator/slider/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/pom.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/pom.xml (original)
+++ incubator/slider/trunk/pom.xml Wed May 7 16:26:15 2014
@@ -149,7 +149,7 @@
-->
<project.java.src.version>1.7</project.java.src.version>
<enforced.java.version>${project.java.src.version}</enforced.java.version>
-
+ <groovy.version>2.2.2</groovy.version>
<!--
options
@@ -180,7 +180,6 @@
<commons-lang.version>2.6</commons-lang.version>
<curator.version>2.4.1</curator.version>
<easymock.version>3.1</easymock.version>
- <groovy.version>2.1.9</groovy.version>
<guava.version>11.0.2</guava.version>
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
@@ -208,9 +207,9 @@
<!-- Plugin versions -->
<gmavenVersion>1.5</gmavenVersion>
<gmavenProviderSelection>2.0</gmavenProviderSelection>
- <buildnumber-maven-plugin.version>1.2</buildnumber-maven-plugin.version>
<groovy-eclipse-compiler.version>2.8.0-01</groovy-eclipse-compiler.version>
- <groovy-eclipse-batch.version>2.1.3-01</groovy-eclipse-batch.version>
+ <groovy-eclipse-batch.version>2.1.8-01</groovy-eclipse-batch.version>
+ <buildnumber-maven-plugin.version>1.2</buildnumber-maven-plugin.version>
<maven.version.range>[3.0.0,)</maven.version.range>
Modified: incubator/slider/trunk/slider-agent/pom.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-agent/pom.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-agent/pom.xml (original)
+++ incubator/slider/trunk/slider-agent/pom.xml Wed May 7 16:26:15 2014
@@ -37,6 +37,7 @@
<build>
<plugins>
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
@@ -54,6 +55,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.0</version>
@@ -83,6 +85,7 @@
</execution>
</executions>
</plugin>
+
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
Modified: incubator/slider/trunk/slider-assembly/pom.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-assembly/pom.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-assembly/pom.xml (original)
+++ incubator/slider/trunk/slider-assembly/pom.xml Wed May 7 16:26:15 2014
@@ -61,6 +61,7 @@
<artifactId>maven-assembly-plugin</artifactId>
<version>${maven-assembly-plugin.version}</version>
<configuration>
+ <tarLongFileMode>gnu</tarLongFileMode>
<descriptors>
<descriptor>src/assembly/slider-bin.xml</descriptor>
</descriptors>
@@ -100,6 +101,33 @@
</executions>
</plugin>
+ <!-- copy in the agent tar file -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy</id>
+ <phase>package</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <type>tar.gz</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/agent</outputDirectory>
+ <destFileName>slider-agent.tar.gz</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -177,6 +205,18 @@
<version>${project.version}</version>
</dependency>
+ <!--
+ needed to order the build and ensure the agent tar is found
+ the test scope ensures that it isn't copied into the lib dir
+ -->
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>tar.gz</type>
+ </dependency>
+
<dependency>
<groupId>com.beust</groupId>
<artifactId>jcommander</artifactId>
@@ -188,13 +228,7 @@
<artifactId>hadoop-client</artifactId>
<type>pom</type>
</dependency>
-
- <!--
- <dependency>
- <groupId>org.apache.zookeeper</groupId>
- <artifactId>zookeeper</artifactId>
- </dependency>
- -->
+
</dependencies>
Modified: incubator/slider/trunk/slider-assembly/src/assembly/slider-bin.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-assembly/src/assembly/slider-bin.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-assembly/src/assembly/slider-bin.xml (original)
+++ incubator/slider/trunk/slider-assembly/src/assembly/slider-bin.xml Wed May 7 16:26:15 2014
@@ -58,10 +58,10 @@
</fileSet>
<fileSet>
- <directory>${project.build.directory}/../../slider-agent/target</directory>
+ <directory>${project.build.directory}/agent</directory>
<outputDirectory>agent</outputDirectory>
<includes>
- <include>*.tar.gz</include>
+ <include>slider-agent.tar.gz</include>
</includes>
</fileSet>
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java Wed May 7 16:26:15 2014
@@ -501,8 +501,8 @@ public class CoreFileSystem {
* @throws SliderException if the path does not exist
*/
public Path createPathThatMustExist(String uri) throws
- SliderException,
- IOException {
+ SliderException, IOException {
+ Preconditions.checkNotNull(uri);
Path path = new Path(uri);
verifyPathExists(path);
return path;
Modified: incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java (original)
+++ incubator/slider/trunk/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java Wed May 7 16:26:15 2014
@@ -1275,11 +1275,10 @@ public final class SliderUtils {
}
public static Path extractImagePath(CoreFileSystem fs, MapOperations internalOptions) throws
- SliderException,
- IOException {
+ SliderException, IOException {
Path imagePath;
String imagePathOption =
- internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
+ internalOptions.get(OptionKeys.INTERNAL_APPLICATION_IMAGE_PATH);
String appHomeOption = internalOptions.get(OptionKeys.INTERNAL_APPLICATION_HOME);
if (!isUnset(imagePathOption)) {
imagePath = fs.createPathThatMustExist(imagePathOption);
Modified: incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy (original)
+++ incubator/slider/trunk/slider-core/src/test/groovy/org/apache/slider/common/tools/TestZKIntegration.groovy Wed May 7 16:26:15 2014
@@ -31,7 +31,6 @@ import org.junit.Before
import org.junit.Test
@Slf4j
-@CompileStatic
class TestZKIntegration extends YarnZKMiniClusterTestBase implements KeysForTests {
Modified: incubator/slider/trunk/slider-funtest/pom.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/pom.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-funtest/pom.xml (original)
+++ incubator/slider/trunk/slider-funtest/pom.xml Wed May 7 16:26:15 2014
@@ -249,8 +249,16 @@
<groupId>org.apache.bigtop.itest</groupId>
<artifactId>itest-common</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ <type>tar.gz</type>
+ </dependency>
+
+
</dependencies>
Modified: incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy (original)
+++ incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy Wed May 7 16:26:15 2014
@@ -56,7 +56,7 @@ abstract class CommandTestBase extends S
SLIDER_BIN_DIR).canonicalFile
public static final File SLIDER_SCRIPT = new File(
SLIDER_BIN_DIRECTORY,
- "bin/slider").canonicalFile
+ BIN_SLIDER).canonicalFile
public static final File SLIDER_CONF_DIRECTORY = new File(SLIDER_CONF_DIR).canonicalFile
public static final File SLIDER_CONF_XML = new File(SLIDER_CONF_DIRECTORY,
CLIENT_CONFIG_FILENAME).canonicalFile
@@ -70,8 +70,6 @@ abstract class CommandTestBase extends S
public static final boolean ACCUMULO_TESTS_ENABLED
public static final boolean FUNTESTS_ENABLED
- public static final boolean AGENTTESTS_ENABLED
- public static final String AGENT_PKG
static {
@@ -93,10 +91,6 @@ abstract class CommandTestBase extends S
ACCUMULO_TESTS_ENABLED =
SLIDER_CONFIG.getBoolean(KEY_TEST_ACCUMULO_ENABLED, false)
- AGENTTESTS_ENABLED =
- SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, true)
- AGENT_PKG =
- SLIDER_CONFIG.getRaw(KEY_TEST_AGENT_TAR)
}
@Rule
@@ -405,14 +399,6 @@ abstract class CommandTestBase extends S
assert action != null
assert clustername != null
-
-
- List<String> roleList = [];
- roles.each { String role, Integer val ->
- log.info("Role $role := $val")
- roleList << ARG_COMPONENT << role << Integer.toString(val)
- }
-
List<String> argsList = [action, clustername]
argsList << ARG_ZKHOSTS <<
@@ -423,6 +409,12 @@ abstract class CommandTestBase extends S
argsList << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
}
+ List<String> roleList = [];
+ roles.each { String role, Integer val ->
+ log.info("Role $role := $val")
+ roleList << ARG_COMPONENT << role << Integer.toString(val)
+ }
+
argsList += roleList;
//now inject any cluster options
@@ -511,9 +503,4 @@ abstract class CommandTestBase extends S
assume(ACCUMULO_TESTS_ENABLED, "Accumulo tests disabled")
}
- public void assumeAgentTestsEnabled() {
- assumeFunctionalTestsEnabled()
- assume(AGENTTESTS_ENABLED, "Agent tests disabled")
- }
-
}
Modified: incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy (original)
+++ incubator/slider/trunk/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/FuntestProperties.groovy Wed May 7 16:26:15 2014
@@ -60,4 +60,6 @@ public interface FuntestProperties exten
String SCRIPT_NAME = "slider"
static final String KEY_TEST_CONF_XML = "slider.test.conf.xml"
static final String KEY_TEST_CONF_DIR = "slider.test.conf.dir"
+ static final String BIN_SLIDER = "bin/slider"
+ static final String AGENT_SLIDER_GZ = "agent/slider-agent.tar.gz"
}
Added: incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy?rev=1593056&view=auto
==============================================================================
--- incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy (added)
+++ incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentCommandTestBase.groovy Wed May 7 16:26:15 2014
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.CommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.Before
+import org.junit.BeforeClass
+
+
+@Slf4j
+class AgentCommandTestBase extends CommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ public static final boolean AGENTTESTS_ENABLED
+
+ protected static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
+ protected static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
+ protected static final File LOCAL_SLIDER_AGENT_TARGZ
+
+ protected static Path agentTarballPath;
+
+ static {
+ AGENTTESTS_ENABLED = SLIDER_CONFIG.getBoolean(KEY_TEST_AGENT_ENABLED, false)
+ LOCAL_SLIDER_AGENT_TARGZ = new File(
+ SLIDER_BIN_DIRECTORY,
+ AGENT_SLIDER_GZ).canonicalFile
+ }
+
+ @BeforeClass
+ public static void setupAgent() {
+ assumeAgentTestsEnabled()
+ assume(LOCAL_SLIDER_AGENT_TARGZ.exists(), "Slider agent not found at $LOCAL_SLIDER_AGENT_TARGZ")
+ agentTarballPath = new Path(clusterFS.homeDirectory, "agent.tar.gz")
+
+ Path localTarball = new Path(LOCAL_SLIDER_AGENT_TARGZ.toURI());
+ clusterFS.copyFromLocalFile(false, true, localTarball, agentTarballPath)
+ }
+
+ public static void assumeAgentTestsEnabled() {
+ assumeFunctionalTestsEnabled()
+ assume(AGENTTESTS_ENABLED, "Agent tests disabled")
+ }
+
+ public static void logShell(SliderShell shell) {
+ for (String str in shell.out) {
+ log.info str
+ }
+ }
+
+ public static void assertComponentCount(String component, int count, SliderShell shell) {
+ log.info("Asserting component count.")
+ String entry = findLineEntry(shell, ["instances", component] as String[])
+ log.info(entry)
+ assert entry != null
+ int instanceCount = 0
+ int index = entry.indexOf("container_")
+ while (index != -1) {
+ instanceCount++;
+ index = entry.indexOf("container_", index + 1)
+ }
+
+ assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
+ }
+
+ public static String findLineEntry(SliderShell shell, String[] locators) {
+ int index = 0;
+ for (String str in shell.out) {
+ if (str.contains("\"" + locators[index] + "\"")) {
+ if (locators.size() == index + 1) {
+ return str;
+ } else {
+ index++;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ public static boolean isAppRunning(String text, SliderShell shell) {
+ boolean exists = false
+ for (String str in shell.out) {
+ if (str.contains(text)) {
+ exists = true
+ }
+ }
+
+ return exists
+ }
+}
Modified: incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy (original)
+++ incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestAppsThroughAgent.groovy Wed May 7 16:26:15 2014
@@ -20,26 +20,21 @@ package org.apache.slider.funtest.lifecy
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.params.Arguments
-import org.apache.slider.common.params.SliderActions
-import org.apache.slider.funtest.framework.CommandTestBase
import org.apache.slider.funtest.framework.SliderShell
import org.junit.Test
-import static org.apache.slider.common.SliderExitCodes.EXIT_UNKNOWN_INSTANCE
-
@CompileStatic
@Slf4j
-public class TestAppsThroughAgent extends CommandTestBase {
+public class TestAppsThroughAgent extends AgentCommandTestBase
+// implements FuntestProperties, Arguments, SliderExitCodes, SliderActions
+{
private static String COMMAND_LOGGER = "COMMAND_LOGGER"
private static String APPLICATION_NAME = "agenttst"
- private static String APP_RESOURCE = "../slider-core/src/test/app_packages/test_command_log/resources.json"
- private static String APP_TEMPLATE = "../slider-core/src/test/app_packages/test_command_log/appConfig.json"
@Test
public void testUsage() throws Throwable {
- SliderShell shell = slider(0, [SliderActions.ACTION_USAGE])
+ SliderShell shell = slider(EXIT_SUCCESS, [ACTION_USAGE])
assertSuccess(shell)
}
@@ -52,25 +47,21 @@ public class TestAppsThroughAgent extend
cleanup()
try {
- SliderShell shell = slider(0, [
- SliderActions.ACTION_CREATE,
- TestAppsThroughAgent.APPLICATION_NAME,
- Arguments.ARG_IMAGE,
- AGENT_PKG,
- Arguments.ARG_TEMPLATE,
- TestAppsThroughAgent.APP_TEMPLATE,
- Arguments.ARG_RESOURCES,
- TestAppsThroughAgent.APP_RESOURCE])
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
logShell(shell)
- assertSuccess(shell)
-
int attemptCount = 0
while (attemptCount < 10) {
- shell = slider(0, [
- SliderActions.ACTION_LIST,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS, [
+ ACTION_LIST,
+ APPLICATION_NAME])
if (isAppRunning("RUNNING", shell)) {
break
@@ -83,25 +74,28 @@ public class TestAppsThroughAgent extend
}
//flex
- slider(0, [
- SliderActions.ACTION_FLEX,
- TestAppsThroughAgent.APPLICATION_NAME,
- Arguments.ARG_COMPONENT,
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_FLEX,
+ APPLICATION_NAME,
+ ARG_COMPONENT,
COMMAND_LOGGER,
"2"])
// sleep till the new instance starts
sleep(1000 * 10)
- shell = slider(0, [
- SliderActions.ACTION_STATUS,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
assertComponentCount(COMMAND_LOGGER, 2, shell)
- shell = slider(0, [
- SliderActions.ACTION_LIST,
- TestAppsThroughAgent.APPLICATION_NAME])
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_LIST,
+ APPLICATION_NAME])
assert isAppRunning("RUNNING", shell), 'App is not running.'
@@ -111,59 +105,12 @@ public class TestAppsThroughAgent extend
}
}
- public void logShell(SliderShell shell) {
- for (String str in shell.out) {
- log.info str
- }
- }
-
-
- public void assertComponentCount(String component, int count, SliderShell shell) {
- log.info("Asserting component count.")
- String entry = findLineEntry(shell, ["instances", component] as String[])
- log.info(entry)
- assert entry != null
- int instanceCount = 0
- int index = entry.indexOf("container_")
- while (index != -1) {
- instanceCount++;
- index = entry.indexOf("container_", index + 1)
- }
-
- assert instanceCount == count, 'Instance count for component did not match expected. Parsed: ' + entry
- }
-
- public String findLineEntry(SliderShell shell, String[] locators) {
- int index = 0;
- for (String str in shell.out) {
- if (str.contains("\"" + locators[index] + "\"")) {
- if (locators.size() == index + 1) {
- return str;
- } else {
- index++;
- }
- }
- }
-
- return null;
- }
-
- public boolean isAppRunning(String text, SliderShell shell) {
- boolean exists = false
- for (String str in shell.out) {
- if (str.contains(text)) {
- exists = true
- }
- }
-
- return exists
- }
public void cleanup() throws Throwable {
- log.info "Cleaning app instance, if exists, by name " + TestAppsThroughAgent.APPLICATION_NAME
+ log.info "Cleaning app instance, if exists, by name " + APPLICATION_NAME
SliderShell shell = slider([
- SliderActions.ACTION_FREEZE,
- TestAppsThroughAgent.APPLICATION_NAME])
+ ACTION_FREEZE,
+ APPLICATION_NAME])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
logShell(shell)
@@ -174,8 +121,8 @@ public class TestAppsThroughAgent extend
sleep(1000 * 5)
shell = slider([
- SliderActions.ACTION_DESTROY,
- TestAppsThroughAgent.APPLICATION_NAME])
+ ACTION_DESTROY,
+ APPLICATION_NAME])
if (shell.ret != 0 && shell.ret != EXIT_UNKNOWN_INSTANCE) {
logShell(shell)
Copied: incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy (from r1592947, incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy)
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy?p2=incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy&p1=incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy&r1=1592947&r2=1593056&rev=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterBuildDestroy.groovy (original)
+++ incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterBuildDestroy.groovy Wed May 7 16:26:15 2014
@@ -16,27 +16,24 @@
* limitations under the License.
*/
-package org.apache.slider.providers.hbase.funtest
+package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
-import org.apache.slider.core.main.LauncherExitCodes
+import org.apache.slider.common.SliderExitCodes
import org.apache.slider.common.SliderKeys
import org.apache.slider.common.SliderXmlConfKeys
-import org.apache.slider.funtest.framework.CommandTestBase
-import org.apache.slider.funtest.framework.FuntestProperties
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.providers.hbase.HBaseKeys
+import org.apache.slider.funtest.framework.FuntestProperties
import org.junit.AfterClass
import org.junit.BeforeClass
import org.junit.Test
@CompileStatic
@Slf4j
-public class TestClusterBuildDestroy extends HBaseCommandTestBase
- implements FuntestProperties, Arguments {
+public class TestClusterBuildDestroy extends AgentCommandTestBase {
static String CLUSTER = "test_cluster_build_destroy"
@@ -58,29 +55,20 @@ public class TestClusterBuildDestroy ext
def clusterDir = SliderKeys.SLIDER_BASE_DIRECTORY + "/cluster/$CLUSTER"
def clusterDirPath = new Path(clusterFS.homeDirectory, clusterDir)
clusterFS.delete(clusterDirPath, true)
- slider(0,
+ slider(EXIT_SUCCESS,
[
- SliderActions.ACTION_BUILD,
+ ACTION_BUILD,
CLUSTER,
- ARG_PROVIDER, HBaseKeys.PROVIDER_HBASE,
ARG_ZKHOSTS,
SLIDER_CONFIG.get(SliderXmlConfKeys.REGISTRY_ZK_QUORUM, DEFAULT_SLIDER_ZK_HOSTS),
- ARG_IMAGE,
- SLIDER_CONFIG.get(KEY_TEST_HBASE_TAR),
- ARG_CONFDIR,
- SLIDER_CONFIG.get(KEY_TEST_HBASE_APPCONF),
- ARG_COMPONENT, HBaseKeys.ROLE_MASTER, "1",
- ARG_COMPONENT, HBaseKeys.ROLE_WORKER, "1",
- ARG_OPTION, "site.hbase.master.info.port", "8180",
])
-
assert clusterFS.exists(clusterDirPath)
//cluster exists if you don't want it to be live
- exists(0, CLUSTER, false)
+ exists(EXIT_SUCCESS, CLUSTER, false)
// condition returns false if it is required to be live
- exists(LauncherExitCodes.EXIT_FALSE, CLUSTER, true)
+ exists(EXIT_FALSE, CLUSTER, true)
destroy(CLUSTER)
}
Copied: incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy (from r1592947, incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy)
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy?p2=incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy&p1=incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy&r1=1592947&r2=1593056&rev=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-providers/hbase/hbase-funtests/src/test/groovy/org/apache/slider/providers/hbase/funtest/TestClusterLifecycle.groovy (original)
+++ incubator/slider/trunk/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/TestClusterLifecycle.groovy Wed May 7 16:26:15 2014
@@ -16,25 +16,25 @@
* limitations under the License.
*/
-package org.apache.slider.providers.hbase.funtest
+package org.apache.slider.funtest.lifecycle
import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
-import org.apache.slider.common.SliderExitCodes
-import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.StatusKeys
-import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
import org.apache.slider.common.params.Arguments
import org.apache.slider.common.params.SliderActions
-import org.apache.slider.client.SliderClient
+import org.apache.slider.funtest.framework.FuntestProperties
import org.junit.After
import org.junit.Before
import org.junit.Test
@CompileStatic
@Slf4j
-public class TestClusterLifecycle extends HBaseCommandTestBase
+public class TestClusterLifecycle extends AgentCommandTestBase
implements FuntestProperties, Arguments, SliderExitCodes {
@@ -60,6 +60,7 @@ public class TestClusterLifecycle extend
def clusterpath = buildClusterPath(CLUSTER)
assert !clusterFS.exists(clusterpath)
+/*
Map<String, Integer> roleMap = createHBaseCluster(CLUSTER,
0,
@@ -67,6 +68,7 @@ public class TestClusterLifecycle extend
[],
[:])
+*/
//at this point the cluster should exist.
assertPathExists(clusterFS,"Cluster parent directory does not exist", clusterpath.parent)
Modified: incubator/slider/trunk/src/site/markdown/developing/agent_test_setup.md
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/agent_test_setup.md?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/site/markdown/developing/agent_test_setup.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/agent_test_setup.md Wed May 7 16:26:15 2014
@@ -40,8 +40,8 @@ Get the hbase tarball, hbase-0.96.1-hado
## Permissions
Ensure that the user creating the hbase cluster has necessary permission for the resource management library and the application spec. Perform necessary **chown** and **chmod**.
-1. /share/hbase/hbase-0.96.1-hadoop2/conf
-2. /usr/lib/python2.6/site-packages/resource_management
-3. /var/lib/ambari-agent/cache/stacks/HDP
-4. /var/log/hbase, /var/run/hbase (or appropriate log and run directories)
+1. `/share/hbase/hbase-0.96.1-hadoop2/conf`
+2. `/usr/lib/python2.6/site-packages/resource_management`
+3. `/var/lib/ambari-agent/cache/stacks/HDP`
+4. `/var/log/hbase` , `/var/run/hbase` (or appropriate log and run directories)
5. Ensure hbase root/staging HDFS directories have appropriate permission
Modified: incubator/slider/trunk/src/site/markdown/developing/building.md
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/building.md?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/site/markdown/developing/building.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/building.md Wed May 7 16:26:15 2014
@@ -27,6 +27,10 @@ Here's how to set this up.
The network on the development system must be functional, with hostname lookup
of the local host working. Tests will fail without this.
+### Java 7
+
+Slider is built on Java 7 -please have a JDK for Java 7 or 8 set up
+
### Maven
You will need a version of Maven 3.0+, set up with enough memory
@@ -39,12 +43,12 @@ You will need a version of Maven 3.0+, s
### Protoc
-You need a copy of the `protoc` compile
+You need a copy of the `protoc` compiler for protobuf compilation
1. OS/X: `brew install protobuf`
1. Others: consult (Building Hadoop documentation)[http://wiki.apache.org/hadoop/HowToContribute].
-The version of protoc installed must be the same as that used by Hadoop itself.
+The version of `protoc` installed must be the same as that used by Hadoop itself.
This is absolutely critical to prevent JAR version problems.
## Building a compatible Hadoop version
@@ -297,9 +301,10 @@ You can create the JAR file and set up i
# Development Notes
-
+<!---
## Git branch model
+
The git branch model uses is
[Git Flow](http://nvie.com/posts/a-successful-git-branching-model/).
@@ -328,8 +333,9 @@ merge them back in when they are ready.
git flow release start 0.4.0
git flow release finish 0.4.0
-
-## Attn OS/X developers
+-->
+
+## Attention OS/X developers
YARN on OS/X doesn't terminate subprocesses the way it does on Linux, so
HBase Region Servers created by the hbase shell script remain running
Copied: incubator/slider/trunk/src/site/markdown/developing/functional_tests.md (from r1592947, incubator/slider/trunk/src/site/markdown/developing/testing.md)
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/functional_tests.md?p2=incubator/slider/trunk/src/site/markdown/developing/functional_tests.md&p1=incubator/slider/trunk/src/site/markdown/developing/testing.md&r1=1592947&r2=1593056&rev=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/site/markdown/developing/testing.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/functional_tests.md Wed May 7 16:26:15 2014
@@ -22,94 +22,7 @@
"OPTIONAL" in this document are to be interpreted as described in
RFC 2119.
-## Standalone Tests
-
-Slider core contains a suite of tests that are designed to run on the local machine,
-using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
-one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
-AM and spawned HBase and Accumulo processes run independently.
-
-Requirements
-* A copy of hbase.tar.gz in the local filesystem
-* A an expanded hbase.tar.gz in the local filesystem
-
-* A copy of accumulo.tar.gz in the local filesystem,
-* An expanded accumulo.tar.gz in the local filesystem,
-* an expanded Zookeeper installation
-
-All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
-
-Here's
-
- <configuration>
-
- <property>
- <name>slider.test.hbase.enabled</name>
- <description>Flag to enable/disable HBase tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.hbase.home</name>
- <value>/home/slider/hbase-0.98.0</value>
- <description>HBASE Home</description>
- </property>
-
- <property>
- <name>slider.test.hbase.tar</name>
- <value>/home/slider/Projects/hbase-0.98.0-bin.tar.gz</value>
- <description>HBASE archive URI</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.enabled</name>
- <description>Flag to enable/disable Accumulo tests</description>
- <value>true</value>
- </property>
-
- <property>
- <name>slider.test.accumulo.home</name>
- <value>
- /home/slider/accumulo-1.6.0-SNAPSHOT/</value>
- <description>Accumulo Home</description>
- </property>
-
- <property>
- <name>slider.test.accumulo.tar</name>
- <value>/home/slider/accumulo-1.6.0-SNAPSHOT-bin.tar</value>
- <description>Accumulo archive URI</description>
- </property>
-
- <property>
- <name>slider.test.am.restart.time</name>
- <description>Time in millis to await an AM restart</description>
- <value>30000</value>
- </property>
-
- <property>
- <name>zk.home</name>
- <value>/home/slider/zookeeper</value>
- <description>Zookeeper home dir on target systems</description>
- </property>
-
- <property>
- <name>hadoop.home</name>
- <value>/home/slider/hadoop-2.2.0</value>
- <description>Hadoop home dir on target systems</description>
- </property>
-
- </configuration>
-
-*Important:* For the local tests, a simple local filesystem path is used for
-all the values.
-
-For the functional tests, the accumulo and hbase tar properties will
-need to be set to a URL of a tar file that is accessible to all the
-nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
-
-
-
-## Functional Tests
+# Functional Tests
The functional test suite is designed to run the executables against
a live cluster.
@@ -117,9 +30,10 @@ a live cluster.
For these to work you need
1. A YARN Cluster -secure or insecure
1. A `slider-client.xml` file configured to interact with the cluster
-1. HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+1. Agent
+1. HBase tests: HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
directory
-1. Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
+1. Accumulo Tests Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
directory
## Configuration of functional tests
@@ -205,9 +119,7 @@ the tests do not copy the .tar/.tar.gz f
directories may be local or remote -they are copied into the `.slider` directory
during cluster creation.
-##
-
-Provider-specific parameters
+## Provider-specific parameters
An individual provider can pick up settings from their own
`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
@@ -217,7 +129,25 @@ We strongly advice placing all the value
away
2. There's one place to keep all the configuration values in sync.
-### HBase Parameters
+### Agent Tests
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+The agent tarball needs to be pushed to HBase. This is the tar
+file created
+
+ <property>
+ <name>slider.test.agent.tar</name>
+ <description>Path to the Agent Tar file in HDFS</description>
+ <value>hdfs://sandbox:8020/user/slider/agent.tar.gz</value>
+ </property>
+
+
+### HBase Tests
The HBase tests can be enabled or disabled
@@ -314,7 +244,7 @@ to keep the log files around after an ap
To test against a secure cluster
-1. `slider-client.xml` must be configured as per [Security](security.html).
+1. `slider-client.xml` must be configured as per [Security](../security.html).
1. the client must have the kerberos tokens issued so that the user running
the tests has access to HDFS and YARN.
@@ -438,4 +368,6 @@ in `~/.slider/cluster/$testname`
1. If you are using a secure cluster, make sure that the clocks
are synchronized, and that you have a current token -`klist` will
-tell you this. In a VM: install and enable `ntp`.
+tell you this. In a VM: install and enable `ntp`, consider rebooting if ther
+are any problems. Check also that it has the same time zone settings
+as the host OS.
Modified: incubator/slider/trunk/src/site/markdown/developing/index.md
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/index.md?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/site/markdown/developing/index.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/index.md Wed May 7 16:26:15 2014
@@ -23,6 +23,8 @@ Slider
* [Building](building.html)
* [Debugging](debugging.html)
* [Testing](testing.html)
+* [Functional Testing](functional_tests.html)
+* [Manual Testing](manual_testing.html)
* [Agent test setup](agent_test_setup.html)
* [Releasing](releasing.html)
Copied: incubator/slider/trunk/src/site/markdown/developing/manual_testing.md (from r1592947, incubator/slider/trunk/slider-funtest/src/test/manual/testing.md)
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/manual_testing.md?p2=incubator/slider/trunk/src/site/markdown/developing/manual_testing.md&p1=incubator/slider/trunk/slider-funtest/src/test/manual/testing.md&r1=1592947&r2=1593056&rev=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/slider-funtest/src/test/manual/testing.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/manual_testing.md Wed May 7 16:26:15 2014
@@ -17,35 +17,37 @@
# Manual Testing
-Manual testing invloves using Slider package and an AppPackage to perform basic cluster functionalities such as create/destroy, flex up/down, and freeze/thaw. A python helper script is provided that can be used to automatically test and app package.
+Manual testing invloves using Slider package and an AppPackage to perform basic
+ cluster functionalities such as create/destroy, flex up/down, and freeze/thaw.
+ A python helper script is provided that can be used to automatically test and app package.
-## SliderTester.py
+## `SliderTester.py`
Details to be added.
-## SliderTester.ini
+## `SliderTester.ini`
The various config parameters are:
-* **slider**
- * *package:* location of the slider package
- * *jdk.path:* jdk path on the test hosts
-
-* **app**
- * *package:* location of the app package
-
-* **cluster**
- * *yarn.application.classpath:* yarn application classpaths
- * *slider.zookeeper.quorum:* the ZK quorum hosts
- * *yarn.resourcemanager.address:*
- * *yarn.resourcemanager.scheduler.address:*
- * *fs.defaultFS:* e.g. hdfs://NN_HOST:8020
-
-* **test**
- * *app.user:* user to use for app creation
- * *hdfs.root.user:* hdfs root user
- * *hdfs.root.dir:* HDFS root, default /slidertst
- * *hdfs.user.dir:* HDFS user dir, default /user
- * *test.root:* local test root folder, default /test
- * *cluster.name:* name of the test cluster, default tst1
- * *cluster.type:* cluster type to build and test, e.g. hbase,storm,accumulo
+### slider
+* `package`: location of the slider package
+* `jdk.path`: jdk path on the test hosts
+
+### app
+* `package`: location of the app package
+
+### cluster
+* `yarn.application.classpath`: yarn application classpaths
+* `slider.zookeeper.quorum`: the ZK quorum hosts
+* `yarn.resourcemanager.address`:
+* `yarn.resourcemanager.scheduler.address`:
+* `fs.defaultFS`: e.g. `hdfs://NN_HOST:8020`
+
+### test
+* `app.user`: user to use for app creation
+* `hdfs.root.user`: hdfs root user
+* `hdfs.root.dir`: HDFS root, default /slidertst
+* `hdfs.user.dir`: HDFS user dir, default /user
+* `test.root`: local test root folder, default /test
+* `cluster.name`: name of the test cluster, default tst1
+* `cluster.type`: cluster type to build and test, e.g. hbase,storm,accumulo
-* **agent**
+### agent
Modified: incubator/slider/trunk/src/site/markdown/developing/testing.md
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/site/markdown/developing/testing.md?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/site/markdown/developing/testing.md (original)
+++ incubator/slider/trunk/src/site/markdown/developing/testing.md Wed May 7 16:26:15 2014
@@ -27,19 +27,25 @@
Slider core contains a suite of tests that are designed to run on the local machine,
using Hadoop's `MiniDFSCluster` and `MiniYARNCluster` classes to create small,
one-node test clusters. All the YARN/HDFS code runs in the JUnit process; the
-AM and spawned HBase and Accumulo processes run independently.
+AM and spawned processeses run independently.
+
+
+
+### For HBase Tests in `slider-providers/hbase`
Requirements
-* A copy of hbase.tar.gz in the local filesystem
-* A an expanded hbase.tar.gz in the local filesystem
+* A copy of `hbase.tar.gz` in the local filesystem
+* A an expanded `hbase.tar.gz` in the local filesystem
+
-* A copy of accumulo.tar.gz in the local filesystem,
-* An expanded accumulo.tar.gz in the local filesystem,
+### For Accumulo Tests in `slider-providers/accumulo`
+* A copy of `accumulo.tar.gz` in the local filesystem,
+* An expanded `accumulo.tar.gz` in the local filesystem,
* an expanded Zookeeper installation
All of these need to be defined in the file `slider-core/src/test/resources/slider-test.xml`
-Here's
+Example:
<configuration>
@@ -108,106 +114,7 @@ need to be set to a URL of a tar file th
nodes in the cluster -which usually means HDFS, and so an `hdfs://` URL
-
-## Functional Tests
-
-The functional test suite is designed to run the executables against
-a live cluster.
-
-For these to work you need
-1. A YARN Cluster -secure or insecure
-1. A `slider-client.xml` file configured to interact with the cluster
-1. HBase `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-1. Accumulo `.tar.gz` uploaded to HDFS, and a local or remote accumulo conf
-directory
-
-## Configuration of functional tests
-
-Maven needs to be given
-1. A path to the expanded test archive
-1. A path to a slider configuration directory for the cluster
-
-The path for the expanded test is automatically calculated as being the directory under
-`..\slider-assembly\target` where an untarred slider distribution can be found.
-If it is not present, the tests will fail
-
-The path to the configuration directory must be supplied in the property
-`slider.conf.dir` which can be set on the command line
-
- mvn test -Dslider.conf.dir=src/test/configs/sandbox/slider
-
-It can also be set in the (optional) file `slider-funtest/build.properties`:
-
- slider.conf.dir=src/test/configs/sandbox/slider
-
-This file is loaded whenever a slider build or test run takes place
-
-## Configuration of `slider-client.xml`
-
-The `slider-client.xml` must have extra configuration options for both the HBase and
-Accumulo tests, as well as a common set for actually talking to a YARN cluster.
-
-## Disabling the functional tests entirely
-
-All functional tests which require a live YARN cluster
-can be disabled through the property `slider.funtest.enabled`
-
- <property>
- <name>slider.funtest.enabled</name>
- <value>false</value>
- </property>
-
-There is a configuration do do exactly this in
-`src/test/configs/offline/slider`:
-
- slider.conf.dir=src/test/configs/offline/slider
-
-Tests which do not require a live YARN cluster will still run;
-these verify that the `bin/slider` script works.
-
-### Non-mandatory options
-
-The following test options may be added to `slider-client.xml` if the defaults
-need to be changed
-
- <property>
- <name>slider.test.zkhosts</name>
- <description>comma separated list of ZK hosts</description>
- <value>localhost</value>
- </property>
-
- <property>
- <name>slider.test.thaw.wait.seconds</name>
- <description>Time to wait in seconds for a thaw to result in a running AM</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.freeze.wait.seconds</name>
- <description>Time to wait in seconds for a freeze to halt the cluster</description>
- <value>60000</value>
- </property>
-
- <property>
- <name>slider.test.timeout.millisec</name>
- <description>Time out in milliseconds before a test is considered to have failed.
- There are some maven properties which also define limits and may need adjusting</description>
- <value>180000</value>
- </property>
-
-
-
-Note that while the same properties need to be set in
-`slider-core/src/test/resources/slider-client.xml`, those tests take a file in the local
-filesystem -here a URI to a path visible across all nodes in the cluster are required
-the tests do not copy the .tar/.tar.gz files over. The application configuration
-directories may be local or remote -they are copied into the `.slider` directory
-during cluster creation.
-
-##
-
-Provider-specific parameters
+## Provider-specific parameters
An individual provider can pick up settings from their own
`src/test/resources/slider-client.xml` file, or the one in `slider-core`.
@@ -217,7 +124,10 @@ We strongly advice placing all the value
away
2. There's one place to keep all the configuration values in sync.
-### HBase Parameters
+### Agent Tests
+
+
+### HBase Tests
The HBase tests can be enabled or disabled
@@ -270,172 +180,3 @@ Optional parameters
<value>180000</value>
</property>
-
-
-### Configuring the YARN cluster for tests
-
-
-Here are the configuration options we use in `yarn-site.xml` for testing:
-
-These tell YARN to ignore memory requirements in allocating VMs, and
-to keep the log files around after an application run.
-
- <property>
- <name>yarn.scheduler.minimum-allocation-mb</name>
- <value>1</value>
- </property>
- <property>
- <description>Whether physical memory limits will be enforced for
- containers.
- </description>
- <name>yarn.nodemanager.pmem-check-enabled</name>
- <value>false</value>
- </property>
- <!-- we really don't want checking here-->
- <property>
- <name>yarn.nodemanager.vmem-check-enabled</name>
- <value>false</value>
- </property>
-
- <!-- how long after a failure to see what is left in the directory-->
- <property>
- <name>yarn.nodemanager.delete.debug-delay-sec</name>
- <value>60000</value>
- </property>
-
- <!--ten seconds before the process gets a -9 -->
- <property>
- <name>yarn.nodemanager.sleep-delay-before-sigkill.ms</name>
- <value>30000</value>
- </property>
-
-
-### Testing against a secure cluster
-
-To test against a secure cluster
-
-1. `slider-client.xml` must be configured as per [Security](security.html).
-1. the client must have the kerberos tokens issued so that the user running
-the tests has access to HDFS and YARN.
-
-If there are problems authenticating (including the cluster being offline)
-the tests appear to hang
-
-### Validating the configuration
-
- mvn test -Dtest=TestBuildSetup
-
-### Using relative paths in test configurations
-
-When you are sharing configurations across machines via SCM or similar,
-its impossible to have absolute paths in the configuration options to
-the location of items in the local filesystem (e.g. configuration directories).
-
-There's two techniques
-
-1. Keep the data in HDFS and refer to it there. This works if there is a shared,
-persistent HDFS cluster.
-
-1. Use the special property `slider.test.conf.dir` that is set to the path
-of the directory, and which can then be used to create an absolute path
-from paths relative to the configuration dir:
-
- <property>
- <name>slider.test.hbase.appconf</name>
- <description>Path to the directory containing the HBase application config</description>
- <value>file://${slider.test.conf.dir}/../hbase</value>
- </property>
-
-
-If the actual XML file path is required, a similar property
-`slider.test.conf.xml` is set.
-
-
-## Parallel execution
-
-Attempts to run test cases in parallel failed -even with a configuration
-to run methods in a class sequentially, but separate classes independently.
-
-Even after identifying and eliminating some unintended sharing of static
-mutable variables, trying to run test cases in parallel seemed to hang
-tests and produce timeouts.
-
-For this reason parallel tests have been disabled. To accelerate test runs
-through parallelization, run different tests on different hosts instead.
-
-## Other constraints
-
-* Port assignments SHOULD NOT be fixed, as this will cause clusters to fail if
-there are too many instances of a role on a same host, or if other tests are
-using the same port.
-* If a test does need to fix a port, it MUST be for a single instance of a role,
-and it must be different from all others. The assignment should be set in
-`org.apache.slider.funtest.itest.PortAssignments` so as to ensure uniqueness
-over time. Otherwise: use the value of `0` to allow the OS to assign free ports
-on demand.
-
-## Test Requirements
-
-
-1. Test cases should be written so that each class works with exactly one
-Slider-deployed cluster
-1. Every test MUST have its own cluster name -preferably derived from the
-classname.
-1. This cluster should be deployed in an `@BeforeClass` method.
-1. The `@AfterClass` method MUST tear this cluster down.
-1. Tests must skip their execution if functional tests -or the
-specific hbase or accumulo categories- are disabled.
-1. Tests within the suite (i.e. class) must be designed to be independent
--to work irrespectively of the ordering of other tests.
-
-## Running and debugging the functional tests.
-
-The functional tests all
-
-1. In the root `slider` directory, build a complete Slider release
-
- mvn install -DskipTests
-1. Start the YARN cluster/set up proxies to connect to it, etc.
-
-1. In the `slider-funtest` dir, run the test
-
- mvn test -Dtest=TestHBaseCreateCluster
-
-A common mistake during development is to rebuild the `slider-core` JARs
-then the `slider-funtest` tests without rebuilding the `slider-assembly`.
-In this situation, the tests are in sync with the latest build of the code
--including any bug fixes- but the scripts executed by those tests are
-of a previous build of `slider-core.jar`. As a result, the fixes are not picked
-up.
-
-#### To propagate changes in slider-core through to the funtest classes for
-testing, you must build/install all the slider packages from the root assembly.
-
- mvn clean install -DskipTests
-
-## Limitations of slider-funtest
-
-1. All tests run from a single client -workload can't scale
-1. Output from failed AM and containers aren't collected
-
-## Troubleshooting the functional tests
-
-1. If you are testing in a local VM and stops responding, it'll have been
-swapped out to RAM. Rebooting can help, but for a long term fix go through
-all the Hadoop configurations (HDFS, YARN, Zookeeper) and set their heaps to
-smaller numbers, like 256M each. Also: turn off unused services (hcat, oozie,
-webHDFS)
-
-1. The YARN UI will list the cluster launches -look for the one
-with a name close to the test and view its logs
-
-1. Container logs will appear "elsewhere". The log lists
-the containers used -you may be able to track the logs
-down from the specific nodes.
-
-1. If you browse the filesystem, look for the specific test clusters
-in `~/.slider/cluster/$testname`
-
-1. If you are using a secure cluster, make sure that the clocks
-are synchronized, and that you have a current token -`klist` will
-tell you this. In a VM: install and enable `ntp`.
Modified: incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml
URL: http://svn.apache.org/viewvc/incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml?rev=1593056&r1=1593055&r2=1593056&view=diff
==============================================================================
--- incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml (original)
+++ incubator/slider/trunk/src/test/clusters/sandbox/slider/slider-client.xml Wed May 7 16:26:15 2014
@@ -50,6 +50,19 @@
/etc/hadoop/conf,/usr/lib/hadoop/*,/usr/lib/hadoop/lib/*,/usr/lib/hadoop-hdfs/*,/usr/lib/hadoop-hdfs/lib/*,/usr/lib/hadoop-yarn/*,/usr/lib/hadoop-yarn/lib/*,/usr/lib/hadoop-mapreduce/*,/usr/lib/hadoop-mapreduce/lib/*
</value>
</property>
+
+ <property>
+ <name>slider.test.agent.enabled</name>
+ <description>Flag to enable/disable Agent tests</description>
+ <value>true</value>
+ </property>
+
+
+ <property>
+ <name>slider.test.hbase.enabled</name>
+ <description>Flag to enable/disable HBase tests</description>
+ <value>true</value>
+ </property>
<property>
<name>slider.test.hbase.tar</name>
@@ -91,7 +104,7 @@
<description>Path to the directory containing the Accumulo application
config
</description>
- <value>file:///home/slider/projects//Projects/slider/test-configs/sandbox/accumulo</value>
+ <value>file:///home/slider/projects/Projects/slider/test-configs/sandbox/accumulo</value>
</property>