You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/09/11 19:41:26 UTC
[1/4] git commit: SLIDER-280. Add unit and functional tests for
agents discovering AM after restart (Gour Saha via smohanty)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/slider_install_package b3f742d1b -> db673631d
SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/29ad5199
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/29ad5199
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/29ad5199
Branch: refs/heads/feature/slider_install_package
Commit: 29ad51990bd8d20e1a06798e0ef15aee0f890583
Parents: f1c1f70
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 11 10:39:52 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 11 10:40:00 2014 -0700
----------------------------------------------------------------------
.../apache/slider/providers/agent/TestAgentProviderService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/29ad5199/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 48a5f18..aa9dd60 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -440,7 +440,7 @@ public class TestAgentProviderService {
doReturn(true).when(mockAps).isMaster(anyString());
doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"),
eq("mockcontainer_1"), any(HeartBeatResponse.class),
- eq("scripts/hbase_master.py"));
+ eq("scripts/hbase_master.py"), eq(600L));
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
}
[3/4] git commit: SLIDER-280. Add unit and functional tests for
agents discovering AM after restart (Gour Saha via smohanty)
Posted by sm...@apache.org.
SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a509561f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a509561f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a509561f
Branch: refs/heads/feature/slider_install_package
Commit: a509561f0c710e61f1f7612e14bc6c70176d29f9
Parents: fa7130d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Aug 20 15:07:21 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 11 10:40:00 2014 -0700
----------------------------------------------------------------------
.../web/rest/agent/ExecutionCommand.java | 16 ++
.../appmaster/web/rest/agent/StatusCommand.java | 13 ++
.../agent/TestAgentProviderService.java | 214 +++++++++++++++++++
3 files changed, 243 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index c184e63..94aa29f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -179,4 +179,20 @@ public class ExecutionCommand {
public void setComponentName(String componentName) {
this.componentName = componentName;
}
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("ExecutionCommand [commandType=").append(commandType)
+ .append(", clusterName=").append(clusterName).append(", taskId=")
+ .append(taskId).append(", commandId=").append(commandId)
+ .append(", hostname=").append(hostname).append(", role=").append(role)
+ .append(", hostLevelParams=").append(hostLevelParams)
+ .append(", roleParams=").append(roleParams).append(", roleCommand=")
+ .append(roleCommand).append(", configurations=").append(configurations)
+ .append(", commandParams=").append(commandParams)
+ .append(", serviceName=").append(serviceName)
+ .append(", componentName=").append(componentName).append("]");
+ return builder.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
index 0610cac..52fb8dd 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
@@ -125,4 +125,17 @@ public class StatusCommand {
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("StatusCommand [agentCommandType=").append(agentCommandType)
+ .append(", clusterName=").append(clusterName).append(", serviceName=")
+ .append(serviceName).append(", componentName=").append(componentName)
+ .append(", configurations=").append(configurations)
+ .append(", commandParams=").append(commandParams)
+ .append(", hostLevelParams=").append(hostLevelParams)
+ .append(", roleCommand=").append(roleCommand).append("]");
+ return builder.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a509561f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 3711e12..48a5f18 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
@@ -41,6 +42,7 @@ import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ContainerLauncher;
+import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.providers.agent.application.metadata.CommandScript;
@@ -58,6 +60,7 @@ import org.apache.slider.server.appmaster.model.mock.MockFileSystem;
import org.apache.slider.server.appmaster.model.mock.MockNodeId;
import org.apache.slider.server.appmaster.state.ProviderAppState;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
+import org.apache.slider.server.appmaster.web.rest.agent.AgentCommandType;
import org.apache.slider.server.appmaster.web.rest.agent.CommandReport;
import org.apache.slider.server.appmaster.web.rest.agent.ComponentStatus;
import org.apache.slider.server.appmaster.web.rest.agent.ExecutionCommand;
@@ -381,6 +384,217 @@ public class TestAgentProviderService {
Assert.assertEquals(2, hbr.getResponseId());
}
+ private AggregateConf prepareConfForAgentStateTests() {
+ ConfTree tree = new ConfTree();
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+
+ AggregateConf instanceDefinition = new AggregateConf();
+ instanceDefinition.setInternal(tree);
+ instanceDefinition.setAppConf(tree);
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.APP_DEF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.AGENT_CONF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions()
+ .put(AgentKeys.AGENT_VERSION, ".");
+ return instanceDefinition;
+ }
+
+ private AgentProviderService prepareProviderServiceForAgentStateTests()
+ throws IOException {
+ ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
+ Container container = createNiceMock(Container.class);
+ String role = "HBASE_MASTER";
+ SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
+ FileSystem mockFs = new MockFileSystem();
+ expect(sliderFileSystem.getFileSystem()).andReturn(
+ new FilterFileSystem(mockFs)).anyTimes();
+ expect(
+ sliderFileSystem.createAmResource(anyObject(Path.class),
+ anyObject(LocalResourceType.class))).andReturn(
+ createNiceMock(LocalResource.class)).anyTimes();
+ expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
+ expect(container.getNodeId()).andReturn(new MockNodeId("localhost"))
+ .anyTimes();
+ expect(container.getPriority()).andReturn(Priority.newInstance(1));
+
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+
+ doReturn(access).when(mockAps).getAmState();
+ doReturn("scripts/hbase_master.py").when(mockAps)
+ .getScriptPathFromMetainfo(anyString());
+ Metainfo metainfo = new Metainfo();
+ Application application = new Application();
+ metainfo.setApplication(application);
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(
+ any(SliderFileSystem.class), anyString());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+
+ Configuration conf = new Configuration();
+ conf.set(SliderXmlConfKeys.REGISTRY_PATH,
+ SliderXmlConfKeys.DEFAULT_REGISTRY_PATH);
+
+ try {
+ doReturn(true).when(mockAps).isMaster(anyString());
+ doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"),
+ eq("mockcontainer_1"), any(HeartBeatResponse.class),
+ eq("scripts/hbase_master.py"));
+ doReturn(conf).when(mockAps).getConfig();
+ } catch (SliderException e) {
+ }
+
+ doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(),
+ anyString(), anyMap());
+ expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ ClusterDescription desc = new ClusterDescription();
+ desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
+ desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getClusterStatus()).andReturn(desc).anyTimes();
+
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getOrAddComponent("HBASE_MASTER")
+ .put(AgentKeys.WAIT_HEARTBEAT, "0");
+ treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ treeOps.set("java_home", "/usr/jdk7/");
+ treeOps.set("site.fs.defaultFS", "hdfs://c6409.ambari.apache.org:8020");
+ treeOps.set(InternalKeys.INTERNAL_DATA_DIR_PATH, "hdfs://c6409.ambari.apache.org:8020/user/yarn/.slider/cluster/cl1/data");
+ expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
+ expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
+ replay(access, ctx, container, sliderFileSystem);
+
+ List<Container> containers = new ArrayList<Container>();
+ containers.add(container);
+ Map<Integer, ProviderRole> providerRoleMap = new HashMap<Integer, ProviderRole>();
+ ProviderRole providerRole = new ProviderRole(role, 1);
+ providerRoleMap.put(1, providerRole);
+ mockAps.rebuildContainerDetails(containers, "mockcontainer_1",
+ providerRoleMap);
+ return mockAps;
+ }
+
+ @Test
+ public void testAgentStateStarted() throws IOException, SliderException {
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
+ AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setHostname("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap<String,String>();
+ ports.put("a","100");
+ reg.setAllocatedPorts(ports);
+
+ // Simulating agent in STARTED state
+ reg.setActualState(State.STARTED);
+
+ mockAps.initializeApplicationConfiguration(instanceDefinition,
+ null);
+
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertNotNull(
+ "Status command from AM cannot be null when agent's actualState "
+ + "is set to STARTED during registration", hbr.getStatusCommands());
+ Assert.assertTrue(
+ "Status command from AM cannot be empty when agent's actualState "
+ + "is set to STARTED during registration", hbr.getStatusCommands()
+ .size() > 0);
+ Assert.assertEquals(
+ "AM should directly send a STATUS request if agent's actualState is "
+ + "set to STARTED during registration",
+ AgentCommandType.STATUS_COMMAND, hbr.getStatusCommands().get(0)
+ .getCommandType());
+ Assert.assertEquals(
+ "AM should directly request for CONFIG if agent's actualState is "
+ + "set to STARTED during registration",
+ "GET_CONFIG", hbr.getStatusCommands().get(0)
+ .getRoleCommand());
+ Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent());
+ }
+
+ @Test
+ public void testAgentStateInstalled() throws IOException, SliderException {
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
+ AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
+
+ Metainfo metainfo = new Metainfo();
+ Application application = new Application();
+ CommandOrder cmdOrder = new CommandOrder();
+ cmdOrder.setCommand("HBASE_MASTER-START");
+ cmdOrder.setRequires("HBASE_MASTER-INSTALLED");
+ application.addCommandOrder(cmdOrder);
+ metainfo.setApplication(application);
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(
+ any(SliderFileSystem.class), anyString());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+ doNothing().when(mockAps).addRoleRelatedTokens(anyMap());
+
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setHostname("mockcontainer_1___HBASE_MASTER");
+ Map<String,String> ports = new HashMap<String,String>();
+ ports.put("a","100");
+ reg.setAllocatedPorts(ports);
+
+ // Simulating agent in INSTALLED state
+ reg.setActualState(State.INSTALLED);
+
+ mockAps.initializeApplicationConfiguration(instanceDefinition,
+ null);
+
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ Mockito.verify(mockAps, Mockito.times(1)).processAllocatedPorts(
+ anyString(),
+ anyString(),
+ anyString(),
+ anyMap()
+ );
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertNotNull(
+ "Execution command from AM cannot be null when agent's actualState "
+ + "is set to INSTALLED during registration", hbr.getExecutionCommands());
+ Assert.assertTrue(
+ "Execution command from AM cannot be empty when agent's actualState "
+ + "is set to INSTALLED during registration", hbr.getExecutionCommands()
+ .size() > 0);
+ Assert.assertEquals(
+ "AM should send an EXECUTION command if agent's actualState is "
+ + "set to INSTALLED during registration",
+ AgentCommandType.EXECUTION_COMMAND, hbr.getExecutionCommands().get(0)
+ .getCommandType());
+ Assert.assertEquals(
+ "AM should request for START if agent's actualState is "
+ + "set to INSTALLED during registration",
+ "START", hbr.getExecutionCommands().get(0)
+ .getRoleCommand());
+ Assert.assertFalse("AM cannot ask agent to restart", hbr.isRestartAgent());
+ }
+
@Test
public void testRoleHostMapping() throws Exception {
AgentProviderService aps = new AgentProviderService();
[2/4] git commit: SLIDER-280. Add unit and functional tests for
agents discovering AM after restart (Gour Saha via smohanty)
Posted by sm...@apache.org.
SLIDER-280. Add unit and functional tests for agents discovering AM after restart (Gour Saha via smohanty)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f1c1f705
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f1c1f705
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f1c1f705
Branch: refs/heads/feature/slider_install_package
Commit: f1c1f705fe24af7e768fcf706370d0f0bb625f11
Parents: a509561
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 11 10:18:23 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 11 10:40:00 2014 -0700
----------------------------------------------------------------------
.../command-logger/slider-pkg/appConfig.json | 2 +-
.../command-logger/slider-pkg/metainfo.xml | 2 +-
.../test_command_log/appConfig.json | 2 +-
slider-funtest/pom.xml | 7 +-
slider-funtest/pom.xml.orig | 357 +++++++++++++++++++
.../org/apache/chaos/remote/BuildPidPath.groovy | 54 +++
.../org/apache/chaos/remote/Clustat.groovy | 64 ++++
.../apache/chaos/remote/JschToCommonsLog.groovy | 78 ++++
.../chaos/remote/RemoteDaemonOperations.groovy | 60 ++++
.../org/apache/chaos/remote/RemoteServer.groovy | 188 ++++++++++
.../org/apache/chaos/remote/SshCommands.groovy | 190 ++++++++++
.../funtest/lifecycle/AMFailuresIT.groovy | 191 ++++++++++
12 files changed, 1191 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/appConfig.json b/app-packages/command-logger/slider-pkg/appConfig.json
index bfdffb8..791043f 100644
--- a/app-packages/command-logger/slider-pkg/appConfig.json
+++ b/app-packages/command-logger/slider-pkg/appConfig.json
@@ -11,7 +11,7 @@
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/app-packages/command-logger/slider-pkg/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml b/app-packages/command-logger/slider-pkg/metainfo.xml
index 976fa81..b8de9a3 100644
--- a/app-packages/command-logger/slider-pkg/metainfo.xml
+++ b/app-packages/command-logger/slider-pkg/metainfo.xml
@@ -42,7 +42,7 @@
<packages>
<package>
<type>tarball</type>
- <name>files/command_log.tar</name>
+ <name>files/command-logger.tar</name>
</package>
</packages>
</osSpecific>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-core/src/test/app_packages/test_command_log/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/appConfig.json b/slider-core/src/test/app_packages/test_command_log/appConfig.json
index 684924d..7586a0c 100644
--- a/slider-core/src/test/app_packages/test_command_log/appConfig.json
+++ b/slider-core/src/test/app_packages/test_command_log/appConfig.json
@@ -12,7 +12,7 @@
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/command-logger",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
- "site.cl-site.logfile.location": "${AGENT_LOG_ROOT}/app/log/operations.log",
+ "site.cl-site.logfile.location": "${AGENT_WORK_ROOT}/app/install/command-logger-app/operations.log",
"site.cl-site.datetime.format": "%A, %d. %B %Y %I:%M%p"
},
"components": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/pom.xml
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml b/slider-funtest/pom.xml
index 124251f..55b6351 100644
--- a/slider-funtest/pom.xml
+++ b/slider-funtest/pom.xml
@@ -121,6 +121,7 @@
<test.app.pkg.file>apache-slider-command-logger.zip</test.app.pkg.file>
<test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
<test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
+ <vagrant.current.working.dir>/Users/gsaha/work/workspace/ambari-vagrant/centos6.4</vagrant.current.working.dir>
</systemPropertyVariables>
</configuration>
</plugin>
@@ -288,7 +289,11 @@
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
</dependency>
-
+ <dependency>
+ <groupId>com.jcraft</groupId>
+ <artifactId>jsch</artifactId>
+ <version>0.1.51</version>
+ </dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/pom.xml.orig
----------------------------------------------------------------------
diff --git a/slider-funtest/pom.xml.orig b/slider-funtest/pom.xml.orig
new file mode 100644
index 0000000..124251f
--- /dev/null
+++ b/slider-funtest/pom.xml.orig
@@ -0,0 +1,357 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <artifactId>slider-funtest</artifactId>
+ <name>Slider Functional Tests</name>
+ <packaging>jar</packaging>
+ <description>
+ Functional testing for slider: testing is always fun
+ </description>
+ <parent>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider</artifactId>
+ <version>0.51.0-incubating-SNAPSHOT</version>
+ </parent>
+ <properties>
+ <work.dir>package-tmp</work.dir>
+ </properties>
+
+ <build>
+
+ <!-- resources are filtered for dynamic updates. This gets build info in-->
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+
+ <plugins>
+
+ <!--read in a build.properties file if defined-->
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>properties-maven-plugin</artifactId>
+ <version>${maven.properties.version}</version>
+ <executions>
+ <execution>
+ <phase>initialize</phase>
+ <goals>
+ <goal>read-project-properties</goal>
+ </goals>
+ <configuration>
+ <quiet>true</quiet>
+ <files>
+ <file>build.properties</file>
+ <file>../build.properties</file>
+ </files>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-compiler</artifactId>
+ <version>${groovy-eclipse-compiler.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-eclipse-batch</artifactId>
+ <version>${groovy-eclipse-batch.version}</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+
+
+ <!-- test -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ <version>${maven-failsafe-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>run-integration-tests</id>
+ <goals>
+ <goal>integration-test</goal>
+ <goal>verify</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <!--mvn process fork options-->
+ <reuseForks>${test.reuseForks}</reuseForks>
+ <forkMode>${test.forkMode}</forkMode>
+ <forkCount>1</forkCount>
+ <forkedProcessTimeoutInSeconds>${test.forkedProcessTimeoutInSeconds}
+ </forkedProcessTimeoutInSeconds>
+ <threadCount>1</threadCount>
+ <argLine>${test.argLine}</argLine>
+ <failIfNoTests>${test.funtests.failIfNoTests}</failIfNoTests>
+
+ <trimStackTrace>false</trimStackTrace>
+ <redirectTestOutputToFile>${build.redirect.test.output.to.file}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.net.preferIPv4Stack>true</java.net.preferIPv4Stack>
+ <java.awt.headless>true</java.awt.headless>
+ <java.security.krb5.realm>${slider.test.java.security.krb5.realm}</java.security.krb5.realm>
+ <java.security.krb5.kdc>${slider.test.java.security.krb5.kdc}</java.security.krb5.kdc>
+ <!-- this property must be supplied-->
+ <slider.conf.dir>${slider.conf.dir}</slider.conf.dir>
+ <slider.bin.dir>../slider-assembly/target/slider-${project.version}-all/slider-${project.version}</slider.bin.dir>
+ <test.app.pkg.dir>../app-packages/command-logger/slider-pkg/target</test.app.pkg.dir>
+ <test.app.pkg.file>apache-slider-command-logger.zip</test.app.pkg.file>
+ <test.app.resource>../slider-core/src/test/app_packages/test_command_log/resources.json</test.app.resource>
+ <test.app.template>../slider-core/src/test/app_packages/test_command_log/appConfig.json</test.app.template>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <version>${maven-dependency-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>copy-dependencies</id>
+ <phase>process-resources</phase>
+ <goals>
+ <goal>copy-dependencies</goal>
+ </goals>
+ <configuration>
+ <includeArtifactIds>apache-slider-command-logger</includeArtifactIds>
+ <includeTypes>zip</includeTypes>
+ <outputDirectory>${project.build.directory}/${work.dir}</outputDirectory>
+ </configuration>
+ </execution>
+ <execution>
+ <id>copy</id>
+ <phase>test</phase>
+ <goals>
+ <goal>copy</goal>
+ </goals>
+ <configuration>
+ <artifactItems>
+ <artifactItem>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>apache-slider-command-logger</artifactId>
+ <type>zip</type>
+ <overWrite>false</overWrite>
+ <outputDirectory>${project.build.directory}/${work.dir}</outputDirectory>
+ <destFileName>apache-slider-command-logger.zip</destFileName>
+ </artifactItem>
+ </artifactItems>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+
+ </plugins>
+ </build>
+
+ <reporting>
+ <plugins>
+
+
+
+ </plugins>
+ </reporting>
+
+ <dependencies>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-core</artifactId>
+ <type>test-jar</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-assembly</artifactId>
+ <classifier>all</classifier>
+ <type>tar.gz</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>apache-slider-command-logger</artifactId>
+ <version>${project.version}</version>
+ <type>zip</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-common</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <classifier>tests</classifier>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-hadoop2-compat</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-server</artifactId>
+ <classifier>tests</classifier>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.bigtop.itest</groupId>
+ <artifactId>itest-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.slider</groupId>
+ <artifactId>slider-agent</artifactId>
+ <scope>test</scope>
+ <type>tar.gz</type>
+ </dependency>
+
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.codehaus.groovy</groupId>
+ <artifactId>groovy-all</artifactId>
+ </dependency>
+
+
+ </dependencies>
+
+ <profiles>
+ <profile>
+ <id>tests-on-from-CLI</id>
+ <activation>
+ <property>
+ <name>slider.conf.dir</name>
+ </property>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on-from-build.properties</id>
+ <activation>
+ <file>
+ <exists>../build.properties</exists>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <properties>
+ <maven.test.skip>true</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/regionservers</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy
new file mode 100644
index 0000000..3daa2f4
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/BuildPidPath.groovy
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import org.apache.hadoop.conf.Configured
+
+/**
+ *
+ */
+class BuildPidPath extends Configured {
+
+ public static final String PID_DIR = "/var/run/hadoop"
+
+
+ String buildPidName(String user, String service) {
+ "${user}-${service}"
+ }
+
+ File buildPidPath(String pidName) {
+ new File(PID_DIR, "hadoop-${pidName}.pid")
+ }
+
+ /**
+ * Find the Pid string value of a hadoop process
+ * @param user user ID
+ * @param service
+ * @return the pid as a string or null for none found. If the string
+ * is empty it means the pid file was empty -there's no validation of
+ * it.
+ */
+ String findPid(String pidName) {
+ File pidFile = buildPidPath(pidName)
+ if (!pidFile.exists()) {
+ return null
+ } else {
+ return pidFile.text.trim()
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy
new file mode 100644
index 0000000..742aa2e
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/Clustat.groovy
@@ -0,0 +1,64 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.chaos.remote
+
+/**
+ * Parses RHAT HA Clustat files pulled back over the network
+ */
+class Clustat {
+
+ Node xml;
+
+ Clustat(String source) {
+ xml = new XmlParser().parseText(source)
+ }
+
+/**
+ * ser
+ * @param service something like service:NameNodeService
+ * @return the owner
+ */
+ Node serviceGroup(String service) {
+ NodeList groups = xml.groups
+ def found = groups.group.findAll { it.@name == service}
+ if (found.empty) {
+ return null
+ }
+ Node serviceGroup = found[0]
+ return serviceGroup
+ }
+
+ String hostRunningService(String service) {
+ Node serviceGroup = serviceGroup("service:NameNodeService")
+ return serviceGroup?.@owner
+ }
+
+ /**
+ * Get all the nodes in the cluster
+ * @return the XML nodes element; every node is an elt underneath with @name==server
+ */
+ NodeList clusterNodes() {
+ return xml.nodes.node
+ }
+
+ Node clusterNode(String name) {
+ def all = clusterNodes() findAll { it.@name == name}
+ return all.size() > 0 ? all[0] : null
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy
new file mode 100644
index 0000000..49eb856
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/JschToCommonsLog.groovy
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.Logger
+import org.apache.commons.logging.Log;
+
+/**
+ * relay from jsch to commons logging.
+ * JSCH is very chatty, so it is downgraded to debug on its info ops
+ */
+
+
+class JschToCommonsLog implements Logger {
+
+
+ final Log log
+
+ JschToCommonsLog(Log log) {
+ this.log = log
+ }
+
+ @Override
+ void log(int level, String message) {
+ switch (level) {
+ case FATAL:
+ log.fatal(message);
+ break;
+ case ERROR:
+ log.error(message);
+ break;
+ case WARN:
+ log.info(message);
+ break;
+ case INFO:
+ case DEBUG:
+ if (message.contains("Caught an exception")) {
+ log.debug(message, new Exception('here'))
+ }
+ log.debug(message);
+ break;
+ default:
+ log.info(message);
+ break;
+ }
+ }
+
+ @Override
+ boolean isEnabled(int level) {
+ switch (level) {
+ case FATAL:
+ return log.isFatalEnabled()
+ case ERROR:
+ return log.isErrorEnabled()
+ case WARN:
+ return log.isInfoEnabled()
+ case INFO:
+ case DEBUG:
+ return log.isDebugEnabled()
+ }
+ return false
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy
new file mode 100644
index 0000000..946fbf7
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteDaemonOperations.groovy
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+/**
+ * Operations to perform on a remote init.d service by ssh-ing in and issuing
+ * the command as root.
+ * Use the full name, such as "hadoop-namenode"
+ */
+class RemoteDaemonOperations {
+
+ public static final int TIMEOUT = 0
+ RemoteServer server
+ String name;
+
+ RemoteDaemonOperations(RemoteServer server, String name) {
+ this.server = server
+ this.name = name
+ }
+
+ def start() {
+ exec("start")
+ }
+
+ def stop() {
+ exec("stop")
+ }
+
+ def restart() {
+ exec("restart")
+ }
+
+ def status() {
+ exec("status")
+ }
+
+ /**
+ *
+ * @param action action to exec
+ * @return ( rv , out )
+ */
+ List exec(String action) {
+ server.exec([["service", name, action], "exit"])
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy
new file mode 100644
index 0000000..5dc9dab
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/RemoteServer.groovy
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.JSch
+import com.jcraft.jsch.Session
+import com.jcraft.jsch.UIKeyboardInteractive
+import com.jcraft.jsch.UserInfo
+import groovy.util.logging.Commons
+import org.apache.commons.logging.LogFactory
+
+/**
+ * A remote server you can SSH to
+ */
+@Commons
+class RemoteServer implements UserInfo, UIKeyboardInteractive {
+
+
+ boolean publicKeyAuth = true
+ File publicKeyFile
+ File knownHosts
+ String username
+ String password
+ String host
+ int port = 22
+ int timeout = 60000
+ int connectTimeout = 10000
+ private JSch jsch
+
+ @Override
+ String getPassphrase() {
+ return password
+ }
+
+ @Override
+ boolean promptPassword(String message) {
+ log.debug("Password prompt: $message")
+ return false
+ }
+
+ @Override
+ boolean promptPassphrase(String message) {
+ log.debug("Passphrase prompt: $message")
+ return false
+ }
+
+ /**
+ * This is for the "do you trust this host" message. We do
+ * @param message
+ * @return
+ */
+ @Override
+ boolean promptYesNo(String message) {
+ log.debug("YesNo prompt: $message")
+ return true
+ }
+
+ @Override
+ String[] promptKeyboardInteractive(String destination, String name, String instruction, String[] prompt, boolean[] echo) {
+ return new String[0]
+ }
+
+ @Override
+ void showMessage(String message) {
+ log.info(message)
+ }
+
+ private synchronized JSch createJschInstance() {
+ if (jsch) {
+ return jsch
+ }
+ jsch = new JSch()
+ //do not try and convert a groovy shortcut as overloaded methods then cause confusion
+ jsch.setLogger(new JschToCommonsLog(LogFactory.getLog("org.apache.chaos.remote.RemoteServer.ssh")))
+ if (publicKeyAuth) {
+ assert publicKeyFile
+ assert publicKeyFile.exists() && publicKeyFile.canRead()
+ jsch.addIdentity(publicKeyFile.toString(), (String) password);
+ } else {
+ assert password != null
+ }
+ if (knownHosts) {
+ assert knownHosts.exists() && knownHosts.canRead()
+ jsch.setKnownHosts(knownHosts.toString())
+ }
+ jsch
+ }
+
+ public JSch getJschInstance() {
+ return jsch ?: createJschInstance()
+ }
+
+ Session connect(JSch jsch, String host, int port) {
+ assert host
+ Session session = jsch.getSession(username, host, port)
+ session.userInfo = this
+ if (!publicKeyAuth) {
+ session.setPassword(password)
+ }
+ session.timeout = timeout
+ session.connect(connectTimeout)
+ session
+ }
+
+ /**
+ * Connect to the (host, port)
+ * @return a new session
+ */
+ Session connect() {
+ return connect(jschInstance, host, port)
+ }
+
+ /**
+ * Connect to the server and issue the commands, one per line. Nest arguments
+ * in inner lists for better structure
+ * @param args multi-dimensional list
+ * @return ( return value , output )
+ */
+ List exec(List args) {
+ def session = connect()
+ SshCommands cmds = new SshCommands(session)
+ return cmds.exec(args)
+ }
+
+ List command(String command) {
+ def session = connect()
+ SshCommands cmds = new SshCommands(session)
+ def (status, text) = cmds.command([command])
+ log.info("$status : $text")
+ [status, text]
+ }
+
+ /**
+ * Connect to a host and kill a process there
+ * @param pidFile the PID file
+ * @param signal the signal
+ * @return the outcome of the command
+ */
+ List kill(int signal, String pidFile) {
+ return command("kill -$signal `cat $pidFile`");
+ }
+
+ Clustat clustat() {
+ def (status, text) = command("clustat -x")
+ if (status != 0) {
+ throw new IOException("Clustat command failed [$status]: $text")
+ }
+ Clustat clustat = new Clustat(text)
+ return clustat
+ }
+
+ void waitForServerLive(int timeout) {
+ long endtime = System.currentTimeMillis() + timeout
+ boolean live = false;
+
+ while (!live && System.currentTimeMillis() < endtime) {
+ try {
+ command("true")
+ live = true
+ break
+ } catch (IOException ignored) {
+ Thread.sleep(1000)
+ }
+
+ }
+
+ }
+
+ @Override
+ String toString() {
+ return "Remote server user=$username @ host=$host"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy b/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy
new file mode 100644
index 0000000..334475d
--- /dev/null
+++ b/slider-funtest/src/main/groovy/org/apache/chaos/remote/SshCommands.groovy
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.chaos.remote
+
+import com.jcraft.jsch.ChannelExec
+import com.jcraft.jsch.ChannelShell
+import com.jcraft.jsch.Session
+import groovy.util.logging.Commons
+import org.apache.tools.ant.util.KeepAliveOutputStream
+import org.apache.tools.ant.util.TeeOutputStream
+
+/**
+ * Send lists strings of commands
+ */
+@Commons
+class SshCommands {
+
+ private final Session session
+ private volatile boolean stop = false
+ private volatile Thread thread
+ long commandsTimeout = 10000
+
+ SshCommands(Session session) {
+ this.session = session
+ }
+
+ private synchronized void begin() {
+ thread = Thread.currentThread()
+ Thread.start("ssh command waker") {
+ sleep(commandsTimeout);
+ stop()
+ }
+ }
+
+ private synchronized end() {
+ thread = null
+ stop = false
+ }
+
+ public synchronized void stop() {
+ if (!stop) {
+ stop = true;
+ if (thread) {
+ thread.interrupt()
+ }
+ }
+ }
+
+ /**
+ * Execute a list of commands, return the exit status. Output goes to the
+ * output stream
+ * @param args , one per line -all converted to strings first. If the list
+ * element is itself a collection, it is flattened then joined with spaces. This
+ * can be used to build up more complex commands.
+ * @return the result of the execution -the last response from the command
+ */
+ List exec(List args) {
+ assert session && session.connected
+ List<String> commandSet = args.collect() {
+ (it instanceof Collection) ? (it.flatten().join(" ")) : it.toString()
+ }
+ String commandLine = commandSet.join("\n")
+ log.info("commands: $commandLine")
+ ChannelShell channel = null
+ try {
+ begin()
+ channel = (ChannelShell) session.openChannel("shell")
+
+ //connect triggers a linkup and the streaming
+ channel.connect();
+ assert channel.connected
+ channel.pty = true
+ OutputStream toServer = channel.outputStream
+ Reader fromServer = new BufferedReader(
+ new InputStreamReader(channel.inputStream, "UTF-8"));
+ toServer.write(commandLine.bytes)
+ toServer.flush()
+ StringBuilder builder = new StringBuilder()
+
+ String line
+ while (!channel.closed && !stop) {
+ line = fromServer.readLine()
+ StringBuilder append = builder.append(line)
+ log.debug(line);
+ }
+
+ int status = channel.exitStatus
+ if (Thread.interrupted()) {
+ log.debug("Interrupted while waiting for the end of the SSH stream")
+ throw new InterruptedIOException()
+ }
+ return [status, builder.toString()]
+ } finally {
+ end()
+ boolean interrupted = Thread.interrupted()
+ channel?.disconnect()
+ }
+ }
+ /**
+ * Execute a single of commands, return the exit status. Output goes to the
+ * output stream
+ * @param args -a list that is flattened then joined with spaces. This
+ * can be used to build up more complex commands.
+ * @return the result of the execution -the response from the command
+ */
+ List command(String arg) {
+ return command([arg])
+ }
+
+ List command(List args) {
+ assert session && session.connected
+ session.timeout = commandsTimeout
+ String commandLine = args.join(" ")
+ log.info("commands: $commandLine")
+ ChannelExec channel = null
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ TeeOutputStream tee =
+ new TeeOutputStream(out,
+ KeepAliveOutputStream.wrapSystemOut());
+
+ try {
+ begin()
+ channel = (ChannelExec) session.openChannel("exec")
+
+
+ channel.setCommand(commandLine);
+ channel.setOutputStream(tee);
+ channel.setExtOutputStream(tee);
+ channel.pty = true
+
+ //connect triggers a linkup and the streaming
+ channel.connect();
+ assert channel.connected
+
+ Thread thread = Thread.start() {
+ while (!channel.isClosed()) {
+ if (thread == null) {
+ return;
+ }
+ try {
+ sleep(500);
+ } catch (Exception e) {
+ log.debug("In channel thread $e", e)
+ // ignored
+ }
+ }
+ }
+
+ thread.join(commandsTimeout);
+ if (thread.isAlive()) {
+ // ran out of time
+ thread = null;
+ throw new IOException("Timeout executing command " + commandLine);
+ }
+
+ String textResult = out.toString("UTF-8")
+ int status = channel.exitStatus
+ if (Thread.interrupted()) {
+ log.debug("Interrupted while waiting for the end of the SSH stream")
+ throw new InterruptedIOException()
+ }
+ return [status, textResult]
+ } finally {
+ end()
+ channel?.disconnect()
+ }
+ }
+
+/* List exec(List args, int timeout) {
+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
+ int rv = exec(outputStream, args, timeout)
+ String outstr = outputStream.toString("UTF-8")
+ [rv, outstr]
+}*/
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f1c1f705/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
new file mode 100644
index 0000000..b92207c
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AMFailuresIT.groovy
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import com.jcraft.jsch.Session
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.bigtop.itest.shell.Shell
+import org.apache.chaos.remote.RemoteServer
+import org.apache.chaos.remote.SshCommands
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AMFailuresIT extends AgentCommandTestBase
+implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ private static String COMMAND_LOGGER = "COMMAND_LOGGER"
+ private static String APPLICATION_NAME = "am-started-agents-started"
+ public static final String TEST_REMOTE_SSH_KEY = "test.remote.ssh.key"
+ public static final String VAGRANT_CWD = "vagrant.current.working.dir"
+ File sshkey
+
+ @After
+ public void destroyCluster() {
+ cleanup(APPLICATION_NAME)
+ }
+
+ @Test
+ public void testAMKilledWithStateAMStartedAgentsStarted() throws Throwable {
+ cleanup(APPLICATION_NAME)
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CREATE, APPLICATION_NAME,
+ ARG_IMAGE, agentTarballPath.toString(),
+ ARG_TEMPLATE, APP_TEMPLATE,
+ ARG_RESOURCES, APP_RESOURCE
+ ])
+ logShell(shell)
+
+ ensureApplicationIsUp(APPLICATION_NAME)
+ repeatUntilTrue(this.&hasContainerCountExceeded, 15, 1000 * 10, ['arg1': '1']);
+ // Wait for 20 secs for AM and agent to both reach STARTED state
+ sleep(1000 * 20)
+
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+ assertComponentCount(COMMAND_LOGGER, 1, shell)
+ String live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
+ assert live != null && live.isInteger() && live.toInteger() == 1,
+ 'At least 1 container must be live now'
+
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+ assertSuccess(shell)
+
+ // Now kill the AM
+ log.info("Killing AM now ...")
+// killAMUsingJsch()
+// killAMUsingAmSuicide()
+ killAMUsingVagrantShell()
+
+ // Check that the application is not running (and is in ACCEPTED state)
+ assert isApplicationInState("ACCEPTED", APPLICATION_NAME),
+ 'App should be in ACCEPTED state (since AM got killed)'
+ log.info("After AM KILL: application {} is in ACCEPTED state", APPLICATION_NAME)
+
+ // Wait until AM comes back up and verify container count again
+ ensureApplicationIsUp(APPLICATION_NAME)
+
+ // There should be exactly 1 live container
+ shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ live = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.live"] as String[])
+ assert live != null && live.isInteger() && live.toInteger() == 1,
+ 'At least 1 container must be live now'
+ log.info("After AM KILL: agent container is still live")
+
+ // No new containers should be requested for the agents
+ String requested = findLineEntryValue(shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ assert requested != null && requested.isInteger() && requested.toInteger() == 0,
+ 'No new agent containers should be requested'
+ log.info("After AM KILL: no new agent containers were requested")
+
+ assert isApplicationInState("RUNNING", APPLICATION_NAME), 'App is not running.'
+ assertSuccess(shell)
+ }
+
+ boolean hasContainerCountExceeded(Map<String, String> args) {
+ int expectedCount = args['arg1'].toInteger();
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_STATUS,
+ APPLICATION_NAME])
+
+ String requested = findLineEntryValue(
+ shell, ["statistics", COMMAND_LOGGER, "containers.requested"] as String[])
+ if (requested != null && requested.isInteger() && requested.toInteger() >= expectedCount) {
+ return true
+ }
+
+ return false
+ }
+
+ protected void killAMUsingAmSuicide() {
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_AM_SUICIDE,
+ ARG_MESSAGE, "testAMRestart",
+ APPLICATION_NAME])
+ logShell(shell)
+ assertSuccess(shell)
+ }
+
+ protected void killAMUsingVagrantShell() {
+ String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
+ assert hostname != null && !hostname.isEmpty()
+ String vagrantVmName = hostname.split("\\.")[0]
+
+ String vagrantCwd = sysprop(VAGRANT_CWD)
+ log.info("VAGRANT_CWD = {}", vagrantCwd)
+ File dirCheck = new File(vagrantCwd)
+ assert dirCheck.exists(), "Please set $VAGRANT_CWD to the directory which contains the Vagrantfile"
+
+ String cmd = "export VAGRANT_CWD=$vagrantCwd; /usr/bin/vagrant ssh -c \"sudo -u root runuser -l yarn " +
+ "-c \\\"ps -ef | grep SliderAppMaster | egrep -v 'grep|bash' | sed 's/^[a-z]* *\\([^ ]*\\) *.*/\\1/' " +
+ "| xargs kill -9\\\"\" $vagrantVmName 2>/dev/null"
+ log.info("Vagrant Shell Command = {}", cmd)
+
+ Shell vagrantShell = new Shell('/bin/bash -s')
+ vagrantShell.exec(cmd)
+ }
+
+ protected void killAMUsingJsch() {
+ String hostname = SLIDER_CONFIG.get(YarnConfiguration.RM_ADDRESS).split(":")[0]
+ String user = UserGroupInformation.currentUser
+ assert hostname != null && !hostname.isEmpty()
+ assert user != null && !user.isEmpty()
+
+ bindSSHKey()
+ RemoteServer remoteServer = new RemoteServer(
+ host: hostname,
+ username: user,
+ publicKeyFile: sshkey)
+ Session session = remoteServer.connect()
+ SshCommands cmds = new SshCommands(session)
+// def (String rv, String out) = cmds.command([
+// "ps -ef | grep SliderAppMaster | egrep -v 'grep|bash' | xargs kill -9"
+// ])
+// assert rv == 0
+// log.info("Kill cmd output", out)
+ }
+
+ /**
+ * Bind to the SSH key -assert that the file actually exists
+ */
+ protected void bindSSHKey() {
+ sshkey = new File(sysprop("user.home"), ".ssh/id_rsa")
+ sshkey = new File(sysprop(TEST_REMOTE_SSH_KEY), sshkey.toString())
+ assert sshkey.exists()
+ }
+}
[4/4] git commit: Merge branch 'develop' into
feature/slider_install_package
Posted by sm...@apache.org.
Merge branch 'develop' into feature/slider_install_package
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/db673631
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/db673631
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/db673631
Branch: refs/heads/feature/slider_install_package
Commit: db673631db77b9ef1e27ce443d653e2e8211d9cb
Parents: b3f742d 29ad519
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Sep 11 10:41:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Sep 11 10:41:13 2014 -0700
----------------------------------------------------------------------
.../command-logger/slider-pkg/appConfig.json | 2 +-
.../command-logger/slider-pkg/metainfo.xml | 2 +-
.../web/rest/agent/ExecutionCommand.java | 16 +
.../appmaster/web/rest/agent/StatusCommand.java | 13 +
.../test_command_log/appConfig.json | 2 +-
.../agent/TestAgentProviderService.java | 214 +++++++++++
slider-funtest/pom.xml | 7 +-
slider-funtest/pom.xml.orig | 357 +++++++++++++++++++
.../org/apache/chaos/remote/BuildPidPath.groovy | 54 +++
.../org/apache/chaos/remote/Clustat.groovy | 64 ++++
.../apache/chaos/remote/JschToCommonsLog.groovy | 78 ++++
.../chaos/remote/RemoteDaemonOperations.groovy | 60 ++++
.../org/apache/chaos/remote/RemoteServer.groovy | 188 ++++++++++
.../org/apache/chaos/remote/SshCommands.groovy | 190 ++++++++++
.../funtest/lifecycle/AMFailuresIT.groovy | 191 ++++++++++
15 files changed, 1434 insertions(+), 4 deletions(-)
----------------------------------------------------------------------