You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2015/01/27 02:18:54 UTC
incubator-slider git commit: SLIDER-755. AgentProvider doesn't raise
and exception when the agent tar isn't there SLIDER-723. Memcached component
launch fail does not propagate
Repository: incubator-slider
Updated Branches:
refs/heads/develop 98d676b90 -> 657f734c1
SLIDER-755. AgentProvider doesn't raise and exception when the agent tar isn't there
SLIDER-723. Memcached component launch fail does not propagate
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/657f734c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/657f734c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/657f734c
Branch: refs/heads/develop
Commit: 657f734c1104be650318241c5c0bdbcd0d61533e
Parents: 98d676b
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jan 26 17:18:02 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jan 26 17:18:10 2015 -0800
----------------------------------------------------------------------
.../slider/providers/agent/AgentKeys.java | 16 +--
.../providers/agent/AgentProviderService.java | 21 ++-
.../slider/providers/agent/TestAgentEcho.groovy | 4 +-
.../appmaster/TestDelayInContainerLaunch.groovy | 1 +
.../agent/TestAgentProviderService.java | 132 ++++++++++++++++++-
5 files changed, 155 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index e682b13..7977181 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -25,20 +25,7 @@ public interface AgentKeys {
String PROVIDER_AGENT = "agent";
String ROLE_NODE = "echo";
- /**
- * {@value}
- */
- String REGION_SERVER = "regionserver";
- /**
- * What is the command for hbase to print a version: {@value}
- */
- String COMMAND_VERSION = "version";
- String ACTION_START = "start";
- String ACTION_STOP = "stop";
- /**
- * Config directory : {@value}
- */
- String ARG_CONFIG = "--config";
+
/**
* Template stored in the slider classpath -to use if there is
* no site-specific template
@@ -100,6 +87,7 @@ public interface AgentKeys {
String KEY_AGENT_TWO_WAY_SSL_ENABLED = "ssl.server.client.auth";
String CERT_FILE_LOCALIZATION_PATH = "certs/ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
+ String TEST_RELAX_VERIFICATION = "test.relax.validation";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index e4680e2..439b5b6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -353,13 +353,21 @@ public class AgentProviderService extends AbstractProviderService implements
agentImagePath = new Path(agentImage);
}
- // TODO: throw exception when agent tarball is not available
-
if (fileSystem.getFileSystem().exists(agentImagePath)) {
LocalResource agentImageRes = fileSystem.createAmResource(agentImagePath, LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.AGENT_INSTALL_DIR, agentImageRes);
} else {
- log.error("Required agent image slider-agent.tar.gz is unavailable.");
+ String msg =
+ String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
+ MapOperations compOps = instanceDefinition.
+ getAppConfOperations().getComponent(role);
+ boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
+ getOptionBool(AgentKeys.TEST_RELAX_VERIFICATION, false)) : false;
+ log.error(msg);
+
+ if (!relaxVerificationForTest) {
+ throw new SliderException(SliderExitCodes.EXIT_DEPLOYMENT_FAILED, msg);
+ }
}
log.info("Using {} for agent.", scriptPath);
@@ -750,6 +758,13 @@ public class AgentProviderService extends AbstractProviderService implements
&& command == Command.NOP && isMarkedAutoRestart(roleName)) {
response.setRestartEnabled(true);
}
+
+ //If INSTALL_FAILED and no INSTALL is scheduled let the agent fail
+ if(componentStatus.getState() == State.INSTALL_FAILED && command == command.NOP) {
+ log.warn("Sending terminate signal to container that failed installation: {}", label);
+ response.setTerminateAgent(true);
+ }
+
} catch (SliderException e) {
log.warn("Component instance failed operation.", e);
componentStatus.applyCommandResult(CommandResult.FAILED, command);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
index 2eb39e3..973114d 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestAgentEcho.groovy
@@ -99,7 +99,9 @@ class TestAgentEcho extends AgentTestBase {
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
ARG_DEFINE,
- SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false"
+ SliderXmlConfKeys.KEY_SLIDER_AM_DEPENDENCY_CHECKS_DISABLED + "=false",
+ ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true",
+
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
index 26a2c95..51e01d5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/TestDelayInContainerLaunch.groovy
@@ -107,6 +107,7 @@ class TestDelayInContainerLaunch extends AgentTestBase {
ARG_RES_COMP_OPT, role, ResourceKeys.COMPONENT_PRIORITY, "1",
ARG_COMP_OPT, role, SCRIPT_PATH, echo_py,
ARG_COMP_OPT, role, SERVICE_NAME, "Agent",
+ ARG_COMP_OPT, role, TEST_RELAX_VERIFICATION, "true",
],
true, true,
true)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/657f734c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index dbc1b0a..3b78fb8 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -496,6 +496,136 @@ public class TestAgentProviderService {
}
@Test
+ public void testThreeInstallFailures() throws IOException, SliderException {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ ConfTree tree = new ConfTree();
+ tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
+
+ Configuration conf = new Configuration();
+ AgentProviderService aps = createAgentProviderService(conf);
+ YarnRegistryViewForProviders registryViewForProviders = aps.getYarnRegistry();
+ assertNotNull(registryViewForProviders);
+
+ ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
+ AggregateConf instanceDefinition = new AggregateConf();
+
+ instanceDefinition.setInternal(tree);
+ instanceDefinition.setAppConf(tree);
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
+ instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+
+ Container container = createNiceMock(Container.class);
+ String role_hm = "HBASE_MASTER";
+ SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
+ ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
+ Path generatedConfPath = new Path(".", "test");
+ MapOperations resourceComponent = new MapOperations();
+ MapOperations appComponent = new MapOperations();
+ Path containerTmpDirPath = new Path(".", "test");
+ FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
+ expect(sliderFileSystem.getFileSystem())
+ .andReturn(mockFs).anyTimes();
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
+ expect(sliderFileSystem.createAmResource(anyObject(Path.class),
+ anyObject(LocalResourceType.class)))
+ .andReturn(createNiceMock(LocalResource.class)).anyTimes();
+ expect(container.getId()).andReturn(new MockContainerId(1)).anyTimes();
+ expect(container.getNodeId()).andReturn(new MockNodeId("localhost")).anyTimes();
+ StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
+
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doReturn(access).when(mockAps).getAmState();
+ doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
+ doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).
+ initializeDefaultConfigs(any(SliderFileSystem.class), anyString(), any(Metainfo.class));
+
+
+ try {
+ doReturn(true).when(mockAps).isMaster(anyString());
+ doNothing().when(mockAps).addInstallCommand(
+ anyString(),
+ anyString(),
+ any(HeartBeatResponse.class),
+ anyString(),
+ Mockito.anyLong());
+ doReturn(conf).when(mockAps).getConfig();
+ } catch (SliderException e) {
+ }
+
+ expect(access.isApplicationLive()).andReturn(true).anyTimes();
+ ClusterDescription desc = new ClusterDescription();
+ desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
+ desc.setInfo(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getClusterStatus()).andReturn(desc).anyTimes();
+
+ AggregateConf aggConf = new AggregateConf();
+ ConfTreeOperations treeOps = aggConf.getAppConfOperations();
+ treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
+ treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
+ expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ replay(access, ctx, container, sliderFileSystem, mockFs);
+
+ try {
+ mockAps.buildContainerLaunchContext(launcher,
+ instanceDefinition,
+ container,
+ role_hm,
+ sliderFileSystem,
+ generatedConfPath,
+ resourceComponent,
+ appComponent,
+ containerTmpDirPath);
+
+ Register reg = new Register();
+ reg.setResponseId(0);
+ reg.setLabel("mockcontainer_1___HBASE_MASTER");
+ RegistrationResponse resp = mockAps.handleRegistration(reg);
+ Assert.assertEquals(0, resp.getResponseId());
+ Assert.assertEquals(RegistrationStatus.OK, resp.getResponseStatus());
+
+ HeartBeat hb = new HeartBeat();
+ hb.setResponseId(1);
+ hb.setHostname("mockcontainer_1___HBASE_MASTER");
+ HeartBeatResponse hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(2, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ hb.setResponseId(2);
+ CommandReport cr = new CommandReport();
+ cr.setRole("HBASE_MASTER");
+ cr.setRoleCommand("INSTALL");
+ cr.setStatus("FAILED");
+ hb.setReports(Arrays.asList(cr));
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(3, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ hb.setResponseId(3);
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(4, hbr.getResponseId());
+ Assert.assertFalse(hbr.isTerminateAgent());
+
+ //Third failure triggers a stop to the agent
+ hb.setResponseId(4);
+ hbr = mockAps.handleHeartBeat(hb);
+ Assert.assertEquals(5, hbr.getResponseId());
+ Assert.assertTrue(hbr.isTerminateAgent());
+ Mockito.verify(mockAps, Mockito.times(3)).addInstallCommand(anyString(),
+ anyString(),
+ any(HeartBeatResponse.class),
+ anyString(),
+ Mockito.anyLong());
+ } catch (SliderException he) {
+ log.warn(he.getMessage());
+ } catch (IOException he) {
+ log.warn(he.getMessage());
+ }
+ }
+
+ @Test
public void testAgentStateStarted() throws IOException, SliderException {
AggregateConf instanceDefinition = prepareConfForAgentStateTests();
AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
@@ -1056,7 +1186,7 @@ public class TestAgentProviderService {
FilterFileSystem mockFs = createNiceMock(FilterFileSystem.class);
expect(sliderFileSystem.getFileSystem())
.andReturn(mockFs).anyTimes();
- expect(mockFs.exists(anyObject(Path.class))).andReturn(true);
+ expect(mockFs.exists(anyObject(Path.class))).andReturn(true).anyTimes();
expect(sliderFileSystem.createAmResource(anyObject(Path.class),
anyObject(LocalResourceType.class)))
.andReturn(createNiceMock(LocalResource.class)).anyTimes();