You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/10/12 02:58:16 UTC
[28/50] git commit: SLIDER-105. Agent provider service/client should
reject component instance count when it exceeds the specified maximum or
below the minimum SLIDER-274. Flex'ing with unknown role name results in
BadConfigException
SLIDER-105. Agent provider service/client should reject component instance count when it exceeds the specified maximum or below the minimum
SLIDER-274. Flex'ing with unknown role name results in BadConfigException
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b32b8f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b32b8f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b32b8f7
Branch: refs/heads/feature/SLIDER-481_allow_dedicated_handling_of_exports
Commit: 3b32b8f7d333bfc506b6539c90d16f000005a55c
Parents: e375726
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Oct 8 14:43:29 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Oct 8 14:43:38 2014 -0700
----------------------------------------------------------------------
app-packages/hbase-win/metainfo.xml | 2 -
app-packages/hbase/metainfo.xml | 4 -
app-packages/hbase/pom.xml | 1 +
app-packages/storm/metainfo.xml | 3 +
.../org/apache/slider/client/SliderClient.java | 20 +++--
.../providers/AbstractClientProvider.java | 4 +-
.../providers/agent/AgentClientProvider.java | 79 +++++++++++++++-----
.../slider/providers/agent/AgentKeys.java | 2 +-
.../providers/agent/AgentProviderService.java | 43 ++++++++---
.../agent/application/metadata/Component.java | 28 +++++++
.../agent/application/metadata/Metainfo.java | 19 +++++
.../slideram/SliderAMClientProvider.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 4 +-
.../providers/agent/TestBuildBasicAgent.groovy | 65 ++++++++++++++--
.../agent/TestAgentClientProvider.java | 2 +-
.../publisher/TestAgentProviderService.java | 1 +
slider-core/src/test/python/metainfo.xml | 21 ++++++
.../providers/agent/tests/bad/resources-3.json | 13 ++++
.../providers/agent/tests/good/resources.json | 4 +-
.../accumulo/AccumuloClientProvider.java | 4 +-
.../accumulo/AccumuloProviderService.java | 2 +-
.../providers/hbase/HBaseClientProvider.java | 4 +-
.../providers/hbase/HBaseProviderService.java | 4 +-
23 files changed, 265 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase-win/metainfo.xml b/app-packages/hbase-win/metainfo.xml
index 17b4404..f0c5d4f 100644
--- a/app-packages/hbase-win/metainfo.xml
+++ b/app-packages/hbase-win/metainfo.xml
@@ -59,7 +59,6 @@
<name>HBASE_MASTER</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
<appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor</appExports>
<componentExports>
<componentExport>
@@ -81,7 +80,6 @@
<component>
<name>HBASE_REGIONSERVER</name>
<category>SLAVE</category>
- <minInstanceCount>1</minInstanceCount>
<commandScript>
<script>scripts/hbase_regionserver.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index 921ee48..d5e07a7 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -81,7 +81,6 @@
<name>HBASE_MASTER</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
- <maxInstanceCount>2</maxInstanceCount>
<appExports>QuickLinks-org.apache.slider.jmx,QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.metrics,QuickLinks-org.apache.slider.metrics.ui</appExports>
<componentExports>
<componentExport>
@@ -113,7 +112,6 @@
<component>
<name>HBASE_REST</name>
<category>MASTER</category>
- <minInstanceCount>0</minInstanceCount>
<appExports>QuickLinks-org.apache.slider.hbase.rest</appExports>
<commandScript>
<script>scripts/hbase_rest.py</script>
@@ -124,7 +122,6 @@
<component>
<name>HBASE_THRIFT</name>
<category>MASTER</category>
- <minInstanceCount>0</minInstanceCount>
<appExports>QuickLinks-org.apache.slider.hbase.thrift</appExports>
<commandScript>
<script>scripts/hbase_thrift.py</script>
@@ -146,7 +143,6 @@
<component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
- <minInstanceCount>0</minInstanceCount>
<commandScript>
<script>scripts/hbase_client.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/hbase/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/pom.xml b/app-packages/hbase/pom.xml
index be1b858..b85923b 100644
--- a/app-packages/hbase/pom.xml
+++ b/app-packages/hbase/pom.xml
@@ -32,6 +32,7 @@
<properties>
<work.dir>package-tmp</work.dir>
<app.package.name>${project.artifactId}-${project.version}</app.package.name>
+ <bld.version>0.51.0-incubating-SNAPSHOT</bld.version>
</properties>
<profiles>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index 1579174..e07ae92 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -78,6 +78,7 @@
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
<appExports>QuickLinks-nimbus.host_port,QuickLinks-org.apache.slider.metrics.ui,QuickLinks-org.apache.slider.metrics</appExports>
+ <maxInstanceCount>1</maxInstanceCount>
<commandScript>
<script>scripts/nimbus.py</script>
<scriptType>PYTHON</scriptType>
@@ -107,6 +108,7 @@
<category>MASTER</category>
<publishConfig>true</publishConfig>
<appExports>QuickLinks-org.apache.slider.monitor,QuickLinks-org.apache.slider.jmx</appExports>
+ <maxInstanceCount>1</maxInstanceCount>
<autoStartOnFailure>true</autoStartOnFailure>
<commandScript>
<script>scripts/ui_server.py</script>
@@ -119,6 +121,7 @@
<name>DRPC_SERVER</name>
<category>MASTER</category>
<autoStartOnFailure>true</autoStartOnFailure>
+ <maxInstanceCount>1</maxInstanceCount>
<commandScript>
<script>scripts/drpc_server.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 8d2bcfd..f15bc10 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -841,8 +841,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// providers to validate what there is
AggregateConf instanceDescription = builder.getInstanceDescription();
- validateInstanceDefinition(sliderAM, instanceDescription);
- validateInstanceDefinition(provider, instanceDescription);
+ validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
try {
persistInstanceDefinition(overwrite, appconfdir, builder);
} catch (LockAcquireFailedException e) {
@@ -1973,13 +1973,17 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
resources);
}
SliderAMClientProvider sliderAM = new SliderAMClientProvider(getConfig());
+ AbstractClientProvider provider = createClientProvider(
+ instanceDefinition.getInternalOperations().getGlobalOptions().getMandatoryOption(
+ InternalKeys.INTERNAL_PROVIDER_NAME));
// slider provider to validate what there is
- validateInstanceDefinition(sliderAM, instanceDefinition);
-
+ validateInstanceDefinition(sliderAM, instanceDefinition, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDefinition, sliderFileSystem);
+
int exitCode = EXIT_FALSE;
// save the specification
try {
- InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory,instanceDefinition);
+ InstanceIO.updateInstanceDefinition(sliderFileSystem, clusterDirectory, instanceDefinition);
} catch (LockAcquireFailedException e) {
// lock failure
log.debug("Failed to lock dir {}", clusterDirectory, e);
@@ -2008,12 +2012,12 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
* @throws SliderException if invalid.
*/
protected void validateInstanceDefinition(AbstractClientProvider provider,
- AggregateConf instanceDefinition) throws SliderException {
+ AggregateConf instanceDefinition, SliderFileSystem fs) throws SliderException {
try {
- provider.validateInstanceDefinition(instanceDefinition);
+ provider.validateInstanceDefinition(instanceDefinition, fs);
} catch (SliderException e) {
//problem, reject it
- log.info("Error {} validating application instance definition ", e);
+ log.info("Error {} validating application instance definition ", e.getMessage());
log.debug("Error validating application instance definition ", e);
log.info(instanceDefinition.toString());
throw e;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index 7c2c7f4..f89f842 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -66,7 +66,7 @@ public abstract class AbstractClientProvider extends Configured {
* @param instanceDefinition instance definition
* @throws SliderException if the configuration is not valid
*/
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
List<ProviderRole> roles = getRoles();
@@ -206,7 +206,7 @@ public abstract class AbstractClientProvider extends Configured {
Path generatedConfDirPath,
boolean secure)
throws SliderException, IOException {
- validateInstanceDefinition(instanceDefinition);
+ validateInstanceDefinition(instanceDefinition, sliderFileSystem);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 4c2f213..d7ae245 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -36,6 +36,7 @@ import org.apache.slider.providers.AbstractClientProvider;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -109,25 +110,55 @@ public class AgentClientProvider extends AbstractClientProvider
}
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
log.debug("Validating conf {}", instanceDefinition);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- ConfTreeOperations appConf =
- instanceDefinition.getAppConfOperations();
providerUtils.validateNodeCount(instanceDefinition, ROLE_NODE,
0, -1);
+ try {
+ // Validate the app definition
+ instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ } catch (BadConfigException bce) {
+ throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
+ }
+ String appDef = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
+ log.info("Validating app definition {}", appDef);
+ String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
+ if (!"zip".equalsIgnoreCase(extension)) {
+ throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
+ }
+
Set<String> names = resources.getComponentNames();
names.remove(SliderKeys.COMPONENT_AM);
Map<Integer, String> priorityMap = new HashMap<Integer, String>();
+
+ Metainfo metaInfo = null;
+ if (fs != null) {
+ try {
+ metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef);
+ } catch (IOException ioe) {
+ // Ignore missing metainfo file for now
+ log.info("Missing metainfo.xml {}", ioe.getMessage());
+ }
+ }
+
for (String name : names) {
MapOperations component = resources.getMandatoryComponent(name);
- // Validate count against the metainfo.xml
+ if (metaInfo != null) {
+ Component componentDef = metaInfo.getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
+ }
int priority =
component.getMandatoryOptionInt(ResourceKeys.COMPONENT_PRIORITY);
@@ -150,22 +181,34 @@ public class AgentClientProvider extends AbstractClientProvider
priorityMap.put(priority, name);
}
- try {
- // Validate the app definition
- instanceDefinition.getAppConfOperations().
- getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- } catch (BadConfigException bce) {
- throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
- }
- String appDef = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- log.info("Validating app definition {}", appDef);
- String extension = appDef.substring(appDef.lastIndexOf(".") + 1, appDef.length());
- if (!"zip".equalsIgnoreCase(extension)) {
- throw new BadConfigException("App definition must be packaged as a .zip file. File provided is " + appDef);
+ // fileSystem may be null for tests
+ if (metaInfo != null) {
+ for (String name : names) {
+ Component componentDef = metaInfo.getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
+
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
+ }
}
}
+
@Override
public void prepareAMAndConfigForLaunch(SliderFileSystem fileSystem,
Configuration serviceConf,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index b30c18c..2612865 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -24,7 +24,7 @@ package org.apache.slider.providers.agent;
public interface AgentKeys {
String PROVIDER_AGENT = "agent";
- String ROLE_NODE = "node";
+ String ROLE_NODE = "echo";
/**
* {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 88c8709..a200e19 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -34,6 +34,7 @@ import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
@@ -189,7 +190,35 @@ public class AgentProviderService extends AbstractProviderService implements
public void validateInstanceDefinition(AggregateConf instanceDefinition)
throws
SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
+
+ ConfTreeOperations resources =
+ instanceDefinition.getResourceOperations();
+
+ Set<String> names = resources.getComponentNames();
+ names.remove(SliderKeys.COMPONENT_AM);
+ for (String name : names) {
+ Component componentDef = getMetainfo().getApplicationComponent(name);
+ if (componentDef == null) {
+ throw new BadConfigException(
+ "Component %s is not a member of application.", name);
+ }
+
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
+ }
}
// Reads the metainfo.xml in the application package and loads it
@@ -1073,17 +1102,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return
*/
protected Component getApplicationComponent(String roleName) {
- Application application = getMetainfo().getApplication();
- if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
- } else {
- for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
- return component;
- }
- }
- }
- return null;
+ return getMetainfo().getApplicationComponent(roleName);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 1117ff1..9f3dd0f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -16,6 +16,10 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.core.exceptions.SliderException;
+
import java.util.ArrayList;
import java.util.List;
@@ -82,6 +86,30 @@ public class Component {
return minInstanceCount;
}
+ public int getMinInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(minInstanceCount)) {
+ return 0;
+ }
+
+ try {
+ return Integer.parseInt(minInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for minInstanceCount for %s", name);
+ }
+ }
+
+ public int getMaxInstanceCountInt() throws BadConfigException {
+ if (SliderUtils.isUnset(maxInstanceCount)) {
+ return Integer.MAX_VALUE;
+ }
+
+ try {
+ return Integer.parseInt(maxInstanceCount);
+ } catch (NumberFormatException nfe) {
+ throw new BadConfigException(nfe, "Invalid value for maxInstanceCount for %s", name);
+ }
+ }
+
public void setMinInstanceCount(String minInstanceCount) {
this.minInstanceCount = minInstanceCount;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
index 2455e8e..b34cba1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -16,10 +16,16 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
/**
* Application metainfo uber class
*/
public class Metainfo {
+ protected static final Logger log =
+ LoggerFactory.getLogger(Metainfo.class);
+
String schemaVersion;
Application application;
@@ -41,4 +47,17 @@ public class Metainfo {
public void setApplication(Application application) {
this.application = application;
}
+
+ public Component getApplicationComponent(String roleName) {
+ if (application == null) {
+ log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+ } else {
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ return component;
+ }
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
index 748e5f1..aa1312c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMClientProvider.java
@@ -139,10 +139,10 @@ public class SliderAMClientProvider extends AbstractClientProvider
* @param instanceDefinition instance definition
* @throws SliderException if the configuration is not valid
*/
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
// make sure there is no negative entry in the instance count
Map<String, Map<String, String>> instanceMap =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 50fc265..31a9a28 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -811,9 +811,9 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//Give the provider restricted access to the state, registry
providerService.bind(stateForProviders, registry, actionQueues,
- liveContainers);
+ liveContainers);
sliderAMProvider.bind(stateForProviders, registry, actionQueues,
- liveContainers);
+ liveContainers);
// chaos monkey
maybeStartMonkey();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
index 2f6a15e..d0129c5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/providers/agent/TestBuildBasicAgent.groovy
@@ -79,7 +79,7 @@ class TestBuildBasicAgent extends AgentTestBase {
true,
false)
buildAgentCluster("test_build_basic_agent_node_only",
- [(ROLE_NODE): 5],
+ [(ROLE_NODE): 1],
[
ARG_OPTION, CONTROLLER_URL, "http://localhost",
ARG_PACKAGE, ".",
@@ -96,7 +96,7 @@ class TestBuildBasicAgent extends AgentTestBase {
def rs = "hbase-rs"
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 1,
(master): 1,
(rs): 5
],
@@ -138,7 +138,7 @@ class TestBuildBasicAgent extends AgentTestBase {
def name2 = clustername + "-2"
buildAgentCluster(name2,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
"role3": 1,
"newnode": 5
],
@@ -154,6 +154,26 @@ class TestBuildBasicAgent extends AgentTestBase {
} catch (BadConfigException expected) {
}
+ try {
+ launcher = buildAgentCluster(clustername,
+ [
+ (ROLE_NODE): 4,
+ ],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_OPTION, PACKAGE_PATH, ".",
+ ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
+ ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
+ ARG_COMP_OPT, ROLE_NODE, SCRIPT_PATH, "agent/scripts/agent.py",
+ ARG_RES_COMP_OPT, ROLE_NODE, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(ROLE_NODE, "too many instances")
+ } catch (BadConfigException expected) {
+ assert expected.message.contains("Expected minimum is 1 and maximum is 2")
+ assert expected.message.contains("Component echo, yarn.component.instances value 4 out of range.")
+ }
//duplicate priorities
try {
def name3 = clustername + "-3"
@@ -212,13 +232,13 @@ class TestBuildBasicAgent extends AgentTestBase {
def name5 = clustername + "-5"
buildAgentCluster(name5,
[
- "role": 1,
+ "hbase-rs": 1,
],
[
ARG_OPTION, APP_DEF, "file://" + getAppDef().absolutePath,
ARG_OPTION, AGENT_CONF, "file://" + getAgentConf().absolutePath,
ARG_PACKAGE, ".",
- ARG_RES_COMP_OPT, "role", ResourceKeys.COMPONENT_PRIORITY, "3",
+ ARG_RES_COMP_OPT, "hbase-rs", ResourceKeys.COMPONENT_PRIORITY, "3",
],
true, false,
false)
@@ -240,7 +260,7 @@ class TestBuildBasicAgent extends AgentTestBase {
def rs = "hbase-rs"
ServiceLauncher<SliderClient> launcher = buildAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
(master): 1,
(rs): 5
],
@@ -279,7 +299,7 @@ class TestBuildBasicAgent extends AgentTestBase {
// change master priority and rs instances through update action
ServiceLauncher<SliderClient> launcher2 = updateAgentCluster(clustername,
[
- (ROLE_NODE): 5,
+ (ROLE_NODE): 2,
(master): 1,
(rs): 6
],
@@ -357,6 +377,37 @@ class TestBuildBasicAgent extends AgentTestBase {
}
@Test
+ public void testBadAgentArgs_Unknown_Component() throws Throwable {
+ String clustername = createMiniCluster(
+ "",
+ configuration,
+ 1,
+ 1,
+ 1,
+ true,
+ false)
+
+ try {
+ def badArgs1 = "test_bad_agent_unk_comp"
+ buildAgentCluster(clustername,
+ [:],
+ [
+ ARG_OPTION, CONTROLLER_URL, "http://localhost",
+ ARG_PACKAGE, ".",
+ ARG_OPTION, APP_DEF, "file://" + appDef.absolutePath,
+ ARG_RESOURCES, TEST_FILES + "bad/resources-3.json",
+ ARG_TEMPLATE, TEST_FILES + "good/appconf.json"
+ ],
+ true, false,
+ false)
+ failWithBuildSucceeding(badArgs1, "bad component type node")
+ } catch (BadConfigException expected) {
+ log.info("Expected failure.", expected)
+ assert expected.message.contains("Component node is not a member of application")
+ }
+ }
+
+ @Test
public void testSubmitToSpecificQueue() throws Throwable {
String clustername = createMiniCluster(
"",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
index 4cb35aa..0bea8fa 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider.java
@@ -67,7 +67,7 @@ public class TestAgentClientProvider {
AggregateConf instanceDefinition = new AggregateConf();
try {
- provider.validateInstanceDefinition(instanceDefinition);
+ provider.validateInstanceDefinition(instanceDefinition, null);
Assert.assertFalse("Should fail with BadConfigException", true);
} catch (BadConfigException e) {
log.info(e.toString());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index eb368e3..a49a560 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -17,6 +17,7 @@
package org.apache.slider.server.appmaster.web.rest.publisher;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.providers.agent.AgentProviderService;
import org.apache.slider.server.appmaster.actions.QueueAccess;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/python/metainfo.xml
----------------------------------------------------------------------
diff --git a/slider-core/src/test/python/metainfo.xml b/slider-core/src/test/python/metainfo.xml
index 09b314e..cf4afe1 100644
--- a/slider-core/src/test/python/metainfo.xml
+++ b/slider-core/src/test/python/metainfo.xml
@@ -27,6 +27,27 @@
<minHadoopVersion>2.1.0</minHadoopVersion>
<components>
<component>
+ <name>hbase-rs</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <commandScript>
+ <script>echo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
+ <name>hbase-master</name>
+ <category>MASTER</category>
+ <minInstanceCount>0</minInstanceCount>
+ <maxInstanceCount>2</maxInstanceCount>
+ <commandScript>
+ <script>echo.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ <component>
<name>echo</name>
<category>MASTER</category>
<minInstanceCount>1</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json
new file mode 100644
index 0000000..625b10e
--- /dev/null
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/bad/resources-3.json
@@ -0,0 +1,13 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+
+ "global": {
+ },
+ "components": {
+ "node": {
+ "yarn.memory": "256",
+ "yarn.component.instances": "2",
+ "yarn.role.priority":"1"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
index f6111a0..83cdb99 100644
--- a/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
+++ b/slider-core/src/test/resources/org/apache/slider/providers/agent/tests/good/resources.json
@@ -4,9 +4,9 @@
"global": {
},
"components": {
- "node": {
+ "echo": {
"yarn.memory": "256",
- "yarn.component.instances": "5",
+ "yarn.component.instances": "2",
"yarn.role.priority":"1"
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
index 7f99573..8fe5170 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloClientProvider.java
@@ -300,9 +300,9 @@ public class AccumuloClientProvider extends AbstractClientProvider implements
}
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
index b8f4c00..c1bc9fe 100644
--- a/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
+++ b/slider-providers/accumulo/slider-accumulo-provider/src/main/java/org/apache/slider/providers/accumulo/AccumuloProviderService.java
@@ -100,7 +100,7 @@ public class AccumuloProviderService extends AbstractProviderService implements
@Override
public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 9ad872f..81297fd 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -224,9 +224,9 @@ public class HBaseClientProvider extends AbstractClientProvider implements
* @param instanceDefinition instance definition
*/
@Override
- public void validateInstanceDefinition(AggregateConf instanceDefinition) throws
+ public void validateInstanceDefinition(AggregateConf instanceDefinition, SliderFileSystem fs) throws
SliderException {
- super.validateInstanceDefinition(instanceDefinition);
+ super.validateInstanceDefinition(instanceDefinition, fs);
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
Set<String> unknownRoles = resources.getComponentNames();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b32b8f7/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index f75a6c7..4228def 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -106,9 +106,9 @@ public class HBaseProviderService extends AbstractProviderService
* @param instanceDefinition the instance definition to validate
*/
@Override // Client and Server
- public void validateInstanceDefinition(AggregateConf instanceDefinition)
+ public void validateInstanceDefinition(AggregateConf instanceDefinition)
throws SliderException {
- clientProvider.validateInstanceDefinition(instanceDefinition);
+ clientProvider.validateInstanceDefinition(instanceDefinition, null);
}
@Override