You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/08/12 21:46:00 UTC
[2/6] incubator-slider git commit: SLIDER-875 better handling and
support of nested external components through addition of role.prefix
SLIDER-875 better handling and support of nested external components through addition of role.prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a4dc574c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a4dc574c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a4dc574c
Branch: refs/heads/develop
Commit: a4dc574c9f60e2f07906b37f8200e9488269c20d
Parents: 000b38d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jul 13 09:53:47 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jul 13 09:53:47 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/api/RoleKeys.java | 5 +
.../apache/slider/common/tools/SliderUtils.java | 9 ++
.../slider/core/build/InstanceBuilder.java | 108 +++++++++++----
.../slider/core/conf/ConfTreeOperations.java | 39 ++++++
.../providers/agent/AgentClientProvider.java | 6 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 42 +++---
.../slider/providers/agent/AgentUtils.java | 30 ++--
.../providers/agent/ComponentCommandOrder.java | 110 ++++++++++-----
.../slider/server/appmaster/state/AppState.java | 5 +
.../appConfig_external_component_nested.json | 12 ++
.../test_min_pkg/sleep_cmd/metainfo.json | 6 +
.../metainfo_external_component_nested.json | 14 ++
.../resources_external_component_nested.json | 12 ++
.../TestBuildExternalComponent.groovy | 138 +++++++++++++++++++
.../agent/TestAgentProviderService.java | 35 ++---
.../agent/TestComponentCommandOrder.java | 107 +++++++++++++-
.../apache/slider/funtest/ResourcePaths.groovy | 4 +
.../funtest/misc/ExternalComponentIT.groovy | 38 +++++
19 files changed, 589 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 812a6b3..ce413ff 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -35,6 +35,11 @@ public interface RoleKeys {
String ROLE_GROUP = "role.group";
/**
+ * The prefix of a role: {@value}
+ */
+ String ROLE_PREFIX = "role.prefix";
+
+ /**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index c48e109..b798d4d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -123,6 +123,8 @@ import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+
/**
* These are slider-specific Util methods
*/
@@ -2612,4 +2614,11 @@ public final class SliderUtils {
}
return buffer.toString();
}
+
+ public static String trimPrefix(String prefix) {
+ if (prefix != null && prefix.endsWith(COMPONENT_SEPARATOR)) {
+ return prefix.substring(0, prefix.length()-1);
+ }
+ return prefix;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 4afdf7c..6a7975e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -69,6 +69,7 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE;
@@ -85,7 +86,7 @@ public class InstanceBuilder {
private final CoreFileSystem coreFS;
private final InstancePaths instancePaths;
private AggregateConf instanceDescription;
- private Map<Path, Path> externalAppDefs = new HashMap<>();
+ private Map<String, Path> externalAppDefs = new HashMap<>();
private TreeSet<Integer> priorities = new TreeSet<>();
private static final Logger log =
@@ -272,8 +273,7 @@ public class InstanceBuilder {
continue;
}
Map<String, String> options = entry.getValue();
- if (options.containsKey(COMPONENT_TYPE) &&
- EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
+ if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
externalComponents.add(entry.getKey());
}
}
@@ -284,17 +284,37 @@ public class InstanceBuilder {
private void mergeExternalComponent(ConfTreeOperations ops,
ConfTreeOperations externalOps, String externalComponent,
- Integer priority) {
+ Integer priority) throws BadConfigException {
for (String subComponent : externalOps.getComponentNames()) {
if (COMPONENT_AM.equals(subComponent)) {
continue;
}
+ String prefix = externalComponent + COMPONENT_SEPARATOR;
log.debug("Merging options for {} into {}", subComponent,
- externalComponent + COMPONENT_SEPARATOR + subComponent);
- MapOperations subComponentOps = ops.getOrAddComponent(externalComponent +
- COMPONENT_SEPARATOR + subComponent);
+ prefix + subComponent);
+ MapOperations subComponentOps = ops.getOrAddComponent(
+ prefix + subComponent);
+ if (priority == null) {
+ SliderUtils.mergeMaps(subComponentOps,
+ ops.getComponent(externalComponent).options);
+ subComponentOps.remove(COMPONENT_TYPE);
+ }
+
SliderUtils.mergeMapsIgnorePrefixes(subComponentOps,
externalOps.getComponent(subComponent), PREFIXES_TO_SKIP);
+
+ // add prefix to existing prefix
+ String existingPrefix = subComponentOps.get(ROLE_PREFIX);
+ if (existingPrefix != null) {
+ if (!subComponent.startsWith(existingPrefix)) {
+ throw new BadConfigException("Bad prefix " + existingPrefix +
+ " for subcomponent " + subComponent + " of " + externalComponent);
+ }
+ prefix = prefix + existingPrefix;
+ }
+ subComponentOps.set(ROLE_PREFIX, prefix);
+
+ // adjust priority
if (priority != null) {
subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY,
Integer.toString(priority));
@@ -322,10 +342,6 @@ public class InstanceBuilder {
if (COMPONENT_AM.equals(entry.getKey())) {
continue;
}
- if (entry.getKey().contains(COMPONENT_SEPARATOR)) {
- throw new BadConfigException("Components must not contain " +
- COMPONENT_SEPARATOR + ": " + entry.getKey());
- }
if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) {
priorities.add(Integer.parseInt(entry.getValue().get(
ResourceKeys.COMPONENT_PRIORITY)));
@@ -358,31 +374,73 @@ public class InstanceBuilder {
throw new BadConfigException("Couldn't read configuration for " +
"external component " + component);
}
- String externalAppDef = componentConf.getAppConfOperations()
- .getGlobalOptions().get(AgentKeys.APP_DEF);
+
+ ConfTreeOperations componentAppConf = componentConf.getAppConfOperations();
+ String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF);
if (SliderUtils.isSet(externalAppDef)) {
Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
component + "_" + SliderKeys.DEFAULT_APP_PKG);
- componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef);
- externalAppDefs.put(newAppDef, new Path(externalAppDef));
+ componentAppConf.set(AgentKeys.APP_DEF, newAppDef);
+ componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef);
+ log.info("Copying external appdef {} to {} for {}", externalAppDef,
+ newAppDef, component);
+ externalAppDefs.put(externalAppDef, newAppDef);
+ externalAppDef = newAppDef.toString();
}
+
for (String rcomp : componentConf.getResourceOperations()
.getComponentNames()) {
if (COMPONENT_AM.equals(rcomp)) {
continue;
}
log.debug("Adding component {} to appConf for {}", rcomp, component);
- componentConf.getAppConfOperations().getOrAddComponent(rcomp);
+ componentAppConf.getOrAddComponent(rcomp);
}
- SliderUtils.mergeMaps(
- componentConf.getAppConfOperations().getGlobalOptions().options,
- appConf.getComponent(component).options);
- componentConf.getAppConfOperations().getGlobalOptions()
- .remove(COMPONENT_TYPE);
componentConf.resolve();
- mergeExternalComponent(appConf, componentConf.getAppConfOperations(),
- component, null);
+ for (String rcomp : componentConf.getResourceOperations()
+ .getComponentNames()) {
+ if (COMPONENT_AM.equals(rcomp)) {
+ continue;
+ }
+ String componentAppDef = componentAppConf.getComponentOpt(
+ rcomp, AgentKeys.APP_DEF, null);
+ if (SliderUtils.isUnset(componentAppDef) ||
+ componentAppDef.equals(externalAppDef)) {
+ continue;
+ }
+ if (externalAppDefs.containsKey(componentAppDef)) {
+ log.info("Using external appdef {} for {}",
+ externalAppDefs.get(componentAppDef), rcomp);
+ } else {
+ String existingPrefix = componentAppConf.getComponentOpt(rcomp,
+ ROLE_PREFIX, null);
+ if (SliderUtils.isUnset(existingPrefix)) {
+ existingPrefix = "";
+ } else {
+ existingPrefix = COMPONENT_SEPARATOR + SliderUtils.trimPrefix(
+ existingPrefix);
+ }
+ Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
+ component + existingPrefix + "_" + SliderKeys.DEFAULT_APP_PKG);
+ externalAppDefs.put(componentAppDef, newAppDef);
+ log.info("Copying external appdef {} to {} for {}", componentAppDef,
+ newAppDef, component + COMPONENT_SEPARATOR + rcomp);
+ }
+ componentAppConf.setComponentOpt(rcomp, AgentKeys.APP_DEF,
+ externalAppDefs.get(componentAppDef).toString());
+ componentAppConf.appendComponentOpt(rcomp,
+ AgentKeys.APP_DEF_ORIGINAL, componentAppDef);
+ }
+ Set<Path> newAppDefs = new HashSet<>();
+ newAppDefs.addAll(externalAppDefs.values());
+ if (newAppDefs.size() != externalAppDefs.size()) {
+ throw new IllegalStateException("Values repeat in external appdefs "
+ + externalAppDefs);
+ }
+ log.info("External appdefs after {}: {}", component, externalAppDefs);
+
+ mergeExternalComponent(appConf, componentAppConf, component, null);
mergeExternalComponent(resources, componentConf.getResourceOperations(),
component, getNextPriority());
}
@@ -411,8 +469,8 @@ public class InstanceBuilder {
action = new ConfDirSnapshotAction(appconfdir);
}
persister.save(instanceDescription, action);
- for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) {
- SliderUtils.copy(conf, appDef.getValue(), appDef.getKey());
+ for (Entry<String, Path> appDef : externalAppDefs.entrySet()) {
+ SliderUtils.copy(conf, new Path(appDef.getKey()), appDef.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 4a0ae41..038513e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -214,6 +214,23 @@ public class ConfTreeOperations {
public String get(String key) {
return globalOptions.get(key);
}
+ /**
+ * append to a global option
+ * @param key key
+ * @return value
+ *
+ */
+ public String append(String key, String value) {
+ if (SliderUtils.isUnset(value)) {
+ return null;
+ }
+ if (globalOptions.containsKey(key)) {
+ globalOptions.put(key, globalOptions.get(key) + "," + value);
+ } else {
+ globalOptions.put(key, value);
+ }
+ return globalOptions.get(key);
+ }
/**
* Propagate all global keys matching a prefix
@@ -471,4 +488,26 @@ public class ConfTreeOperations {
setComponentOpt(role, option, Long.toString(val));
}
+ /**
+ * append to a component option
+ * @param key key
+ * @return value
+ *
+ */
+ public String appendComponentOpt(String role, String key, String value) {
+ if (SliderUtils.isUnset(value)) {
+ return null;
+ }
+ MapOperations roleopts = getComponent(role);
+ if (roleopts == null) {
+ return null;
+ }
+
+ if (roleopts.containsKey(key)) {
+ roleopts.put(key, roleopts.get(key) + "," + value);
+ } else {
+ roleopts.put(key, value);
+ }
+ return roleopts.get(key);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 6eae75e..8c0a2e4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -184,7 +184,8 @@ public class AgentClientProvider extends AbstractClientProvider
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name));
+ getMetainfoComponentName(name,
+ instanceDefinition.getAppConfOperations()));
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
@@ -214,7 +215,8 @@ public class AgentClientProvider extends AbstractClientProvider
// fileSystem may be null for tests
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name));
+ getMetainfoComponentName(name,
+ instanceDefinition.getAppConfOperations()));
// already checked it wasn't null
// ensure that intance count is 0 for client components
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 8514401..565d73d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -68,6 +68,7 @@ public interface AgentKeys {
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
+ String APP_DEF_ORIGINAL = "application.def.original";
String ADDON_PREFIX = "application.addon.";
String ADDONS = "application.addons";
String AGENT_VERSION = "agent.version";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 66fac99..452122f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -133,7 +133,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
/**
@@ -288,7 +288,8 @@ public class AgentProviderService extends AbstractProviderService implements
SliderFileSystem fileSystem,
String roleGroup)
throws IOException, SliderException {
- String mapKey = getMetainfoMapKey(roleGroup);
+ String mapKey = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
String appDef = SliderUtils.getApplicationDefinitionPath(
instanceDefinition.getAppConfOperations(), roleGroup);
MapOperations component = null;
@@ -319,16 +320,17 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Modifying external metainfo component name to {}",
comp.getName());
}
- String commandPrefix = mapKey.substring(0,
- mapKey.indexOf(COMPONENT_SEPARATOR)+1);
for (CommandOrder co : commandOrders) {
log.info("Adding prefix {} to command order {}",
- commandPrefix, co);
- co.setCommand(commandPrefix + co.getCommand());
- co.setRequires(commandPrefix + co.getRequires());
+ mapKey, co);
+ co.setCommand(mapKey + co.getCommand());
+ co.setRequires(mapKey + co.getRequires());
}
}
- commandOrder.mergeCommandOrders(commandOrders);
+ log.debug("Merging command orders {} for {}", commandOrders,
+ roleGroup);
+ commandOrder.mergeCommandOrders(commandOrders,
+ instanceDefinition.getResourceOperations());
Map<String, DefaultConfig> defaultConfigs =
initializeDefaultConfigs(fileSystem, appDef, metaInfo);
metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
@@ -1393,7 +1395,10 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected Metainfo getMetaInfo(String roleGroup) {
- MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup));
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
+ DEFAULT_METAINFO_MAP_KEY);
+ MetainfoHolder mh = this.metaInfoMap.get(mapKey);
if (mh == null) {
return null;
}
@@ -1468,7 +1473,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) {
- return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs;
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
+ DEFAULT_METAINFO_MAP_KEY);
+ return metaInfoMap.get(mapKey).defaultConfigs;
}
private int getHeartbeatMonitorInterval() {
@@ -2900,14 +2908,14 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
- String mapKey = getMetainfoMapKey(componentGroup);
+ String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX,
+ null);
String dataDirSuffix = "";
- if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
- dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1);
+ if (prefix == null) {
+ prefix = "";
} else {
- mapKey = "";
+ dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix);
}
- mapKey = mapKey.toLowerCase();
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
@@ -2915,8 +2923,8 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
tokens.put("${COMPONENT_NAME}", componentName);
tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
- tokens.put("${COMPONENT_PREFIX}", mapKey);
- tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase());
+ tokens.put("${COMPONENT_PREFIX}", prefix);
+ tokens.put("${COMPONENT_PREFIX.lc}", prefix.toLowerCase());
if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) {
tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length()));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index ed5108c..23e05a3 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
@@ -35,8 +36,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
-import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
/**
*
@@ -135,24 +135,16 @@ public class AgentUtils {
return new DefaultConfigParser().parse(configStream);
}
- static String getMetainfoMapKey(String roleGroup) {
- if (roleGroup == null) {
- return DEFAULT_METAINFO_MAP_KEY;
- }
- int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
- if (lastIndex == -1) {
- return DEFAULT_METAINFO_MAP_KEY;
- } else {
- return roleGroup.substring(0, lastIndex+1);
- }
- }
-
- static String getMetainfoComponentName(String roleGroup) {
- int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
- if (lastIndex == -1) {
+ static String getMetainfoComponentName(String roleGroup,
+ ConfTreeOperations appConf) throws BadConfigException {
+ String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
+ if (prefix == null) {
return roleGroup;
- } else {
- return roleGroup.substring(lastIndex+1);
}
+ if (!roleGroup.startsWith(prefix)) {
+ throw new BadConfigException("Component " + roleGroup + " doesn't start" +
+ " with prefix " + prefix);
+ }
+ return roleGroup.substring(prefix.length());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index e2c879e..4abac7a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -18,6 +18,8 @@
package org.apache.slider.providers.agent;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,9 +27,12 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
+
/**
* Stores the command dependency order for all components in a service. <commandOrder>
* <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START
@@ -39,19 +44,36 @@ public class ComponentCommandOrder {
private static char SPLIT_CHAR = '-';
Map<Command, Map<String, List<ComponentState>>> dependencies =
new HashMap<Command, Map<String, List<ComponentState>>>();
+ Map<String, Collection<String>> prefixRoleMap = new HashMap<>();
+ Map<String, String> rolePrefixMap = new HashMap<>();
public ComponentCommandOrder() {}
- public ComponentCommandOrder(List<CommandOrder> commandOrders) {
- mergeCommandOrders(commandOrders);
+ public ComponentCommandOrder(List<CommandOrder> commandOrders,
+ ConfTreeOperations resources) {
+ mergeCommandOrders(commandOrders, resources);
}
- void mergeCommandOrders(List<CommandOrder> commandOrders) {
+ void mergeCommandOrders(List<CommandOrder> commandOrders,
+ ConfTreeOperations resources) {
+ for (String component : resources.getComponentNames()) {
+ String prefix = SliderUtils.trimPrefix(
+ resources.getComponentOpt(component, ROLE_PREFIX, null));
+ if (prefix != null) {
+ rolePrefixMap.put(component, prefix);
+ if (!prefixRoleMap.containsKey(prefix)) {
+ prefixRoleMap.put(prefix, new HashSet<String>());
+ }
+ prefixRoleMap.get(prefix).add(component);
+ }
+ }
if (commandOrders != null && commandOrders.size() > 0) {
for (CommandOrder commandOrder : commandOrders) {
- ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand());
+ ComponentCommand componentCmd = getComponentCommand(
+ commandOrder.getCommand(), resources);
String requires = commandOrder.getRequires();
- List<ComponentState> requiredStates = parseRequiredStates(requires);
+ List<ComponentState> requiredStates = parseRequiredStates(requires,
+ resources);
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
@@ -71,7 +93,8 @@ public class ComponentCommandOrder {
}
}
- private List<ComponentState> parseRequiredStates(String requires) {
+ private List<ComponentState> parseRequiredStates(String requires,
+ ConfTreeOperations resources) {
if (requires == null || requires.length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
}
@@ -79,13 +102,14 @@ public class ComponentCommandOrder {
String[] componentStates = requires.split(",");
List<ComponentState> retList = new ArrayList<ComponentState>();
for (String componentStateStr : componentStates) {
- retList.add(getComponentState(componentStateStr));
+ retList.add(getComponentState(componentStateStr, resources));
}
return retList;
}
- private ComponentCommand getComponentCommand(String compCmdStr) {
+ private ComponentCommand getComponentCommand(String compCmdStr,
+ ConfTreeOperations resources) {
if (compCmdStr == null || compCmdStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
}
@@ -98,6 +122,11 @@ public class ComponentCommandOrder {
String compStr = compCmdStr.substring(0, splitIndex);
String cmdStr = compCmdStr.substring(splitIndex + 1);
+ if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
+ throw new IllegalArgumentException("Component " + compStr + " specified" +
+ " in command order does not exist");
+ }
+
Command cmd = Command.valueOf(cmdStr);
if (cmd != Command.START) {
@@ -106,7 +135,8 @@ public class ComponentCommandOrder {
return new ComponentCommand(compStr, cmd);
}
- private ComponentState getComponentState(String compStStr) {
+ private ComponentState getComponentState(String compStStr,
+ ConfTreeOperations resources) {
if (compStStr == null || compStStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null.");
}
@@ -119,6 +149,11 @@ public class ComponentCommandOrder {
String compStr = compStStr.substring(0, splitIndex);
String stateStr = compStStr.substring(splitIndex + 1);
+ if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
+ throw new IllegalArgumentException("Component " + compStr + " specified" +
+ " in command order does not exist");
+ }
+
State state = State.valueOf(stateStr);
if (state != State.STARTED && state != State.INSTALLED) {
throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
@@ -129,40 +164,43 @@ public class ComponentCommandOrder {
// dependency is still on component level, but not package level
// so use component name to check dependency, not component-package
public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) {
- boolean canExecute = true;
- if (dependencies.containsKey(command) && dependencies.get(command).containsKey(component)) {
- List<ComponentState> required = dependencies.get(command).get(component);
- for (ComponentState stateToMatch : required) {
- for (ComponentInstanceState currState : currentStates) {
- log.debug("Checking schedule {} {} against dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- if (currState.getComponentName().equals(stateToMatch.componentName)) {
- if (currState.getState() != stateToMatch.state) {
- if (stateToMatch.state == State.STARTED) {
+ if (!dependencies.containsKey(command)) {
+ return true;
+ }
+ List<ComponentState> required = new ArrayList<>();
+ if (dependencies.get(command).containsKey(component)) {
+ required.addAll(dependencies.get(command).get(component));
+ }
+ String prefix = rolePrefixMap.get(component);
+ if (prefix != null && dependencies.get(command).containsKey(prefix)) {
+ required.addAll(dependencies.get(command).get(prefix));
+ }
+
+ for (ComponentState stateToMatch : required) {
+ for (ComponentInstanceState currState : currentStates) {
+ log.debug("Checking schedule {} {} against dependency {} is {}",
+ component, command, currState.getComponentName(), currState.getState());
+ if (currState.getComponentName().equals(stateToMatch.componentName) ||
+ (prefixRoleMap.containsKey(stateToMatch.componentName) &&
+ prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
+ if (currState.getState() != stateToMatch.state) {
+ if (stateToMatch.state == State.STARTED) {
+ log.info("Cannot schedule {} {} as dependency {} is {}",
+ component, command, currState.getComponentName(), currState.getState());
+ return false;
+ } else {
+ //state is INSTALLED
+ if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- canExecute = false;
- } else {
- //state is INSTALLED
- if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- canExecute = false;
- }
+ component, command, currState.getComponentName(), currState.getState());
+ return false;
}
}
}
- if (!canExecute) {
- break;
- }
- }
- if (!canExecute) {
- break;
}
}
}
-
- return canExecute;
+ return true;
}
static class ComponentState {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 3213d93..3ba766f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1808,6 +1808,11 @@ public class AppState {
SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename),
groupOptions.options);
}
+ String prefix = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(role.getGroup(), ROLE_PREFIX, null);
+ if (SliderUtils.isSet(prefix)) {
+ cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix));
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
new file mode 100644
index 0000000..f67f83a
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
@@ -0,0 +1,12 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "test-external-component": {
+ "site.global.component_type": "external"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
index 073d1ff..bfe2afb 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
@@ -2,6 +2,12 @@
"schemaVersion": "2.1",
"application": {
"name": "SLEEPER",
+ "commandOrders": [
+ {
+ "command": "SLEEP_100-START",
+ "requires": "SLEEP_LONG-STARTED"
+ }
+ ],
"components": [
{
"name": "SLEEP_100",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
new file mode 100644
index 0000000..43a2200
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
@@ -0,0 +1,14 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "SLEEPER",
+ "commandOrders": [
+ {
+ "command": "test-external-component-test_sleep-SLEEP_100-START",
+ "requires": "test-external-component-SLEEP_100-STARTED"
+ }
+ ],
+ "components": [
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
new file mode 100644
index 0000000..8aa86b4
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
@@ -0,0 +1,12 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "384"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
new file mode 100644
index 0000000..8dd693e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.agent.AgentKeys
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+
+@CompileStatic
+@Slf4j
+
+class TestBuildExternalComponent extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "verify external components"
+
+ String echo = "echo"
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [(echo): 1],
+ [ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "2"],
+ true,
+ false,
+ agentDefOptions)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(clustername, false)
+
+ String parent1 = clustername + "_ext"
+ launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ parent1,
+ [(echo): 1],
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"],
+ true,
+ false,
+ agentDefOptions)
+ sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(parent1, false)
+ // verify generated conf
+ def aggregateConf = sliderClient.loadPersistedClusterDescription(parent1)
+ assert 3 == aggregateConf.resourceOperations.componentNames.size()
+ assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM)
+ assert aggregateConf.resourceOperations.componentNames.contains(echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo)
+
+ aggregateConf.resolve()
+ assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals(
+ aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF,
+ aggregateConf.appConfOperations.get(AgentKeys.APP_DEF)))
+ SliderFileSystem sfs = createSliderFileSystem()
+ String appdefdir = sfs.buildAppDefDirPath(parent1)
+ checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+
+ String parent2 = "parent"
+ launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ parent2,
+ [(echo): 1],
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"],
+ true,
+ false,
+ agentDefOptions)
+ sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(parent2, false)
+ // verify generated conf
+ aggregateConf = sliderClient.loadPersistedClusterDescription(parent2)
+ assert 5 == aggregateConf.resourceOperations.componentNames.size()
+ assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM)
+ assert aggregateConf.resourceOperations.componentNames.contains(echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo)
+
+ aggregateConf.resolve()
+ assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals(
+ aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF,
+ aggregateConf.appConfOperations.get(AgentKeys.APP_DEF)))
+ appdefdir = sfs.buildAppDefDirPath(parent2)
+ checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+ checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + echo, appdefdir)
+ checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+ }
+
+ private void checkComponent(AggregateConf aggConf, String component,
+ String appdefdir) {
+ String path = new Path(appdefdir, SliderUtils.trimPrefix(
+ aggConf.appConfOperations.getComponentOpt(component, RoleKeys
+ .ROLE_PREFIX, null)) + "_" + SliderKeys.DEFAULT_APP_PKG).toString()
+ assert path.equals(aggConf.appConfOperations.getComponentOpt(component,
+ AgentKeys.APP_DEF, null))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 07d21d7..a6d52b4 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -152,10 +152,6 @@ public class TestAgentProviderService {
+ " <command>HBASE_REGIONSERVER-START</command>\n"
+ " <requires>HBASE_MASTER-STARTED</requires>\n"
+ " </commandOrder>\n"
- + " <commandOrder>\n"
- + " <command>A-START</command>\n"
- + " <requires>B-STARTED</requires>\n"
- + " </commandOrder>\n"
+ " </commandOrders>\n"
+ " <components>\n"
+ " <component>\n"
@@ -434,6 +430,11 @@ public class TestAgentProviderService {
.put(AgentKeys.AGENT_CONF, ".");
instanceDefinition.getAppConfOperations().getGlobalOptions()
.put(AgentKeys.AGENT_VERSION, ".");
+
+ instanceDefinition.getResourceOperations().getOrAddComponent(
+ "HBASE_MASTER");
+ instanceDefinition.getResourceOperations().getOrAddComponent(
+ "HBASE_REGIONSERVER");
return instanceDefinition;
}
@@ -530,13 +531,7 @@ public class TestAgentProviderService {
assertNotNull(registryViewForProviders);
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
- AggregateConf instanceDefinition = new AggregateConf();
-
- instanceDefinition.setInternal(tree);
- instanceDefinition.setAppConf(tree);
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
Container container = createNiceMock(Container.class);
ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
@@ -1116,19 +1111,15 @@ public class TestAgentProviderService {
Assert.assertEquals(found, 2);
List<CommandOrder> cmdOrders = application.getCommandOrders();
- Assert.assertEquals(cmdOrders.size(), 2);
+ Assert.assertEquals(cmdOrders.size(), 1);
found = 0;
for (CommandOrder co : application.getCommandOrders()) {
if (co.getCommand().equals("HBASE_REGIONSERVER-START")) {
Assert.assertTrue(co.getRequires().equals("HBASE_MASTER-STARTED"));
found++;
}
- if (co.getCommand().equals("A-START")) {
- Assert.assertEquals(co.getRequires(), "B-STARTED");
- found++;
- }
}
- Assert.assertEquals(found, 2);
+ Assert.assertEquals(found, 1);
List<ConfigFile> configFiles = application.getConfigFiles();
Assert.assertEquals(configFiles.size(), 2);
@@ -1204,8 +1195,6 @@ public class TestAgentProviderService {
// Start of HBASE_RS depends on the start of HBASE_MASTER
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
- ConfTree tree = new ConfTree();
- tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
Configuration conf = new Configuration();
AgentProviderService aps = createAgentProviderService(conf);
@@ -1213,13 +1202,7 @@ public class TestAgentProviderService {
assertNotNull(registryViewForProviders);
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
- AggregateConf instanceDefinition = new AggregateConf();
-
- instanceDefinition.setInternal(tree);
- instanceDefinition.setAppConf(tree);
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
Container container = createNiceMock(Container.class);
ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
index c123fbb..0e3e3ad 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
@@ -18,20 +18,38 @@
package org.apache.slider.providers.agent;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
public class TestComponentCommandOrder {
protected static final Logger log =
LoggerFactory.getLogger(TestComponentCommandOrder.class);
private final MockContainerId containerId = new MockContainerId(1);
+ private static ConfTreeOperations resources = new ConfTreeOperations(
+ new ConfTree());
+
+ @BeforeClass
+ public static void init() {
+ resources.getOrAddComponent("A");
+ resources.getOrAddComponent("B");
+ resources.getOrAddComponent("C");
+ resources.getOrAddComponent("D");
+ resources.getOrAddComponent("E");
+ }
+
@Test
public void testComponentCommandOrder() throws Exception {
CommandOrder co1 = new CommandOrder();
@@ -44,7 +62,8 @@ public class TestComponentCommandOrder {
co3.setCommand("B-START");
co3.setRequires("C-STARTED,D-STARTED,E-INSTALLED");
- ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co1, co2, co3));
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1, co2, co3), resources);
ComponentInstanceState cisB = new ComponentInstanceState("B",
containerId, "aid");
ComponentInstanceState cisC = new ComponentInstanceState("C", containerId, "aid");
@@ -100,13 +119,14 @@ public class TestComponentCommandOrder {
cisB.setState(State.STARTED);
cisC.setState(State.STARTED);
- ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co));
+ ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co),
+ resources);
Assert.assertTrue(cco.canExecute("A", Command.START, Arrays.asList(cisB, cisC)));
co.setCommand(" A-STAR");
co.setRequires("B-STARTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -115,7 +135,7 @@ public class TestComponentCommandOrder {
co.setCommand(" -START");
co.setRequires("B-STARTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -124,7 +144,7 @@ public class TestComponentCommandOrder {
co.setCommand(" A-START");
co.setRequires("B-STRTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -133,7 +153,7 @@ public class TestComponentCommandOrder {
co.setCommand(" A-START");
co.setRequires("B-STARTED , C-");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -142,10 +162,83 @@ public class TestComponentCommandOrder {
co.setCommand(" A-INSTALL");
co.setRequires("B-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
+ Assert.fail("Instantiation should have failed.");
+ } catch (IllegalArgumentException ie) {
+ log.info(ie.getMessage());
+ }
+ }
+
+ @Test
+ public void testComponentCommandOrderBadComponent() throws Exception {
+ ConfTreeOperations resourcesGood = new ConfTreeOperations(new ConfTree());
+ resourcesGood.getOrAddComponent("A");
+ resourcesGood.getOrAddComponent("Z");
+ ConfTreeOperations resourcesBad = new ConfTreeOperations(new ConfTree());
+
+ CommandOrder co1 = new CommandOrder();
+ co1.setCommand("A-START");
+ co1.setRequires("Z-STARTED");
+ CommandOrder co2 = new CommandOrder();
+ co2.setCommand("Z-START");
+ co2.setRequires("A-STARTED");
+
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1), resourcesGood);
+ try {
+ cco = new ComponentCommandOrder(Arrays.asList(co1), resourcesBad);
+ Assert.fail("Instantiation should have failed.");
+ } catch (IllegalArgumentException ie) {
+ log.info(ie.getMessage());
+ }
+
+ cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesGood);
+ try {
+ cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesBad);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
}
}
+
+ @Test
+ public void testComponentCommandOrderPrefixes() throws Exception {
+ ConfTreeOperations resources = new ConfTreeOperations(new ConfTree());
+ resources.getOrAddComponent("a-A").put(ROLE_PREFIX, "a-");
+ resources.getOrAddComponent("b-B1").put(ROLE_PREFIX, "b-");
+ resources.getOrAddComponent("b-B2").put(ROLE_PREFIX, "b-");
+ resources.getOrAddComponent("c-C").put(ROLE_PREFIX, "c-");
+
+ CommandOrder co1 = new CommandOrder();
+ co1.setCommand("b-START");
+ co1.setRequires("a-STARTED");
+ CommandOrder co2 = new CommandOrder();
+ co2.setCommand("c-START");
+ co2.setRequires("b-STARTED");
+
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1, co2), resources);
+
+ ComponentInstanceState cisA = new ComponentInstanceState("a-A", containerId, "aid");
+ ComponentInstanceState cisB1 = new ComponentInstanceState("b-B1", containerId, "aid");
+ ComponentInstanceState cisB2 = new ComponentInstanceState("b-B2", containerId, "aid");
+ ComponentInstanceState cisC = new ComponentInstanceState("c-C", containerId, "aid");
+ cisA.setState(State.INSTALLED);
+ cisB1.setState(State.INSTALLED);
+ cisB2.setState(State.INSTALLED);
+ cisC.setState(State.INSTALLED);
+ List<ComponentInstanceState> states = Arrays.asList(cisA, cisB1, cisB2, cisC);
+ Assert.assertTrue(cco.canExecute("a-A", Command.START, states));
+ Assert.assertFalse(cco.canExecute("b-B1", Command.START, states));
+ Assert.assertFalse(cco.canExecute("b-B2", Command.START, states));
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisA.setState(State.STARTED);
+ Assert.assertTrue(cco.canExecute("b-B1", Command.START, states));
+ Assert.assertTrue(cco.canExecute("b-B2", Command.START, states));
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisB1.setState(State.STARTED);
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisB2.setState(State.STARTED);
+ Assert.assertTrue(cco.canExecute("c-C", Command.START, states));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
index 1cb6f0f..13919df 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@ -40,4 +40,8 @@ interface ResourcePaths {
String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json"
String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json"
+
+ String NESTED_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component_nested.json"
+ String NESTED_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json"
+ String NESTED_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
index b5e0270..292508a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
@@ -40,6 +40,7 @@ public class ExternalComponentIT extends AgentCommandTestBase
static String NAME = "test-external-component"
static String EXT_NAME = "test_sleep"
+ static String NESTED_NAME = "test-external-component-nested"
static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG
static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
@@ -47,23 +48,30 @@ public class ExternalComponentIT extends AgentCommandTestBase
static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG
static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
static String TEST_METAINFO = ResourcePaths.SLEEP_META
+ static String NEST_APPCONFIG = ResourcePaths.NESTED_APPCONFIG
+ static String NEST_RESOURCES = ResourcePaths.NESTED_RESOURCES
+ static String NEST_METAINFO = ResourcePaths.NESTED_META
public static final String SLEEP_100 = "SLEEP_100"
public static final String SLEEP_LONG = "SLEEP_LONG"
public static final String EXT_SLEEP_100 = EXT_NAME +
SliderKeys.COMPONENT_SEPARATOR + SLEEP_100
public static final String EXT_SLEEP_LONG = EXT_NAME +
SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG
+ public static final String NESTED_PREFIX = NAME +
+ SliderKeys.COMPONENT_SEPARATOR
@Before
public void prepareCluster() {
setupCluster(NAME)
setupCluster(EXT_NAME)
+ setupCluster(NESTED_NAME)
}
@After
public void destroyCluster() {
cleanup(NAME)
cleanup(EXT_NAME)
+ cleanup(NESTED_NAME)
}
@Test
@@ -129,5 +137,35 @@ public class ExternalComponentIT extends AgentCommandTestBase
expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0,
CONTAINER_LAUNCH_TIMEOUT)
+ cleanup(NAME)
+
+ describe NESTED_NAME
+
+ slider(0, [ACTION_BUILD, NAME, ARG_METAINFO, TEST_METAINFO,
+ ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES])
+
+ slider(0, [ACTION_CREATE, NESTED_NAME, ARG_METAINFO, NEST_METAINFO,
+ ARG_TEMPLATE, NEST_APPCONFIG, ARG_RESOURCES, NEST_RESOURCES])
+
+ ensureApplicationIsUp(NESTED_NAME)
+ status(0, NESTED_NAME)
+
+ cd = execStatus(NESTED_NAME)
+
+ assert 5 == cd.statistics.size()
+ assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM,
+ NESTED_PREFIX + SLEEP_100,
+ NESTED_PREFIX + SLEEP_LONG,
+ NESTED_PREFIX + EXT_SLEEP_100,
+ NESTED_PREFIX + EXT_SLEEP_LONG])
+
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
}
}