You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by bi...@apache.org on 2016/08/12 21:45:59 UTC
[1/6] incubator-slider git commit: SLIDER-875 initial uber app support
Repository: incubator-slider
Updated Branches:
refs/heads/develop 7428cae4c -> 50db94ce4
SLIDER-875 initial uber app support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/000b38d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/000b38d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/000b38d5
Branch: refs/heads/develop
Commit: 000b38d5c9363efe7561a0104d9c65291c9f5214
Parents: c0f4e92
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Jun 21 16:59:34 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Jun 23 09:06:08 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 4 +-
.../org/apache/slider/common/SliderKeys.java | 7 +
.../apache/slider/common/tools/SliderUtils.java | 79 ++++++++
.../slider/core/build/InstanceBuilder.java | 148 +++++++++++++++
.../slider/core/conf/ConfTreeOperations.java | 11 ++
.../providers/AbstractProviderService.java | 3 +-
.../slider/providers/ProviderService.java | 4 +-
.../providers/agent/AgentClientProvider.java | 30 +--
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 181 +++++++++++++------
.../slider/providers/agent/AgentUtils.java | 24 +++
.../providers/agent/ComponentCommandOrder.java | 6 +
.../server/appmaster/SliderAppMaster.java | 3 +-
.../appmaster/web/rest/agent/AgentResource.java | 4 +-
.../sleep_cmd/appConfig_external_component.json | 12 ++
.../sleep_cmd/resources_external_component.json | 22 +++
.../slider/client/TestReplaceTokens.groovy | 5 +-
.../model/mock/MockProviderService.groovy | 3 +-
.../agent/TestAgentProviderService.java | 36 ++--
.../apache/slider/funtest/ResourcePaths.groovy | 2 +
.../funtest/misc/ExternalComponentIT.groovy | 133 ++++++++++++++
21 files changed, 626 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dd90e46..372ec48 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1597,6 +1597,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
resources.mergeComponents(buildInfo.getResourceCompOptionMap());
builder.init(providerName, instanceDefinition);
+ builder.resolve();
builder.propagateFilename();
builder.propagatePrincipals();
builder.setImageDetailsIfAvailable(buildInfo.getImage(),
@@ -1786,8 +1787,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private static String replaceTokens(String s, String userName,
String clusterName) throws IOException {
return s.replaceAll(Pattern.quote("${USER}"), userName)
- .replaceAll(Pattern.quote("${USER_NAME}"), userName)
- .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName);
+ .replaceAll(Pattern.quote("${USER_NAME}"), userName);
}
public FsPermission getClusterDirectoryPermissions(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 05c7048..3921324 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -70,6 +70,13 @@ public interface SliderKeys extends SliderXmlConfKeys {
String APP_TYPE = "org-apache-slider";
/**
+ * Key for component type. This MUST NOT be set in app_config/global {@value}
+ */
+ String COMPONENT_TYPE = "site.global.component_type";
+ String EXTERNAL_COMPONENT = "external";
+ String COMPONENT_SEPARATOR = "-";
+
+ /**
* Key for application version. This must be set in app_config/global {@value}
*/
String APP_VERSION = "site.global.app_version";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 746e468..c48e109 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -69,6 +69,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.ClasspathConstructor;
import org.apache.slider.core.main.LauncherExitCodes;
import org.apache.slider.providers.agent.AgentKeys;
+import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.server.services.utility.PatternValidator;
import org.apache.slider.server.services.workflow.ForkedProcessService;
import org.apache.zookeeper.server.util.KerberosUtil;
@@ -475,6 +476,32 @@ public final class SliderUtils {
return srcFileCount;
}
+ /**
+ * Copy a file to a new FS -both paths must be qualified.
+ * @param conf conf file
+ * @param srcFile src file
+ * @param destFile dest file
+ */
+ public static void copy(Configuration conf,
+ Path srcFile,
+ Path destFile) throws
+ IOException,
+ BadClusterStateException {
+ FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
+ //list all paths in the src.
+ if (!srcFS.exists(srcFile)) {
+ throw new FileNotFoundException("Source dir not found " + srcFile);
+ }
+ if (!srcFS.isFile(srcFile)) {
+ throw new FileNotFoundException(
+ "Source dir not a file " + srcFile);
+ }
+ FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
+ if (destFS.exists(destFile)) {
+ throw new IOException("Dest file already exists " + destFile);
+ }
+ FileUtil.copy(srcFS, srcFile, destFS, destFile, false, true, conf);
+ }
public static String stringify(Throwable t) {
StringWriter sw = new StringWriter();
@@ -927,6 +954,38 @@ public final class SliderUtils {
}
/**
+ * Merge string maps excluding prefixes
+ * @param first first map
+ * @param second second map
+ * @param prefixes prefixes to ignore
+ * @return 'first' merged with the second
+ */
+ public static Map<String, String> mergeMapsIgnorePrefixes(
+ Map<String, String> first, Map<String, String> second,
+ String... prefixes) {
+ Preconditions.checkArgument(first != null, "Null 'first' value");
+ Preconditions.checkArgument(second != null, "Null 'second' value");
+ Preconditions.checkArgument(prefixes != null, "Null 'prefixes' value");
+ for (Map.Entry<String, String> entry : second.entrySet()) {
+ String key = entry.getKey();
+ boolean hasPrefix = false;
+ for (String prefix : prefixes) {
+ if (key.startsWith(prefix)) {
+ hasPrefix = true;
+ break;
+ }
+ }
+ if (hasPrefix) {
+ continue;
+ }
+ if (!first.containsKey(key)) {
+ first.put(key, entry.getValue());
+ }
+ }
+ return first;
+ }
+
+ /**
* Convert a map to a multi-line string for printing
* @param map map to stringify
* @return a string representation of the map
@@ -2340,8 +2399,28 @@ public final class SliderUtils {
*/
public static String getApplicationDefinitionPath(ConfTreeOperations conf)
throws BadConfigException {
+ return getApplicationDefinitionPath(conf, null);
+ }
+
+ /**
+ * return the HDFS path where the application package has been uploaded
+ * manually or by using slider client (install package command)
+ *
+ * @param conf configuration
+ * @param roleGroup name of component
+ * @return
+ */
+ public static String getApplicationDefinitionPath(ConfTreeOperations conf,
+ String roleGroup)
+ throws BadConfigException {
String appDefPath = conf.getGlobalOptions().getMandatoryOption(
AgentKeys.APP_DEF);
+ if (roleGroup != null) {
+ MapOperations component = conf.getComponent(roleGroup);
+ if (component != null) {
+ appDefPath = component.getOption(AgentKeys.APP_DEF, appDefPath);
+ }
+ }
return appDefPath;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 7756055..4afdf7c 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -25,7 +25,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
+import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.StatusKeys;
+import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.SliderUtils;
@@ -42,11 +44,17 @@ import org.apache.slider.core.persist.LockAcquireFailedException;
import org.apache.slider.core.persist.LockHeldAction;
import org.apache.slider.core.zk.ZKPathBuilder;
import org.apache.slider.core.zk.ZookeeperUtils;
+import org.apache.slider.providers.agent.AgentKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeSet;
import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH;
import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH;
@@ -61,6 +69,11 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
+import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE;
+import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT;
+import static org.apache.slider.common.tools.SliderUtils.isClusternameValid;
/**
* Build up the instance of a cluster.
@@ -72,6 +85,8 @@ public class InstanceBuilder {
private final CoreFileSystem coreFS;
private final InstancePaths instancePaths;
private AggregateConf instanceDescription;
+ private Map<Path, Path> externalAppDefs = new HashMap<>();
+ private TreeSet<Integer> priorities = new TreeSet<>();
private static final Logger log =
LoggerFactory.getLogger(InstanceBuilder.class);
@@ -244,6 +259,136 @@ public class InstanceBuilder {
}
+ private void getExternalComponents(ConfTreeOperations ops,
+ Set<String> externalComponents) throws BadConfigException {
+ if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) {
+ throw new BadConfigException(COMPONENT_TYPE + " must be " +
+ "specified per-component, not in global");
+ }
+
+ for (Entry<String, Map<String, String>> entry : ops.getComponents()
+ .entrySet()) {
+ if (COMPONENT_AM.equals(entry.getKey())) {
+ continue;
+ }
+ Map<String, String> options = entry.getValue();
+ if (options.containsKey(COMPONENT_TYPE) &&
+ EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
+ externalComponents.add(entry.getKey());
+ }
+ }
+ }
+
+ private static String[] PREFIXES_TO_SKIP = {"zookeeper.",
+ "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."};
+
+ private void mergeExternalComponent(ConfTreeOperations ops,
+ ConfTreeOperations externalOps, String externalComponent,
+ Integer priority) {
+ for (String subComponent : externalOps.getComponentNames()) {
+ if (COMPONENT_AM.equals(subComponent)) {
+ continue;
+ }
+ log.debug("Merging options for {} into {}", subComponent,
+ externalComponent + COMPONENT_SEPARATOR + subComponent);
+ MapOperations subComponentOps = ops.getOrAddComponent(externalComponent +
+ COMPONENT_SEPARATOR + subComponent);
+ SliderUtils.mergeMapsIgnorePrefixes(subComponentOps,
+ externalOps.getComponent(subComponent), PREFIXES_TO_SKIP);
+ if (priority != null) {
+ subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY,
+ Integer.toString(priority));
+ priorities.add(priority);
+ priority++;
+ }
+ }
+ }
+
+ private int getNextPriority() {
+ if (priorities.isEmpty()) {
+ return 1;
+ } else {
+ return priorities.last() + 1;
+ }
+ }
+
+ public void resolve()
+ throws BadConfigException, IOException, BadClusterStateException {
+ ConfTreeOperations appConf = instanceDescription.getAppConfOperations();
+ ConfTreeOperations resources = instanceDescription.getResourceOperations();
+
+ for (Entry<String, Map<String, String>> entry : resources.getComponents()
+ .entrySet()) {
+ if (COMPONENT_AM.equals(entry.getKey())) {
+ continue;
+ }
+ if (entry.getKey().contains(COMPONENT_SEPARATOR)) {
+ throw new BadConfigException("Components must not contain " +
+ COMPONENT_SEPARATOR + ": " + entry.getKey());
+ }
+ if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) {
+ priorities.add(Integer.parseInt(entry.getValue().get(
+ ResourceKeys.COMPONENT_PRIORITY)));
+ }
+ }
+
+ Set<String> externalComponents = new HashSet<>();
+ getExternalComponents(appConf, externalComponents);
+ if (!externalComponents.isEmpty()) {
+ log.info("Found external components {}", externalComponents);
+ }
+
+ for (String component : externalComponents) {
+ if (!isClusternameValid(component)) {
+ throw new BadConfigException(component + " is not a valid external " +
+ "component");
+ }
+ Path componentClusterDir = coreFS.buildClusterDirPath(component);
+ try {
+ coreFS.verifyPathExists(componentClusterDir);
+ } catch (IOException e) {
+ throw new BadConfigException("external component " + component +
+ " doesn't exist");
+ }
+ AggregateConf componentConf = new AggregateConf();
+ ConfPersister persister = new ConfPersister(coreFS, componentClusterDir);
+ try {
+ persister.load(componentConf);
+ } catch (Exception e) {
+ throw new BadConfigException("Couldn't read configuration for " +
+ "external component " + component);
+ }
+ String externalAppDef = componentConf.getAppConfOperations()
+ .getGlobalOptions().get(AgentKeys.APP_DEF);
+ if (SliderUtils.isSet(externalAppDef)) {
+ Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
+ component + "_" + SliderKeys.DEFAULT_APP_PKG);
+ componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef);
+ externalAppDefs.put(newAppDef, new Path(externalAppDef));
+ }
+ for (String rcomp : componentConf.getResourceOperations()
+ .getComponentNames()) {
+ if (COMPONENT_AM.equals(rcomp)) {
+ continue;
+ }
+ log.debug("Adding component {} to appConf for {}", rcomp, component);
+ componentConf.getAppConfOperations().getOrAddComponent(rcomp);
+ }
+ SliderUtils.mergeMaps(
+ componentConf.getAppConfOperations().getGlobalOptions().options,
+ appConf.getComponent(component).options);
+ componentConf.getAppConfOperations().getGlobalOptions()
+ .remove(COMPONENT_TYPE);
+ componentConf.resolve();
+
+ mergeExternalComponent(appConf, componentConf.getAppConfOperations(),
+ component, null);
+ mergeExternalComponent(resources, componentConf.getResourceOperations(),
+ component, getNextPriority());
+ }
+ }
+
+
/**
* Persist this
* @param appconfdir conf dir
@@ -266,6 +411,9 @@ public class InstanceBuilder {
action = new ConfDirSnapshotAction(appconfdir);
}
persister.save(instanceDescription, action);
+ for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) {
+ SliderUtils.copy(conf, appDef.getValue(), appDef.getKey());
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 9013edb..4a0ae41 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -257,6 +257,17 @@ public class ConfTreeOperations {
Map<String, String> map,
String prefix,
boolean overwrite) {
+ boolean needsMerge = false;
+ for (Map.Entry<String, String> entry : map.entrySet()) {
+ String key = entry.getKey();
+ if (key.startsWith(prefix)) {
+ needsMerge = true;
+ break;
+ }
+ }
+ if (!needsMerge) {
+ return;
+ }
MapOperations comp = getOrAddComponent(component);
comp.mergeMapPrefixedKeys(map,prefix, overwrite);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index 61b2655..92766f5 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -169,7 +169,8 @@ public abstract class AbstractProviderService
*/
@Override
public void initializeApplicationConfiguration(
- AggregateConf instanceDefinition, SliderFileSystem fileSystem)
+ AggregateConf instanceDefinition, SliderFileSystem fileSystem,
+ String roleGroup)
throws IOException, SliderException {
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index f754eee..3f24665 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -118,11 +118,13 @@ public interface ProviderService extends ProviderCore,
*
* @param instanceDefinition
* @param fileSystem
+ * @param roleGroup
* @throws IOException
* @throws SliderException
*/
void initializeApplicationConfiguration(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException;
+ SliderFileSystem fileSystem, String roleGroup) throws IOException,
+ SliderException;
/**
* This is a validation of the application configuration on the AM.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index f3dcd1d..6eae75e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -74,6 +74,8 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName;
+
/** This class implements the client-side aspects of the agent deployer */
public class AgentClientProvider extends AbstractClientProvider
implements AgentKeys, SliderKeys {
@@ -168,13 +170,21 @@ public class AgentClientProvider extends AbstractClientProvider
names.remove(SliderKeys.COMPONENT_AM);
Map<Integer, String> priorityMap = new HashMap<Integer, String>();
- Metainfo metaInfo = getMetainfo(fs, appDef);
-
for (String name : names) {
+ try {
+ // Validate the app definition
+ appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
+ .getAppConfOperations(), name);
+ } catch (BadConfigException bce) {
+ throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
+ }
+ Metainfo metaInfo = getMetainfo(fs, appDef);
+
MapOperations component = resources.getMandatoryComponent(name);
if (metaInfo != null) {
- Component componentDef = metaInfo.getApplicationComponent(name);
+ Component componentDef = metaInfo.getApplicationComponent(
+ getMetainfoComponentName(name));
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
@@ -200,16 +210,12 @@ public class AgentClientProvider extends AbstractClientProvider
existing);
}
priorityMap.put(priority, name);
- }
- // fileSystem may be null for tests
- if (metaInfo != null) {
- for (String name : names) {
- Component componentDef = metaInfo.getApplicationComponent(name);
- if (componentDef == null) {
- throw new BadConfigException(
- "Component %s is not a member of application.", name);
- }
+ // fileSystem may be null for tests
+ if (metaInfo != null) {
+ Component componentDef = metaInfo.getApplicationComponent(
+ getMetainfoComponentName(name));
+ // already checked it wasn't null
// ensure that intance count is 0 for client components
if ("CLIENT".equals(componentDef.getCategory())) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index b027939..8514401 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -97,6 +97,8 @@ public interface AgentKeys {
String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
+
+ String DEFAULT_METAINFO_MAP_KEY = "DEFAULT";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f20757a..66fac99 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -71,6 +71,7 @@ import org.apache.slider.providers.ProviderRole;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.AbstractComponent;
import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
@@ -124,7 +125,6 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
@@ -132,8 +132,8 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.regex.Pattern;
+import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
/**
@@ -168,9 +168,8 @@ public class AgentProviderService extends AbstractProviderService implements
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Metainfo metaInfo = null;
- private Map<String, DefaultConfig> defaultConfigs = null;
- private ComponentCommandOrder commandOrder = null;
+ private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>();
+ private ComponentCommandOrder commandOrder = new ComponentCommandOrder();
private HeartbeatMonitor monitor;
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@ -205,6 +204,17 @@ public class AgentProviderService extends AbstractProviderService implements
private final Map<String, Set<String>> containerExportsMap =
new HashMap<String, Set<String>>();
+ private static class MetainfoHolder {
+ Metainfo metaInfo;
+ private Map<String, DefaultConfig> defaultConfigs = null;
+
+ public MetainfoHolder(Metainfo metaInfo,
+ Map<String, DefaultConfig> defaultConfigs) {
+ this.metaInfo = metaInfo;
+ this.defaultConfigs = defaultConfigs;
+ }
+ }
+
/**
* Create an instance of AgentProviderService
*/
@@ -249,10 +259,11 @@ public class AgentProviderService extends AbstractProviderService implements
Set<String> names = resources.getComponentNames();
names.remove(SliderKeys.COMPONENT_AM);
for (String name : names) {
- Component componentDef = getMetaInfo().getApplicationComponent(name);
+ Component componentDef = getApplicationComponent(name);
if (componentDef == null) {
- throw new BadConfigException(
- "Component %s is not a member of application.", name);
+ // component member is validated elsewhere, so we don't need to throw
+ // an exception here
+ continue;
}
MapOperations componentConfig = resources.getMandatoryComponent(name);
@@ -274,31 +285,62 @@ public class AgentProviderService extends AbstractProviderService implements
// Reads the metainfo.xml in the application package and loads it
private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException {
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
+ SliderFileSystem fileSystem,
+ String roleGroup)
+ throws IOException, SliderException {
+ String mapKey = getMetainfoMapKey(roleGroup);
+ String appDef = SliderUtils.getApplicationDefinitionPath(
+ instanceDefinition.getAppConfOperations(), roleGroup);
+ MapOperations component = null;
+ if (roleGroup != null) {
+ component = instanceDefinition.getAppConfOperations().getComponent(roleGroup);
+ }
- if (metaInfo == null) {
+ MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey);
+ if (metaInfoHolder == null) {
synchronized (syncLock) {
- if (metaInfo == null) {
+ metaInfoHolder = metaInfoMap.get(mapKey);
+ if (metaInfoHolder == null) {
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
- metaInfo = getApplicationMetainfo(fileSystem, appDef, false);
+ Metainfo metaInfo = getApplicationMetainfo(fileSystem, appDef, false);
log.info("Master package metainfo: {}", metaInfo.toString());
if (metaInfo == null || metaInfo.getApplication() == null) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
throw new SliderException(
"metainfo.xml is required in app package.");
}
- commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders());
- defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo);
+ List<CommandOrder> commandOrders = metaInfo.getApplication()
+ .getCommandOrders();
+ if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
+ for (Component comp : metaInfo.getApplication().getComponents()) {
+ comp.setName(mapKey + comp.getName());
+ log.info("Modifying external metainfo component name to {}",
+ comp.getName());
+ }
+ String commandPrefix = mapKey.substring(0,
+ mapKey.indexOf(COMPONENT_SEPARATOR)+1);
+ for (CommandOrder co : commandOrders) {
+ log.info("Adding prefix {} to command order {}",
+ commandPrefix, co);
+ co.setCommand(commandPrefix + co.getCommand());
+ co.setRequires(commandPrefix + co.getRequires());
+ }
+ }
+ commandOrder.mergeCommandOrders(commandOrders);
+ Map<String, DefaultConfig> defaultConfigs =
+ initializeDefaultConfigs(fileSystem, appDef, metaInfo);
+ metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
// build a map from component to metainfo
String addonAppDefString = instanceDefinition.getAppConfOperations()
.getGlobalOptions().getOption(AgentKeys.ADDONS, null);
+ if (component != null) {
+ addonAppDefString = component.getOption(AgentKeys.ADDONS, addonAppDefString);
+ }
log.debug("All addon appdefs: {}", addonAppDefString);
if (addonAppDefString != null) {
Scanner scanner = new Scanner(addonAppDefString).useDelimiter(",");
@@ -306,6 +348,9 @@ public class AgentProviderService extends AbstractProviderService implements
String addonAppDef = scanner.next();
String addonAppDefPath = instanceDefinition
.getAppConfOperations().getGlobalOptions().get(addonAppDef);
+ if (component != null) {
+ addonAppDefPath = component.getOption(addonAppDef, addonAppDefPath);
+ }
log.debug("Addon package {} is stored at: {}", addonAppDef
+ addonAppDefPath);
Metainfo addonMetaInfo = getApplicationMetainfo(fileSystem,
@@ -324,9 +369,10 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void initializeApplicationConfiguration(
- AggregateConf instanceDefinition, SliderFileSystem fileSystem)
+ AggregateConf instanceDefinition, SliderFileSystem fileSystem,
+ String roleGroup)
throws IOException, SliderException {
- buildMetainfo(instanceDefinition, fileSystem);
+ buildMetainfo(instanceDefinition, fileSystem, roleGroup);
}
@Override
@@ -345,9 +391,9 @@ public class AgentProviderService extends AbstractProviderService implements
String roleName = providerRole.name;
String roleGroup = providerRole.group;
String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
+ .getAppConfOperations(), roleGroup);
- initializeApplicationConfiguration(instanceDefinition, fileSystem);
+ initializeApplicationConfiguration(instanceDefinition, fileSystem, roleGroup);
log.info("Build launch context for Agent");
log.debug(instanceDefinition.toString());
@@ -564,7 +610,7 @@ public class AgentProviderService extends AbstractProviderService implements
// initialize the component instance state
getComponentStatuses().put(label,
new ComponentInstanceState(
- roleName,
+ roleGroup,
container.getId(),
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME),
pkgStatuses));
@@ -740,11 +786,12 @@ public class AgentProviderService extends AbstractProviderService implements
.extractRole(container));
if (role != null) {
String roleName = role.name;
- String label = getContainerLabel(container, roleName, role.group);
+ String roleGroup = role.group;
+ String label = getContainerLabel(container, roleName, roleGroup);
log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
container.getId(), roleName, applicationId);
getComponentStatuses().put(label,
- new ComponentInstanceState(roleName, container.getId(),
+ new ComponentInstanceState(roleGroup, container.getId(),
applicationId));
} else {
log.warn("Role not found for container {} in cluster {}",
@@ -865,7 +912,7 @@ public class AgentProviderService extends AbstractProviderService implements
StateAccessForProviders accessor = getAmState();
CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
- List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleGroup).getCommands();
+ List<ComponentCommand> commands = getApplicationComponent(roleGroup).getCommands();
if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup)
&& (cmdScript == null || cmdScript.getScript() == null)
@@ -1143,7 +1190,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private boolean isDockerContainer(String roleGroup) {
- String type = getMetaInfo().getApplicationComponent(roleGroup).getType();
+ String type = getApplicationComponent(roleGroup).getType();
if (SliderUtils.isSet(type)) {
return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
}
@@ -1151,7 +1198,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private boolean isYarnDockerContainer(String roleGroup) {
- String type = getMetaInfo().getApplicationComponent(roleGroup).getType();
+ String type = getApplicationComponent(roleGroup).getType();
if (SliderUtils.isSet(type)) {
return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
}
@@ -1345,8 +1392,12 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected Metainfo getMetaInfo() {
- return this.metaInfo;
+ protected Metainfo getMetaInfo(String roleGroup) {
+ MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup));
+ if (mh == null) {
+ return null;
+ }
+ return mh.metaInfo;
}
@VisibleForTesting
@@ -1416,8 +1467,8 @@ public class AgentProviderService extends AbstractProviderService implements
return defaultConfigMap;
}
- protected Map<String, DefaultConfig> getDefaultConfigs() {
- return defaultConfigs;
+ protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) {
+ return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs;
}
private int getHeartbeatMonitorInterval() {
@@ -1583,9 +1634,9 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Status report: {}", status.toString());
if (status.getConfigs() != null) {
- Application application = getMetaInfo().getApplication();
+ Application application = getMetaInfo(componentGroup).getApplication();
- if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) {
+ if (canAnyMasterPublishConfig(componentGroup) == false || canPublishConfig(componentGroup)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -1727,7 +1778,7 @@ public class AgentProviderService extends AbstractProviderService implements
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<String, String>();
- Application application = getMetaInfo().getApplication();
+ Application application = getMetaInfo(componentGroup).getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(componentGroup)) {
if (component.getComponentExports().size() > 0) {
@@ -1778,8 +1829,8 @@ public class AgentProviderService extends AbstractProviderService implements
String portVarFormat = "${site.%s}";
String hostNamePattern = "${" + compGroup + "_HOST}";
- List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups();
- Component component = getMetaInfo().getApplicationComponent(compGroup);
+ List<ExportGroup> appExportGroups = getMetaInfo(compGroup).getApplication().getExportGroups();
+ Component component = getApplicationComponent(compGroup);
if (component != null && SliderUtils.isSet(component.getCompExports())
&& SliderUtils.isNotEmpty(appExportGroups)) {
@@ -1881,7 +1932,11 @@ public class AgentProviderService extends AbstractProviderService implements
* @return the component entry or null for no match
*/
protected Component getApplicationComponent(String roleGroup) {
- return getMetaInfo().getApplicationComponent(roleGroup);
+ Metainfo metainfo = getMetaInfo(roleGroup);
+ if (metainfo == null) {
+ return null;
+ }
+ return metainfo.getApplicationComponent(roleGroup);
}
/**
@@ -1950,9 +2005,9 @@ public class AgentProviderService extends AbstractProviderService implements
*
* @return true if the condition holds
*/
- protected boolean canAnyMasterPublishConfig() {
+ protected boolean canAnyMasterPublishConfig(String roleGroup) {
if (canAnyMasterPublish == null) {
- Application application = getMetaInfo().getApplication();
+ Application application = getMetaInfo(roleGroup).getApplication();
if (application == null) {
log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
} else {
@@ -2027,7 +2082,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setPkg(pkg);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
- hostLevelParams.put(PACKAGE_LIST, getPackageList());
+ hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -2076,7 +2131,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setComponentName(roleName);
cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(PACKAGE_LIST, getPackageList());
+ hostLevelParams.put(PACKAGE_LIST, getPackageList(roleGroup));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -2096,7 +2151,7 @@ public class AgentProviderService extends AbstractProviderService implements
configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
- cmd.addContainerDetails(roleGroup, getMetaInfo());
+ cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
Map<String, String> dockerConfig = new HashMap<String, String>();
if(isYarnDockerContainer(roleGroup)){
@@ -2179,8 +2234,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- private String getPackageList() {
- return getPackageListFromApplication(getMetaInfo().getApplication());
+ private String getPackageList(String roleGroup) {
+ return getPackageListFromApplication(getMetaInfo(roleGroup).getApplication());
}
private void prepareExecutionCommand(ExecutionCommand cmd) {
@@ -2345,7 +2400,7 @@ public class AgentProviderService extends AbstractProviderService implements
private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup,
String configName){
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String containerName = getMetaInfo().getApplicationComponent(roleGroup)
+ String containerName = getApplicationComponent(roleGroup)
.getDockerContainers().get(0).getName();
String composedConfigName = null;
String appConfigValue = null;
@@ -2486,7 +2541,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setConfigurations(configurations);
// configurations.get("global").put("exec_cmd", startCommand.getExec());
- cmd.addContainerDetails(roleGroup, getMetaInfo());
+ cmd.addContainerDetails(roleGroup, getMetaInfo(roleGroup));
log.info("Docker- command: {}", cmd.toString());
@@ -2496,7 +2551,7 @@ public class AgentProviderService extends AbstractProviderService implements
private String getConfigFromMetaInfo(String roleGroup, String configName) {
String result = null;
- List<DockerContainer> containers = getMetaInfo().getApplicationComponent(
+ List<DockerContainer> containers = getApplicationComponent(
roleGroup).getDockerContainers();// to support multi container per
// component later
log.debug("Docker- containers metainfo: {}", containers.toString());
@@ -2798,10 +2853,11 @@ public class AgentProviderService extends AbstractProviderService implements
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens, containerId, roleName);
+ configurations, tokens, containerId, roleName,
+ roleGroup);
if (appConf.getComponent(roleGroup) != null) {
addNamedConfiguration(configType, appConf.getComponent(roleGroup).options,
- configurations, tokens, containerId, roleName);
+ configurations, tokens, containerId, roleName, roleGroup);
}
}
@@ -2844,15 +2900,32 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
+ String mapKey = getMetainfoMapKey(componentGroup);
+ String dataDirSuffix = "";
+ if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
+ dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1);
+ } else {
+ mapKey = "";
+ }
+ mapKey = mapKey.toLowerCase();
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
- .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
+ .getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH) + dataDirSuffix);
tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
tokens.put("${COMPONENT_NAME}", componentName);
+ tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
+ tokens.put("${COMPONENT_PREFIX}", mapKey);
+ tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase());
if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) {
tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length()));
}
+ tokens.put("${CLUSTER_NAME}", getClusterName());
+ tokens.put("${CLUSTER_NAME.lc}", getClusterName().toLowerCase());
+ tokens.put("${APP_NAME}", getClusterName());
+ tokens.put("${APP_NAME.lc}", getClusterName().toLowerCase());
+ tokens.put("${APP_COMPONENT_NAME}", componentName);
+ tokens.put("${APP_COMPONENT_NAME.lc}", componentName.toLowerCase());
return tokens;
}
@@ -2877,12 +2950,12 @@ public class AgentProviderService extends AbstractProviderService implements
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
- List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles();
+ List<ConfigFile> configFiles = getMetaInfo(roleGroup).getApplication().getConfigFiles();
for (ConfigFile configFile : configFiles) {
log.info("Expecting config type {}.", configFile.getDictionaryName());
configList.add(configFile.getDictionaryName());
}
- for (Component component : getMetaInfo().getApplication().getComponents()) {
+ for (Component component : getMetaInfo(roleGroup).getApplication().getComponents()) {
if (!component.getName().equals(roleGroup)) {
continue;
}
@@ -2907,7 +2980,7 @@ public class AgentProviderService extends AbstractProviderService implements
private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
Map<String, Map<String, String>> configurations,
Map<String, String> tokens, String containerId,
- String roleName) {
+ String roleName, String roleGroup) {
Map<String, String> config = new HashMap<String, String>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
addDefaultGlobalConfig(config, containerId, roleName);
@@ -2936,9 +3009,9 @@ public class AgentProviderService extends AbstractProviderService implements
}
//apply defaults only if the key is not present and value is not empty
- if (getDefaultConfigs().containsKey(configName)) {
+ if (getDefaultConfigs(roleGroup).containsKey(configName)) {
log.info("Adding default configs for type {}.", configName);
- for (PropertyInfo defaultConfigProp : getDefaultConfigs().get(configName).getPropertyInfos()) {
+ for (PropertyInfo defaultConfigProp : getDefaultConfigs(roleGroup).get(configName).getPropertyInfos()) {
if (!config.containsKey(defaultConfigProp.getName())) {
if (!defaultConfigProp.getName().isEmpty() &&
defaultConfigProp.getValue() != null &&
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index cfcfc5d..ed5108c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -35,6 +35,9 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY;
+
/**
*
*/
@@ -131,4 +134,25 @@ public class AgentUtils {
return new DefaultConfigParser().parse(configStream);
}
+
+ static String getMetainfoMapKey(String roleGroup) {
+ if (roleGroup == null) {
+ return DEFAULT_METAINFO_MAP_KEY;
+ }
+ int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
+ if (lastIndex == -1) {
+ return DEFAULT_METAINFO_MAP_KEY;
+ } else {
+ return roleGroup.substring(0, lastIndex+1);
+ }
+ }
+
+ static String getMetainfoComponentName(String roleGroup) {
+ int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
+ if (lastIndex == -1) {
+ return roleGroup;
+ } else {
+ return roleGroup.substring(lastIndex+1);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index 91f1259..e2c879e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -40,7 +40,13 @@ public class ComponentCommandOrder {
Map<Command, Map<String, List<ComponentState>>> dependencies =
new HashMap<Command, Map<String, List<ComponentState>>>();
+ public ComponentCommandOrder() {}
+
public ComponentCommandOrder(List<CommandOrder> commandOrders) {
+ mergeCommandOrders(commandOrders);
+ }
+
+ void mergeCommandOrders(List<CommandOrder> commandOrders) {
if (commandOrders != null && commandOrders.size() > 0) {
for (CommandOrder commandOrder : commandOrders) {
ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index b767059..040f12b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -882,7 +882,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Configuration providerConf =
providerService.loadProviderConfigurationInformation(confDir);
- providerService.initializeApplicationConfiguration(instanceDefinition, fs);
+ providerService.initializeApplicationConfiguration(instanceDefinition,
+ fs, null);
providerService.validateApplicationConfiguration(instanceDefinition,
confDir,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
index f1e105a..20ef068 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentResource.java
@@ -73,7 +73,7 @@ public class AgentResource extends AbstractSliderResource {
}
@POST
- @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/register")
+ @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/register")
@Consumes({MediaType.APPLICATION_JSON})
@Produces({MediaType.APPLICATION_JSON})
public RegistrationResponse register(Register registration,
@@ -87,7 +87,7 @@ public class AgentResource extends AbstractSliderResource {
}
@POST
- @Path("/{agent_name: [a-zA-Z][a-zA-Z_0-9]*}/heartbeat")
+ @Path("/{agent_name: [a-zA-Z][a-zA-Z0-9_-]*}/heartbeat")
@Consumes(MediaType.APPLICATION_JSON)
@Produces({MediaType.APPLICATION_JSON})
public HeartBeatResponse heartbeat(HeartBeat message,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
new file mode 100644
index 0000000..853310d
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
@@ -0,0 +1,12 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "test_sleep": {
+ "site.global.component_type": "external"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json
new file mode 100644
index 0000000..bccea68
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component.json
@@ -0,0 +1,22 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "384"
+ },
+ "SLEEP_100": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "0",
+ "yarn.memory": "128"
+ },
+ "SLEEP_LONG": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": "1",
+ "yarn.memory": "128"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
index c4c755d..a35bcc0 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestReplaceTokens.groovy
@@ -51,8 +51,9 @@ class TestReplaceTokens extends Assert {
new JsonSerDeser<ConfTree>(ConfTree)
def confTree = confTreeJsonSerDeser.fromResource(PACKAGE + app_configuration)
SliderClient.replaceTokens(confTree, "testUser", "testCluster")
- assert confTree.global.get("site.fs.defaultFS") == "hdfs://testCluster:8020"
- assert confTree.global.get("site.fs.default.name") == "hdfs://testCluster:8020"
+ // cluster name is resolved later now
+ assert confTree.global.get("site.fs.defaultFS") == "hdfs://\${CLUSTER_NAME}:8020"
+ assert confTree.global.get("site.fs.default.name") == "hdfs://\${CLUSTER_NAME}:8020"
assert confTree.global.get("site.hbase.user_name") == "testUser"
assert confTree.global.get("site.hbase.another.user") == "testUser"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 659af0e..1ea2277 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -175,7 +175,8 @@ class MockProviderService implements ProviderService {
@Override
void initializeApplicationConfiguration(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException {
+ SliderFileSystem fileSystem, String roleGroup)
+ throws IOException, SliderException {
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 9fbb3d0..07d21d7 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -471,7 +471,7 @@ public class TestAgentProviderService {
metainfo.setApplication(application);
doReturn(metainfo).when(mockAps).getApplicationMetainfo(
any(SliderFileSystem.class), anyString());
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
@@ -664,7 +664,7 @@ public class TestAgentProviderService {
hbaseMaster.setName("HBASE_MASTER");
application.addComponent(hbaseMaster);
metainfo.setApplication(application);
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
doReturn(metainfo).when(mockAps).getApplicationMetainfo(
any(SliderFileSystem.class), anyString(), anyBoolean());
@@ -679,7 +679,7 @@ public class TestAgentProviderService {
reg.setActualState(State.STARTED);
mockAps.initializeApplicationConfiguration(instanceDefinition,
- null);
+ null, null);
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
@@ -735,7 +735,7 @@ public class TestAgentProviderService {
metainfo.setApplication(application);
doReturn(metainfo).when(mockAps).getApplicationMetainfo(
any(SliderFileSystem.class), anyString(), anyBoolean());
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
doNothing().when(mockAps).addRoleRelatedTokens(anyMap());
Register reg = new Register();
@@ -748,7 +748,7 @@ public class TestAgentProviderService {
// Simulating agent in INSTALLED state
reg.setActualState(State.INSTALLED);
- mockAps.initializeApplicationConfiguration(instanceDefinition, null);
+ mockAps.initializeApplicationConfiguration(instanceDefinition, null, "HBASE_MASTER");
RegistrationResponse resp = mockAps.handleRegistration(reg);
Assert.assertEquals(0, resp.getResponseId());
@@ -853,7 +853,7 @@ public class TestAgentProviderService {
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
Map<String, String> ports = new HashMap<String, String>();
ports.put("global.listen_port", "10010");
@@ -892,7 +892,7 @@ public class TestAgentProviderService {
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
doReturn(access).when(mockAps).getAmState();
PublishedExportsSet pubExpSet = new PublishedExportsSet();
@@ -1005,7 +1005,7 @@ public class TestAgentProviderService {
new MockContainerId(1), "cid");
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
doReturn(access).when(mockAps).getAmState();
@@ -1149,7 +1149,7 @@ public class TestAgentProviderService {
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
CommandScript script = mockAps.getScriptPathForMasterPackage("HBASE_MASTER");
Assert.assertEquals(script.getScript(), "scripts/hbase_master.py");
@@ -1178,22 +1178,24 @@ public class TestAgentProviderService {
AgentProviderService aps1 = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps1);
- doReturn(metainfo).when(mockAps).getMetaInfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
AgentProviderService mockAps2 = Mockito.spy(aps1);
- doReturn(metainfo2).when(mockAps2).getMetaInfo();
+ doReturn(metainfo2).when(mockAps2).getMetaInfo(anyString());
Assert.assertTrue(mockAps.isMaster(role_hm));
Assert.assertFalse(mockAps.isMaster(role_hrs));
Assert.assertTrue(mockAps.canPublishConfig(role_hm));
Assert.assertFalse(mockAps.canPublishConfig(role_hrs));
- Assert.assertTrue(mockAps.canAnyMasterPublishConfig());
+ Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hm));
+ Assert.assertTrue(mockAps.canAnyMasterPublishConfig(role_hrs));
Assert.assertTrue(mockAps2.isMaster(role_hm));
Assert.assertFalse(mockAps2.isMaster(role_hrs));
Assert.assertTrue(mockAps2.canPublishConfig(role_hm));
Assert.assertFalse(mockAps2.canPublishConfig(role_hrs));
- Assert.assertTrue(mockAps2.canAnyMasterPublishConfig());
+ Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hm));
+ Assert.assertTrue(mockAps2.canAnyMasterPublishConfig(role_hrs));
}
@Test
@@ -1628,8 +1630,8 @@ public class TestAgentProviderService {
expect(access.isApplicationLive()).andReturn(true).anyTimes();
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
- doReturn(metainfo).when(mockAps).getMetaInfo();
- doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs();
+ doReturn(metainfo).when(mockAps).getMetaInfo(anyString());
+ doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs(anyString());
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
Map<String, ClusterNode> container = new HashMap<String, ClusterNode>();
@@ -1695,7 +1697,7 @@ public class TestAgentProviderService {
expect(access.isApplicationLive()).andReturn(true).anyTimes();
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
- doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs();
+ doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString());
List<String> configurations = new ArrayList<String>();
configurations.add("hbase-site");
configurations.add("global");
@@ -1783,7 +1785,7 @@ public class TestAgentProviderService {
expect(access.isApplicationLive()).andReturn(true).anyTimes();
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
- doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs();
+ doReturn(defaultConfigMap).when(mockAps).getDefaultConfigs(anyString());
List<String> configurations = new ArrayList<String>();
configurations.add("hbase-site");
configurations.add("global");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
index 5de2b8e..1cb6f0f 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@ -38,4 +38,6 @@ interface ResourcePaths {
String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json"
String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json"
+ String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json"
+ String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/000b38d5/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
new file mode 100644
index 0000000..b5e0270
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.ResourcePaths
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class ExternalComponentIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+ static String NAME = "test-external-component"
+ static String EXT_NAME = "test_sleep"
+
+ static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG
+ static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
+ static String BUILD_METAINFO = ResourcePaths.SLEEP_META
+ static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG
+ static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
+ static String TEST_METAINFO = ResourcePaths.SLEEP_META
+ public static final String SLEEP_100 = "SLEEP_100"
+ public static final String SLEEP_LONG = "SLEEP_LONG"
+ public static final String EXT_SLEEP_100 = EXT_NAME +
+ SliderKeys.COMPONENT_SEPARATOR + SLEEP_100
+ public static final String EXT_SLEEP_LONG = EXT_NAME +
+ SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(NAME)
+ setupCluster(EXT_NAME)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(NAME)
+ cleanup(EXT_NAME)
+ }
+
+ @Test
+ public void testExternalComponent() throws Throwable {
+ assumeAgentTestsEnabled()
+ describe NAME
+
+ slider(0, [ACTION_BUILD, EXT_NAME, ARG_METAINFO, BUILD_METAINFO,
+ ARG_TEMPLATE, BUILD_APPCONFIG, ARG_RESOURCES, BUILD_RESOURCES,
+ ARG_RES_COMP_OPT, SLEEP_100, ResourceKeys.COMPONENT_INSTANCES, "0",
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "1"
+ ])
+
+ slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO,
+ ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES])
+
+ ensureApplicationIsUp(NAME)
+ status(0, NAME)
+
+ ClusterDescription cd = execStatus(NAME)
+
+ assert 5 == cd.statistics.size()
+ assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM,
+ SLEEP_100, SLEEP_LONG,
+ EXT_SLEEP_100, EXT_SLEEP_LONG])
+
+ expectLiveContainerCountReached(NAME, SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ cleanup(NAME)
+
+ // test overriding the number of instances of external components
+ describe NAME + "-2"
+
+ slider(0, [ACTION_CREATE, NAME, ARG_METAINFO, TEST_METAINFO,
+ ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES,
+ ARG_RES_COMP_OPT, SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "0",
+ ARG_RES_COMP_OPT, EXT_SLEEP_LONG, ResourceKeys.COMPONENT_INSTANCES, "2",
+ ])
+
+ ensureApplicationIsUp(NAME)
+ status(0, NAME)
+
+ cd = execStatus(NAME)
+
+ assert 5 == cd.statistics.size()
+ assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM,
+ SLEEP_100, SLEEP_LONG,
+ EXT_SLEEP_100, EXT_SLEEP_LONG])
+
+ expectLiveContainerCountReached(NAME, SLEEP_LONG, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, EXT_SLEEP_LONG, 2,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ }
+}
[5/6] incubator-slider git commit: SLIDER-875 fix compilation errors
in AgentProviderService
Posted by bi...@apache.org.
SLIDER-875 fix compilation errors in AgentProviderService
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/937b45b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/937b45b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/937b45b0
Branch: refs/heads/develop
Commit: 937b45b00dcc24245c6c08d6e60a4ab16d73f580
Parents: 8863393
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Aug 10 08:52:47 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Aug 10 08:52:47 2016 -0700
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 24 ++++++++++++--------
1 file changed, 14 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/937b45b0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bc362b5..4d68ba2 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -492,7 +492,7 @@ public class AgentProviderService extends AbstractProviderService implements
LocalResourceType.ARCHIVE);
launcher.addLocalResource(AgentKeys.APP_DEFINITION_DIR, appDefRes);
- for (Package pkg : getMetaInfo().getApplication().getPackages()) {
+ for (Package pkg : getMetaInfo(roleGroup).getApplication().getPackages()) {
Path pkgPath = fileSystem.buildResourcePath(pkg.getName());
if (!fileSystem.isFile(pkgPath)) {
pkgPath = fileSystem.buildResourcePath(getClusterName(),
@@ -558,7 +558,7 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, Map<String, String>> configurations =
buildCommandConfigurations(instanceDefinition.getAppConfOperations(),
container.getId().toString(), roleName, roleGroup);
- localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(),
+ localizeConfigFiles(launcher, roleName, roleGroup, getMetaInfo(roleGroup),
configurations, launcher.getEnv(), fileSystem);
}
@@ -1417,7 +1417,8 @@ public class AgentProviderService extends AbstractProviderService implements
// identify client component
Component client = null;
- for (Component component : getMetaInfo().getApplication().getComponents()) {
+ for (Component component : getMetaInfo(null).getApplication()
+ .getComponents()) {
if (component.getCategory().equals("CLIENT")) {
client = component;
break;
@@ -1447,22 +1448,22 @@ public class AgentProviderService extends AbstractProviderService implements
throw new IOException(e);
}
- for (ConfigFile configFile : getMetaInfo()
+ for (ConfigFile configFile : getMetaInfo(null)
.getComponentConfigFiles(client.getName())) {
addNamedConfiguration(configFile.getDictionaryName(),
appConf.getGlobalOptions().options, configurations, tokens, null,
- client.getName());
+ client.getName(), client.getName());
if (appConf.getComponent(client.getName()) != null) {
addNamedConfiguration(configFile.getDictionaryName(),
appConf.getComponent(client.getName()).options, configurations,
- tokens, null, client.getName());
+ tokens, null, client.getName(), client.getName());
}
}
//do a final replacement of re-used configs
dereferenceAllConfigs(configurations);
- for (ConfigFile configFile : getMetaInfo()
+ for (ConfigFile configFile : getMetaInfo(null)
.getComponentConfigFiles(client.getName())) {
ConfigFormat configFormat = ConfigFormat.resolve(configFile.getType());
@@ -1581,9 +1582,12 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected Metainfo getMetaInfo(String roleGroup) {
- ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
- DEFAULT_METAINFO_MAP_KEY);
+ String mapKey = DEFAULT_METAINFO_MAP_KEY;
+ if (roleGroup != null) {
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
+ DEFAULT_METAINFO_MAP_KEY);
+ }
MetainfoHolder mh = this.metaInfoMap.get(mapKey);
if (mh == null) {
return null;
[3/6] incubator-slider git commit: SLIDER-875 address gour's comments
and fix accumulo funtest failures
Posted by bi...@apache.org.
SLIDER-875 address gour's comments and fix accumulo funtest failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2893d34b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2893d34b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2893d34b
Branch: refs/heads/develop
Commit: 2893d34bf09fd605971c80af1445a76c4a4d0bbd
Parents: a4dc574
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue Aug 9 19:54:50 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue Aug 9 19:54:50 2016 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 ++
.../org/apache/slider/client/SliderClient.java | 9 ++++--
.../org/apache/slider/common/SliderKeys.java | 10 +++++--
.../apache/slider/common/tools/SliderUtils.java | 6 ++--
.../slider/core/build/InstanceBuilder.java | 29 ++++++++++----------
.../providers/agent/AgentClientProvider.java | 7 ++---
.../slider/providers/agent/AgentKeys.java | 2 +-
.../sleep_cmd/appConfig_external_component.json | 2 +-
.../appConfig_external_component_nested.json | 2 +-
.../TestBuildExternalComponent.groovy | 6 ++--
10 files changed, 42 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 56014d7..81d290a 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -38,6 +38,8 @@ import org.apache.slider.funtest.framework.SliderShell
import org.junit.Before
import org.junit.Test
+import java.util.regex.Pattern
+
import static org.apache.hadoop.registry.client.binding.RegistryUtils.currentUser
import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePath
@@ -90,6 +92,7 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
def keys = tree.credentials.get(jks)
assert keys!=null, "jks specified in $PROVIDER_PROPERTY wasn't requested " +
"in credentials"
+ jks = jks.replaceAll(Pattern.quote('${CLUSTER_NAME}'), clusterName)
Path jksPath = ProviderUtils.unnestUri(new URI(jks))
if (clusterFS.exists(jksPath)) {
clusterFS.delete(jksPath, false)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 372ec48..78e9b89 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -721,7 +721,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
AggregateConf instanceDefinition = loadInstanceDefinitionUnresolved(
clustername, clusterDirectory);
try {
- checkForCredentials(getConfig(), instanceDefinition.getAppConf());
+ checkForCredentials(getConfig(), instanceDefinition.getAppConf(),
+ clustername);
} catch (IOException e) {
sliderFileSystem.getFileSystem().delete(clusterDirectory, true);
throw e;
@@ -903,7 +904,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
protected static void checkForCredentials(Configuration conf,
- ConfTree tree) throws IOException {
+ ConfTree tree, String clusterName) throws IOException {
if (tree.credentials == null || tree.credentials.isEmpty()) {
log.info("No credentials requested");
return;
@@ -912,7 +913,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
BufferedReader br = null;
try {
for (Entry<String, List<String>> cred : tree.credentials.entrySet()) {
- String provider = cred.getKey();
+ String provider = cred.getKey()
+ .replaceAll(Pattern.quote("${CLUSTER_NAME}"), clusterName)
+ .replaceAll(Pattern.quote("${CLUSTER}"), clusterName);
List<String> aliases = cred.getValue();
if (aliases == null || aliases.isEmpty()) {
continue;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 3921324..2c9be09 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -72,9 +72,15 @@ public interface SliderKeys extends SliderXmlConfKeys {
/**
* Key for component type. This MUST NOT be set in app_config/global {@value}
*/
- String COMPONENT_TYPE = "site.global.component_type";
- String EXTERNAL_COMPONENT = "external";
+ String COMPONENT_TYPE_KEY = "site.global.component_type";
+ /**
+ * A component type for an external app that has been predefined using the
+ * slider build command
+ */
+ String COMPONENT_TYPE_EXTERNAL_APP = "external_app";
String COMPONENT_SEPARATOR = "-";
+ String[] COMPONENT_KEYS_TO_SKIP = {"zookeeper.", "env.MALLOC_ARENA_MAX",
+ "site.fs.", "site.dfs."};
/**
* Key for application version. This must be set in app_config/global {@value}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b798d4d..aad8b45 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -492,11 +492,11 @@ public final class SliderUtils {
FileSystem srcFS = FileSystem.get(srcFile.toUri(), conf);
//list all paths in the src.
if (!srcFS.exists(srcFile)) {
- throw new FileNotFoundException("Source dir not found " + srcFile);
+ throw new FileNotFoundException("Source file not found " + srcFile);
}
if (!srcFS.isFile(srcFile)) {
throw new FileNotFoundException(
- "Source dir not a file " + srcFile);
+ "Source file not a file " + srcFile);
}
FileSystem destFS = FileSystem.get(destFile.toUri(), conf);
if (destFS.exists(destFile)) {
@@ -962,7 +962,7 @@ public final class SliderUtils {
* @param prefixes prefixes to ignore
* @return 'first' merged with the second
*/
- public static Map<String, String> mergeMapsIgnorePrefixes(
+ public static Map<String, String> mergeMapsIgnoreDuplicateKeysAndPrefixes(
Map<String, String> first, Map<String, String> second,
String... prefixes) {
Preconditions.checkArgument(first != null, "Null 'first' value");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 6a7975e..8155214 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -72,8 +72,8 @@ import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
-import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE;
-import static org.apache.slider.common.SliderKeys.EXTERNAL_COMPONENT;
+import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_EXTERNAL_APP;
+import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE_KEY;
import static org.apache.slider.common.tools.SliderUtils.isClusternameValid;
/**
@@ -260,10 +260,11 @@ public class InstanceBuilder {
}
- private void getExternalComponents(ConfTreeOperations ops,
- Set<String> externalComponents) throws BadConfigException {
- if (ops.getGlobalOptions().get(COMPONENT_TYPE) != null) {
- throw new BadConfigException(COMPONENT_TYPE + " must be " +
+ private Set<String> getExternalComponents(ConfTreeOperations ops)
+ throws BadConfigException {
+ Set<String> externalComponents = new HashSet<>();
+ if (ops.getGlobalOptions().containsKey(COMPONENT_TYPE_KEY)) {
+ throw new BadConfigException(COMPONENT_TYPE_KEY + " must be " +
"specified per-component, not in global");
}
@@ -273,15 +274,13 @@ public class InstanceBuilder {
continue;
}
Map<String, String> options = entry.getValue();
- if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
+ if (COMPONENT_TYPE_EXTERNAL_APP.equals(options.get(COMPONENT_TYPE_KEY))) {
externalComponents.add(entry.getKey());
}
}
+ return externalComponents;
}
- private static String[] PREFIXES_TO_SKIP = {"zookeeper.",
- "env.MALLOC_ARENA_MAX", "site.fs.", "site.dfs."};
-
private void mergeExternalComponent(ConfTreeOperations ops,
ConfTreeOperations externalOps, String externalComponent,
Integer priority) throws BadConfigException {
@@ -297,11 +296,12 @@ public class InstanceBuilder {
if (priority == null) {
SliderUtils.mergeMaps(subComponentOps,
ops.getComponent(externalComponent).options);
- subComponentOps.remove(COMPONENT_TYPE);
+ subComponentOps.remove(COMPONENT_TYPE_KEY);
}
- SliderUtils.mergeMapsIgnorePrefixes(subComponentOps,
- externalOps.getComponent(subComponent), PREFIXES_TO_SKIP);
+ SliderUtils.mergeMapsIgnoreDuplicateKeysAndPrefixes(subComponentOps,
+ externalOps.getComponent(subComponent),
+ SliderKeys.COMPONENT_KEYS_TO_SKIP);
// add prefix to existing prefix
String existingPrefix = subComponentOps.get(ROLE_PREFIX);
@@ -348,8 +348,7 @@ public class InstanceBuilder {
}
}
- Set<String> externalComponents = new HashSet<>();
- getExternalComponents(appConf, externalComponents);
+ Set<String> externalComponents = getExternalComponents(appConf);
if (!externalComponents.isEmpty()) {
log.info("Found external components {}", externalComponents);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 8c0a2e4..1567fc9 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -74,8 +74,6 @@ import java.util.Set;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
-import static org.apache.slider.providers.agent.AgentUtils.getMetainfoComponentName;
-
/** This class implements the client-side aspects of the agent deployer */
public class AgentClientProvider extends AbstractClientProvider
implements AgentKeys, SliderKeys {
@@ -184,7 +182,7 @@ public class AgentClientProvider extends AbstractClientProvider
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name,
+ AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
if (componentDef == null) {
throw new BadConfigException(
@@ -215,9 +213,8 @@ public class AgentClientProvider extends AbstractClientProvider
// fileSystem may be null for tests
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name,
+ AgentUtils.getMetainfoComponentName(name,
instanceDefinition.getAppConfOperations()));
- // already checked it wasn't null
// ensure that intance count is 0 for client components
if ("CLIENT".equals(componentDef.getCategory())) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 565d73d..8341af4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -99,7 +99,7 @@ public interface AgentKeys {
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
- String DEFAULT_METAINFO_MAP_KEY = "DEFAULT";
+ String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
index 853310d..b026b9d 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component.json
@@ -6,7 +6,7 @@
},
"components": {
"test_sleep": {
- "site.global.component_type": "external"
+ "site.global.component_type": "external_app"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
index f67f83a..a9972cf 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
@@ -6,7 +6,7 @@
},
"components": {
"test-external-component": {
- "site.global.component_type": "external"
+ "site.global.component_type": "external_app"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2893d34b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
index 8dd693e..13f5f3b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
@@ -67,7 +67,7 @@ class TestBuildExternalComponent extends AgentMiniClusterTestBase {
SliderActions.ACTION_BUILD,
parent1,
[(echo): 1],
- [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"],
true,
false,
@@ -97,8 +97,8 @@ class TestBuildExternalComponent extends AgentMiniClusterTestBase {
SliderActions.ACTION_BUILD,
parent2,
[(echo): 1],
- [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
- ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
+ ARG_COMP_OPT, parent1, COMPONENT_TYPE_KEY, COMPONENT_TYPE_EXTERNAL_APP,
ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"],
true,
false,
[2/6] incubator-slider git commit: SLIDER-875 better handling and
support of nested external components through addition of role.prefix
Posted by bi...@apache.org.
SLIDER-875 better handling and support of nested external components through addition of role.prefix
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a4dc574c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a4dc574c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a4dc574c
Branch: refs/heads/develop
Commit: a4dc574c9f60e2f07906b37f8200e9488269c20d
Parents: 000b38d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jul 13 09:53:47 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jul 13 09:53:47 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/api/RoleKeys.java | 5 +
.../apache/slider/common/tools/SliderUtils.java | 9 ++
.../slider/core/build/InstanceBuilder.java | 108 +++++++++++----
.../slider/core/conf/ConfTreeOperations.java | 39 ++++++
.../providers/agent/AgentClientProvider.java | 6 +-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 42 +++---
.../slider/providers/agent/AgentUtils.java | 30 ++--
.../providers/agent/ComponentCommandOrder.java | 110 ++++++++++-----
.../slider/server/appmaster/state/AppState.java | 5 +
.../appConfig_external_component_nested.json | 12 ++
.../test_min_pkg/sleep_cmd/metainfo.json | 6 +
.../metainfo_external_component_nested.json | 14 ++
.../resources_external_component_nested.json | 12 ++
.../TestBuildExternalComponent.groovy | 138 +++++++++++++++++++
.../agent/TestAgentProviderService.java | 35 ++---
.../agent/TestComponentCommandOrder.java | 107 +++++++++++++-
.../apache/slider/funtest/ResourcePaths.groovy | 4 +
.../funtest/misc/ExternalComponentIT.groovy | 38 +++++
19 files changed, 589 insertions(+), 132 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 812a6b3..ce413ff 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -35,6 +35,11 @@ public interface RoleKeys {
String ROLE_GROUP = "role.group";
/**
+ * The prefix of a role: {@value}
+ */
+ String ROLE_PREFIX = "role.prefix";
+
+ /**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index c48e109..b798d4d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -123,6 +123,8 @@ import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
+import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+
/**
* These are slider-specific Util methods
*/
@@ -2612,4 +2614,11 @@ public final class SliderUtils {
}
return buffer.toString();
}
+
+ public static String trimPrefix(String prefix) {
+ if (prefix != null && prefix.endsWith(COMPONENT_SEPARATOR)) {
+ return prefix.substring(0, prefix.length()-1);
+ }
+ return prefix;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 4afdf7c..6a7975e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -69,6 +69,7 @@ import static org.apache.slider.api.OptionKeys.INTERNAL_SNAPSHOT_CONF_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_HOSTS;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_PATH;
import static org.apache.slider.api.OptionKeys.ZOOKEEPER_QUORUM;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.common.SliderKeys.COMPONENT_AM;
import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
import static org.apache.slider.common.SliderKeys.COMPONENT_TYPE;
@@ -85,7 +86,7 @@ public class InstanceBuilder {
private final CoreFileSystem coreFS;
private final InstancePaths instancePaths;
private AggregateConf instanceDescription;
- private Map<Path, Path> externalAppDefs = new HashMap<>();
+ private Map<String, Path> externalAppDefs = new HashMap<>();
private TreeSet<Integer> priorities = new TreeSet<>();
private static final Logger log =
@@ -272,8 +273,7 @@ public class InstanceBuilder {
continue;
}
Map<String, String> options = entry.getValue();
- if (options.containsKey(COMPONENT_TYPE) &&
- EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
+ if (EXTERNAL_COMPONENT.equals(options.get(COMPONENT_TYPE))) {
externalComponents.add(entry.getKey());
}
}
@@ -284,17 +284,37 @@ public class InstanceBuilder {
private void mergeExternalComponent(ConfTreeOperations ops,
ConfTreeOperations externalOps, String externalComponent,
- Integer priority) {
+ Integer priority) throws BadConfigException {
for (String subComponent : externalOps.getComponentNames()) {
if (COMPONENT_AM.equals(subComponent)) {
continue;
}
+ String prefix = externalComponent + COMPONENT_SEPARATOR;
log.debug("Merging options for {} into {}", subComponent,
- externalComponent + COMPONENT_SEPARATOR + subComponent);
- MapOperations subComponentOps = ops.getOrAddComponent(externalComponent +
- COMPONENT_SEPARATOR + subComponent);
+ prefix + subComponent);
+ MapOperations subComponentOps = ops.getOrAddComponent(
+ prefix + subComponent);
+ if (priority == null) {
+ SliderUtils.mergeMaps(subComponentOps,
+ ops.getComponent(externalComponent).options);
+ subComponentOps.remove(COMPONENT_TYPE);
+ }
+
SliderUtils.mergeMapsIgnorePrefixes(subComponentOps,
externalOps.getComponent(subComponent), PREFIXES_TO_SKIP);
+
+ // add prefix to existing prefix
+ String existingPrefix = subComponentOps.get(ROLE_PREFIX);
+ if (existingPrefix != null) {
+ if (!subComponent.startsWith(existingPrefix)) {
+ throw new BadConfigException("Bad prefix " + existingPrefix +
+ " for subcomponent " + subComponent + " of " + externalComponent);
+ }
+ prefix = prefix + existingPrefix;
+ }
+ subComponentOps.set(ROLE_PREFIX, prefix);
+
+ // adjust priority
if (priority != null) {
subComponentOps.put(ResourceKeys.COMPONENT_PRIORITY,
Integer.toString(priority));
@@ -322,10 +342,6 @@ public class InstanceBuilder {
if (COMPONENT_AM.equals(entry.getKey())) {
continue;
}
- if (entry.getKey().contains(COMPONENT_SEPARATOR)) {
- throw new BadConfigException("Components must not contain " +
- COMPONENT_SEPARATOR + ": " + entry.getKey());
- }
if (entry.getValue().containsKey(ResourceKeys.COMPONENT_PRIORITY)) {
priorities.add(Integer.parseInt(entry.getValue().get(
ResourceKeys.COMPONENT_PRIORITY)));
@@ -358,31 +374,73 @@ public class InstanceBuilder {
throw new BadConfigException("Couldn't read configuration for " +
"external component " + component);
}
- String externalAppDef = componentConf.getAppConfOperations()
- .getGlobalOptions().get(AgentKeys.APP_DEF);
+
+ ConfTreeOperations componentAppConf = componentConf.getAppConfOperations();
+ String externalAppDef = componentAppConf.get(AgentKeys.APP_DEF);
if (SliderUtils.isSet(externalAppDef)) {
Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
component + "_" + SliderKeys.DEFAULT_APP_PKG);
- componentConf.getAppConfOperations().set(AgentKeys.APP_DEF, newAppDef);
- externalAppDefs.put(newAppDef, new Path(externalAppDef));
+ componentAppConf.set(AgentKeys.APP_DEF, newAppDef);
+ componentAppConf.append(AgentKeys.APP_DEF_ORIGINAL, externalAppDef);
+ log.info("Copying external appdef {} to {} for {}", externalAppDef,
+ newAppDef, component);
+ externalAppDefs.put(externalAppDef, newAppDef);
+ externalAppDef = newAppDef.toString();
}
+
for (String rcomp : componentConf.getResourceOperations()
.getComponentNames()) {
if (COMPONENT_AM.equals(rcomp)) {
continue;
}
log.debug("Adding component {} to appConf for {}", rcomp, component);
- componentConf.getAppConfOperations().getOrAddComponent(rcomp);
+ componentAppConf.getOrAddComponent(rcomp);
}
- SliderUtils.mergeMaps(
- componentConf.getAppConfOperations().getGlobalOptions().options,
- appConf.getComponent(component).options);
- componentConf.getAppConfOperations().getGlobalOptions()
- .remove(COMPONENT_TYPE);
componentConf.resolve();
- mergeExternalComponent(appConf, componentConf.getAppConfOperations(),
- component, null);
+ for (String rcomp : componentConf.getResourceOperations()
+ .getComponentNames()) {
+ if (COMPONENT_AM.equals(rcomp)) {
+ continue;
+ }
+ String componentAppDef = componentAppConf.getComponentOpt(
+ rcomp, AgentKeys.APP_DEF, null);
+ if (SliderUtils.isUnset(componentAppDef) ||
+ componentAppDef.equals(externalAppDef)) {
+ continue;
+ }
+ if (externalAppDefs.containsKey(componentAppDef)) {
+ log.info("Using external appdef {} for {}",
+ externalAppDefs.get(componentAppDef), rcomp);
+ } else {
+ String existingPrefix = componentAppConf.getComponentOpt(rcomp,
+ ROLE_PREFIX, null);
+ if (SliderUtils.isUnset(existingPrefix)) {
+ existingPrefix = "";
+ } else {
+ existingPrefix = COMPONENT_SEPARATOR + SliderUtils.trimPrefix(
+ existingPrefix);
+ }
+ Path newAppDef = new Path(coreFS.buildAppDefDirPath(clustername),
+ component + existingPrefix + "_" + SliderKeys.DEFAULT_APP_PKG);
+ externalAppDefs.put(componentAppDef, newAppDef);
+ log.info("Copying external appdef {} to {} for {}", componentAppDef,
+ newAppDef, component + COMPONENT_SEPARATOR + rcomp);
+ }
+ componentAppConf.setComponentOpt(rcomp, AgentKeys.APP_DEF,
+ externalAppDefs.get(componentAppDef).toString());
+ componentAppConf.appendComponentOpt(rcomp,
+ AgentKeys.APP_DEF_ORIGINAL, componentAppDef);
+ }
+ Set<Path> newAppDefs = new HashSet<>();
+ newAppDefs.addAll(externalAppDefs.values());
+ if (newAppDefs.size() != externalAppDefs.size()) {
+ throw new IllegalStateException("Values repeat in external appdefs "
+ + externalAppDefs);
+ }
+ log.info("External appdefs after {}: {}", component, externalAppDefs);
+
+ mergeExternalComponent(appConf, componentAppConf, component, null);
mergeExternalComponent(resources, componentConf.getResourceOperations(),
component, getNextPriority());
}
@@ -411,8 +469,8 @@ public class InstanceBuilder {
action = new ConfDirSnapshotAction(appconfdir);
}
persister.save(instanceDescription, action);
- for (Entry<Path, Path> appDef : externalAppDefs.entrySet()) {
- SliderUtils.copy(conf, appDef.getValue(), appDef.getKey());
+ for (Entry<String, Path> appDef : externalAppDefs.entrySet()) {
+ SliderUtils.copy(conf, new Path(appDef.getKey()), appDef.getValue());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 4a0ae41..038513e 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -214,6 +214,23 @@ public class ConfTreeOperations {
public String get(String key) {
return globalOptions.get(key);
}
+ /**
+ * append to a global option
+ * @param key key
+ * @return value
+ *
+ */
+ public String append(String key, String value) {
+ if (SliderUtils.isUnset(value)) {
+ return null;
+ }
+ if (globalOptions.containsKey(key)) {
+ globalOptions.put(key, globalOptions.get(key) + "," + value);
+ } else {
+ globalOptions.put(key, value);
+ }
+ return globalOptions.get(key);
+ }
/**
* Propagate all global keys matching a prefix
@@ -471,4 +488,26 @@ public class ConfTreeOperations {
setComponentOpt(role, option, Long.toString(val));
}
+ /**
+ * append to a component option
+ * @param key key
+ * @return value
+ *
+ */
+ public String appendComponentOpt(String role, String key, String value) {
+ if (SliderUtils.isUnset(value)) {
+ return null;
+ }
+ MapOperations roleopts = getComponent(role);
+ if (roleopts == null) {
+ return null;
+ }
+
+ if (roleopts.containsKey(key)) {
+ roleopts.put(key, roleopts.get(key) + "," + value);
+ } else {
+ roleopts.put(key, value);
+ }
+ return roleopts.get(key);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 6eae75e..8c0a2e4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -184,7 +184,8 @@ public class AgentClientProvider extends AbstractClientProvider
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name));
+ getMetainfoComponentName(name,
+ instanceDefinition.getAppConfOperations()));
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
@@ -214,7 +215,8 @@ public class AgentClientProvider extends AbstractClientProvider
// fileSystem may be null for tests
if (metaInfo != null) {
Component componentDef = metaInfo.getApplicationComponent(
- getMetainfoComponentName(name));
+ getMetainfoComponentName(name,
+ instanceDefinition.getAppConfOperations()));
// already checked it wasn't null
// ensure that intance count is 0 for client components
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 8514401..565d73d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -68,6 +68,7 @@ public interface AgentKeys {
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
+ String APP_DEF_ORIGINAL = "application.def.original";
String ADDON_PREFIX = "application.addon.";
String ADDONS = "application.addons";
String AGENT_VERSION = "agent.version";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 66fac99..452122f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -133,7 +133,7 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import static org.apache.slider.providers.agent.AgentUtils.getMetainfoMapKey;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
/**
@@ -288,7 +288,8 @@ public class AgentProviderService extends AbstractProviderService implements
SliderFileSystem fileSystem,
String roleGroup)
throws IOException, SliderException {
- String mapKey = getMetainfoMapKey(roleGroup);
+ String mapKey = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
String appDef = SliderUtils.getApplicationDefinitionPath(
instanceDefinition.getAppConfOperations(), roleGroup);
MapOperations component = null;
@@ -319,16 +320,17 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Modifying external metainfo component name to {}",
comp.getName());
}
- String commandPrefix = mapKey.substring(0,
- mapKey.indexOf(COMPONENT_SEPARATOR)+1);
for (CommandOrder co : commandOrders) {
log.info("Adding prefix {} to command order {}",
- commandPrefix, co);
- co.setCommand(commandPrefix + co.getCommand());
- co.setRequires(commandPrefix + co.getRequires());
+ mapKey, co);
+ co.setCommand(mapKey + co.getCommand());
+ co.setRequires(mapKey + co.getRequires());
}
}
- commandOrder.mergeCommandOrders(commandOrders);
+ log.debug("Merging command orders {} for {}", commandOrders,
+ roleGroup);
+ commandOrder.mergeCommandOrders(commandOrders,
+ instanceDefinition.getResourceOperations());
Map<String, DefaultConfig> defaultConfigs =
initializeDefaultConfigs(fileSystem, appDef, metaInfo);
metaInfoMap.put(mapKey, new MetainfoHolder(metaInfo, defaultConfigs));
@@ -1393,7 +1395,10 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected Metainfo getMetaInfo(String roleGroup) {
- MetainfoHolder mh = this.metaInfoMap.get(getMetainfoMapKey(roleGroup));
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
+ DEFAULT_METAINFO_MAP_KEY);
+ MetainfoHolder mh = this.metaInfoMap.get(mapKey);
if (mh == null) {
return null;
}
@@ -1468,7 +1473,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
protected Map<String, DefaultConfig> getDefaultConfigs(String roleGroup) {
- return metaInfoMap.get(getMetainfoMapKey(roleGroup)).defaultConfigs;
+ ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
+ String mapKey = appConf.getComponentOpt(roleGroup, ROLE_PREFIX,
+ DEFAULT_METAINFO_MAP_KEY);
+ return metaInfoMap.get(mapKey).defaultConfigs;
}
private int getHeartbeatMonitorInterval() {
@@ -2900,14 +2908,14 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
tokens.put("${DEFAULT_ZK_PATH}", appConf.get(OptionKeys.ZOOKEEPER_PATH));
- String mapKey = getMetainfoMapKey(componentGroup);
+ String prefix = appConf.getComponentOpt(componentGroup, ROLE_PREFIX,
+ null);
String dataDirSuffix = "";
- if (!DEFAULT_METAINFO_MAP_KEY.equals(mapKey)) {
- dataDirSuffix = "_" + mapKey.substring(0, mapKey.length()-1);
+ if (prefix == null) {
+ prefix = "";
} else {
- mapKey = "";
+ dataDirSuffix = "_" + SliderUtils.trimPrefix(prefix);
}
- mapKey = mapKey.toLowerCase();
tokens.put("${DEFAULT_DATA_DIR}", getAmState()
.getInternalsSnapshot()
.getGlobalOptions()
@@ -2915,8 +2923,8 @@ public class AgentProviderService extends AbstractProviderService implements
tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
tokens.put("${COMPONENT_NAME}", componentName);
tokens.put("${COMPONENT_NAME.lc}", componentName.toLowerCase());
- tokens.put("${COMPONENT_PREFIX}", mapKey);
- tokens.put("${COMPONENT_PREFIX.lc}", mapKey.toLowerCase());
+ tokens.put("${COMPONENT_PREFIX}", prefix);
+ tokens.put("${COMPONENT_PREFIX.lc}", prefix.toLowerCase());
if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) {
tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length()));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index ed5108c..23e05a3 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.application.metadata.AbstractMetainfoParser;
import org.apache.slider.providers.agent.application.metadata.AddonPackageMetainfoParser;
@@ -35,8 +36,7 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
-import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
-import static org.apache.slider.providers.agent.AgentKeys.DEFAULT_METAINFO_MAP_KEY;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
/**
*
@@ -135,24 +135,16 @@ public class AgentUtils {
return new DefaultConfigParser().parse(configStream);
}
- static String getMetainfoMapKey(String roleGroup) {
- if (roleGroup == null) {
- return DEFAULT_METAINFO_MAP_KEY;
- }
- int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
- if (lastIndex == -1) {
- return DEFAULT_METAINFO_MAP_KEY;
- } else {
- return roleGroup.substring(0, lastIndex+1);
- }
- }
-
- static String getMetainfoComponentName(String roleGroup) {
- int lastIndex = roleGroup.lastIndexOf(COMPONENT_SEPARATOR);
- if (lastIndex == -1) {
+ static String getMetainfoComponentName(String roleGroup,
+ ConfTreeOperations appConf) throws BadConfigException {
+ String prefix = appConf.getComponentOpt(roleGroup, ROLE_PREFIX, null);
+ if (prefix == null) {
return roleGroup;
- } else {
- return roleGroup.substring(lastIndex+1);
}
+ if (!roleGroup.startsWith(prefix)) {
+ throw new BadConfigException("Component " + roleGroup + " doesn't start" +
+ " with prefix " + prefix);
+ }
+ return roleGroup.substring(prefix.length());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index e2c879e..4abac7a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -18,6 +18,8 @@
package org.apache.slider.providers.agent;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,9 +27,12 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
+
/**
* Stores the command dependency order for all components in a service. <commandOrder>
* <command>SUPERVISOR-START</command> <requires>NIMBUS-STARTED</requires> </commandOrder> Means, SUPERVISOR START
@@ -39,19 +44,36 @@ public class ComponentCommandOrder {
private static char SPLIT_CHAR = '-';
Map<Command, Map<String, List<ComponentState>>> dependencies =
new HashMap<Command, Map<String, List<ComponentState>>>();
+ Map<String, Collection<String>> prefixRoleMap = new HashMap<>();
+ Map<String, String> rolePrefixMap = new HashMap<>();
public ComponentCommandOrder() {}
- public ComponentCommandOrder(List<CommandOrder> commandOrders) {
- mergeCommandOrders(commandOrders);
+ public ComponentCommandOrder(List<CommandOrder> commandOrders,
+ ConfTreeOperations resources) {
+ mergeCommandOrders(commandOrders, resources);
}
- void mergeCommandOrders(List<CommandOrder> commandOrders) {
+ void mergeCommandOrders(List<CommandOrder> commandOrders,
+ ConfTreeOperations resources) {
+ for (String component : resources.getComponentNames()) {
+ String prefix = SliderUtils.trimPrefix(
+ resources.getComponentOpt(component, ROLE_PREFIX, null));
+ if (prefix != null) {
+ rolePrefixMap.put(component, prefix);
+ if (!prefixRoleMap.containsKey(prefix)) {
+ prefixRoleMap.put(prefix, new HashSet<String>());
+ }
+ prefixRoleMap.get(prefix).add(component);
+ }
+ }
if (commandOrders != null && commandOrders.size() > 0) {
for (CommandOrder commandOrder : commandOrders) {
- ComponentCommand componentCmd = getComponentCommand(commandOrder.getCommand());
+ ComponentCommand componentCmd = getComponentCommand(
+ commandOrder.getCommand(), resources);
String requires = commandOrder.getRequires();
- List<ComponentState> requiredStates = parseRequiredStates(requires);
+ List<ComponentState> requiredStates = parseRequiredStates(requires,
+ resources);
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
@@ -71,7 +93,8 @@ public class ComponentCommandOrder {
}
}
- private List<ComponentState> parseRequiredStates(String requires) {
+ private List<ComponentState> parseRequiredStates(String requires,
+ ConfTreeOperations resources) {
if (requires == null || requires.length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and state.");
}
@@ -79,13 +102,14 @@ public class ComponentCommandOrder {
String[] componentStates = requires.split(",");
List<ComponentState> retList = new ArrayList<ComponentState>();
for (String componentStateStr : componentStates) {
- retList.add(getComponentState(componentStateStr));
+ retList.add(getComponentState(componentStateStr, resources));
}
return retList;
}
- private ComponentCommand getComponentCommand(String compCmdStr) {
+ private ComponentCommand getComponentCommand(String compCmdStr,
+ ConfTreeOperations resources) {
if (compCmdStr == null || compCmdStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null and must contain component and command.");
}
@@ -98,6 +122,11 @@ public class ComponentCommandOrder {
String compStr = compCmdStr.substring(0, splitIndex);
String cmdStr = compCmdStr.substring(splitIndex + 1);
+ if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
+ throw new IllegalArgumentException("Component " + compStr + " specified" +
+ " in command order does not exist");
+ }
+
Command cmd = Command.valueOf(cmdStr);
if (cmd != Command.START) {
@@ -106,7 +135,8 @@ public class ComponentCommandOrder {
return new ComponentCommand(compStr, cmd);
}
- private ComponentState getComponentState(String compStStr) {
+ private ComponentState getComponentState(String compStStr,
+ ConfTreeOperations resources) {
if (compStStr == null || compStStr.trim().length() < 2) {
throw new IllegalArgumentException("Input cannot be null.");
}
@@ -119,6 +149,11 @@ public class ComponentCommandOrder {
String compStr = compStStr.substring(0, splitIndex);
String stateStr = compStStr.substring(splitIndex + 1);
+ if (resources.getComponent(compStr) == null && !prefixRoleMap.containsKey(compStr)) {
+ throw new IllegalArgumentException("Component " + compStr + " specified" +
+ " in command order does not exist");
+ }
+
State state = State.valueOf(stateStr);
if (state != State.STARTED && state != State.INSTALLED) {
throw new IllegalArgumentException("Dependency order can only be specified against STARTED/INSTALLED.");
@@ -129,40 +164,43 @@ public class ComponentCommandOrder {
// dependency is still on component level, but not package level
// so use component name to check dependency, not component-package
public boolean canExecute(String component, Command command, Collection<ComponentInstanceState> currentStates) {
- boolean canExecute = true;
- if (dependencies.containsKey(command) && dependencies.get(command).containsKey(component)) {
- List<ComponentState> required = dependencies.get(command).get(component);
- for (ComponentState stateToMatch : required) {
- for (ComponentInstanceState currState : currentStates) {
- log.debug("Checking schedule {} {} against dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- if (currState.getComponentName().equals(stateToMatch.componentName)) {
- if (currState.getState() != stateToMatch.state) {
- if (stateToMatch.state == State.STARTED) {
+ if (!dependencies.containsKey(command)) {
+ return true;
+ }
+ List<ComponentState> required = new ArrayList<>();
+ if (dependencies.get(command).containsKey(component)) {
+ required.addAll(dependencies.get(command).get(component));
+ }
+ String prefix = rolePrefixMap.get(component);
+ if (prefix != null && dependencies.get(command).containsKey(prefix)) {
+ required.addAll(dependencies.get(command).get(prefix));
+ }
+
+ for (ComponentState stateToMatch : required) {
+ for (ComponentInstanceState currState : currentStates) {
+ log.debug("Checking schedule {} {} against dependency {} is {}",
+ component, command, currState.getComponentName(), currState.getState());
+ if (currState.getComponentName().equals(stateToMatch.componentName) ||
+ (prefixRoleMap.containsKey(stateToMatch.componentName) &&
+ prefixRoleMap.get(stateToMatch.componentName).contains(currState.getComponentName()))) {
+ if (currState.getState() != stateToMatch.state) {
+ if (stateToMatch.state == State.STARTED) {
+ log.info("Cannot schedule {} {} as dependency {} is {}",
+ component, command, currState.getComponentName(), currState.getState());
+ return false;
+ } else {
+ //state is INSTALLED
+ if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- canExecute = false;
- } else {
- //state is INSTALLED
- if (currState.getState() != State.STARTING && currState.getState() != State.STARTED) {
- log.info("Cannot schedule {} {} as dependency {} is {}",
- component, command, currState.getComponentName(), currState.getState());
- canExecute = false;
- }
+ component, command, currState.getComponentName(), currState.getState());
+ return false;
}
}
}
- if (!canExecute) {
- break;
- }
- }
- if (!canExecute) {
- break;
}
}
}
-
- return canExecute;
+ return true;
}
static class ComponentState {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 3213d93..3ba766f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1808,6 +1808,11 @@ public class AppState {
SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename),
groupOptions.options);
}
+ String prefix = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(role.getGroup(), ROLE_PREFIX, null);
+ if (SliderUtils.isSet(prefix)) {
+ cd.setRoleOpt(rolename, ROLE_PREFIX, SliderUtils.trimPrefix(prefix));
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
new file mode 100644
index 0000000..f67f83a
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json
@@ -0,0 +1,12 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ },
+ "components": {
+ "test-external-component": {
+ "site.global.component_type": "external"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
index 073d1ff..bfe2afb 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
@@ -2,6 +2,12 @@
"schemaVersion": "2.1",
"application": {
"name": "SLEEPER",
+ "commandOrders": [
+ {
+ "command": "SLEEP_100-START",
+ "requires": "SLEEP_LONG-STARTED"
+ }
+ ],
"components": [
{
"name": "SLEEP_100",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
new file mode 100644
index 0000000..43a2200
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json
@@ -0,0 +1,14 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "SLEEPER",
+ "commandOrders": [
+ {
+ "command": "test-external-component-test_sleep-SLEEP_100-START",
+ "requires": "test-external-component-SLEEP_100-STARTED"
+ }
+ ],
+ "components": [
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
new file mode 100644
index 0000000..8aa86b4
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources_external_component_nested.json
@@ -0,0 +1,12 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "384"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
new file mode 100644
index 0000000..8dd693e
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildExternalComponent.groovy
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.agent.standalone
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.agent.AgentMiniClusterTestBase
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderFileSystem
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.main.ServiceLauncher
+import org.apache.slider.providers.agent.AgentKeys
+import org.junit.Test
+
+import static org.apache.slider.common.params.Arguments.*
+
+@CompileStatic
+@Slf4j
+
+class TestBuildExternalComponent extends AgentMiniClusterTestBase {
+
+ @Test
+ public void testExternalComponentBuild() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "verify external components"
+
+ String echo = "echo"
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [(echo): 1],
+ [ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "2"],
+ true,
+ false,
+ agentDefOptions)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(clustername, false)
+
+ String parent1 = clustername + "_ext"
+ launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ parent1,
+ [(echo): 1],
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "3"],
+ true,
+ false,
+ agentDefOptions)
+ sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(parent1, false)
+ // verify generated conf
+ def aggregateConf = sliderClient.loadPersistedClusterDescription(parent1)
+ assert 3 == aggregateConf.resourceOperations.componentNames.size()
+ assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM)
+ assert aggregateConf.resourceOperations.componentNames.contains(echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo)
+
+ aggregateConf.resolve()
+ assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals(
+ aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF,
+ aggregateConf.appConfOperations.get(AgentKeys.APP_DEF)))
+ SliderFileSystem sfs = createSliderFileSystem()
+ String appdefdir = sfs.buildAppDefDirPath(parent1)
+ checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+
+ String parent2 = "parent"
+ launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ parent2,
+ [(echo): 1],
+ [ARG_COMP_OPT, clustername, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_COMP_OPT, parent1, COMPONENT_TYPE, EXTERNAL_COMPONENT,
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "4"],
+ true,
+ false,
+ agentDefOptions)
+ sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(parent2, false)
+ // verify generated conf
+ aggregateConf = sliderClient.loadPersistedClusterDescription(parent2)
+ assert 5 == aggregateConf.resourceOperations.componentNames.size()
+ assert aggregateConf.resourceOperations.componentNames.contains(COMPONENT_AM)
+ assert aggregateConf.resourceOperations.componentNames.contains(echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(clustername + COMPONENT_SEPARATOR + echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + echo)
+ assert aggregateConf.resourceOperations.componentNames.contains(parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo)
+
+ aggregateConf.resolve()
+ assert aggregateConf.appConfOperations.get(AgentKeys.APP_DEF).equals(
+ aggregateConf.appConfOperations.getComponentOpt(echo, AgentKeys.APP_DEF,
+ aggregateConf.appConfOperations.get(AgentKeys.APP_DEF)))
+ appdefdir = sfs.buildAppDefDirPath(parent2)
+ checkComponent(aggregateConf, clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+ checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + echo, appdefdir)
+ checkComponent(aggregateConf, parent1 + COMPONENT_SEPARATOR + clustername + COMPONENT_SEPARATOR + echo, appdefdir)
+ }
+
+ private void checkComponent(AggregateConf aggConf, String component,
+ String appdefdir) {
+ String path = new Path(appdefdir, SliderUtils.trimPrefix(
+ aggConf.appConfOperations.getComponentOpt(component, RoleKeys
+ .ROLE_PREFIX, null)) + "_" + SliderKeys.DEFAULT_APP_PKG).toString()
+ assert path.equals(aggConf.appConfOperations.getComponentOpt(component,
+ AgentKeys.APP_DEF, null))
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 07d21d7..a6d52b4 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -152,10 +152,6 @@ public class TestAgentProviderService {
+ " <command>HBASE_REGIONSERVER-START</command>\n"
+ " <requires>HBASE_MASTER-STARTED</requires>\n"
+ " </commandOrder>\n"
- + " <commandOrder>\n"
- + " <command>A-START</command>\n"
- + " <requires>B-STARTED</requires>\n"
- + " </commandOrder>\n"
+ " </commandOrders>\n"
+ " <components>\n"
+ " <component>\n"
@@ -434,6 +430,11 @@ public class TestAgentProviderService {
.put(AgentKeys.AGENT_CONF, ".");
instanceDefinition.getAppConfOperations().getGlobalOptions()
.put(AgentKeys.AGENT_VERSION, ".");
+
+ instanceDefinition.getResourceOperations().getOrAddComponent(
+ "HBASE_MASTER");
+ instanceDefinition.getResourceOperations().getOrAddComponent(
+ "HBASE_REGIONSERVER");
return instanceDefinition;
}
@@ -530,13 +531,7 @@ public class TestAgentProviderService {
assertNotNull(registryViewForProviders);
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
- AggregateConf instanceDefinition = new AggregateConf();
-
- instanceDefinition.setInternal(tree);
- instanceDefinition.setAppConf(tree);
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
Container container = createNiceMock(Container.class);
ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
@@ -1116,19 +1111,15 @@ public class TestAgentProviderService {
Assert.assertEquals(found, 2);
List<CommandOrder> cmdOrders = application.getCommandOrders();
- Assert.assertEquals(cmdOrders.size(), 2);
+ Assert.assertEquals(cmdOrders.size(), 1);
found = 0;
for (CommandOrder co : application.getCommandOrders()) {
if (co.getCommand().equals("HBASE_REGIONSERVER-START")) {
Assert.assertTrue(co.getRequires().equals("HBASE_MASTER-STARTED"));
found++;
}
- if (co.getCommand().equals("A-START")) {
- Assert.assertEquals(co.getRequires(), "B-STARTED");
- found++;
- }
}
- Assert.assertEquals(found, 2);
+ Assert.assertEquals(found, 1);
List<ConfigFile> configFiles = application.getConfigFiles();
Assert.assertEquals(configFiles.size(), 2);
@@ -1204,8 +1195,6 @@ public class TestAgentProviderService {
// Start of HBASE_RS depends on the start of HBASE_MASTER
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
- ConfTree tree = new ConfTree();
- tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
Configuration conf = new Configuration();
AgentProviderService aps = createAgentProviderService(conf);
@@ -1213,13 +1202,7 @@ public class TestAgentProviderService {
assertNotNull(registryViewForProviders);
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
- AggregateConf instanceDefinition = new AggregateConf();
-
- instanceDefinition.setInternal(tree);
- instanceDefinition.setAppConf(tree);
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.APP_DEF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_CONF, ".");
- instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
+ AggregateConf instanceDefinition = prepareConfForAgentStateTests();
Container container = createNiceMock(Container.class);
ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
index c123fbb..0e3e3ad 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestComponentCommandOrder.java
@@ -18,20 +18,38 @@
package org.apache.slider.providers.agent;
+import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.conf.ConfTreeOperations;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.server.appmaster.model.mock.MockContainerId;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Arrays;
+import java.util.List;
+
+import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
public class TestComponentCommandOrder {
protected static final Logger log =
LoggerFactory.getLogger(TestComponentCommandOrder.class);
private final MockContainerId containerId = new MockContainerId(1);
+ private static ConfTreeOperations resources = new ConfTreeOperations(
+ new ConfTree());
+
+ @BeforeClass
+ public static void init() {
+ resources.getOrAddComponent("A");
+ resources.getOrAddComponent("B");
+ resources.getOrAddComponent("C");
+ resources.getOrAddComponent("D");
+ resources.getOrAddComponent("E");
+ }
+
@Test
public void testComponentCommandOrder() throws Exception {
CommandOrder co1 = new CommandOrder();
@@ -44,7 +62,8 @@ public class TestComponentCommandOrder {
co3.setCommand("B-START");
co3.setRequires("C-STARTED,D-STARTED,E-INSTALLED");
- ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co1, co2, co3));
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1, co2, co3), resources);
ComponentInstanceState cisB = new ComponentInstanceState("B",
containerId, "aid");
ComponentInstanceState cisC = new ComponentInstanceState("C", containerId, "aid");
@@ -100,13 +119,14 @@ public class TestComponentCommandOrder {
cisB.setState(State.STARTED);
cisC.setState(State.STARTED);
- ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co));
+ ComponentCommandOrder cco = new ComponentCommandOrder(Arrays.asList(co),
+ resources);
Assert.assertTrue(cco.canExecute("A", Command.START, Arrays.asList(cisB, cisC)));
co.setCommand(" A-STAR");
co.setRequires("B-STARTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -115,7 +135,7 @@ public class TestComponentCommandOrder {
co.setCommand(" -START");
co.setRequires("B-STARTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -124,7 +144,7 @@ public class TestComponentCommandOrder {
co.setCommand(" A-START");
co.setRequires("B-STRTED , C-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -133,7 +153,7 @@ public class TestComponentCommandOrder {
co.setCommand(" A-START");
co.setRequires("B-STARTED , C-");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
@@ -142,10 +162,83 @@ public class TestComponentCommandOrder {
co.setCommand(" A-INSTALL");
co.setRequires("B-STARTED");
try {
- cco = new ComponentCommandOrder(Arrays.asList(co));
+ cco = new ComponentCommandOrder(Arrays.asList(co), resources);
+ Assert.fail("Instantiation should have failed.");
+ } catch (IllegalArgumentException ie) {
+ log.info(ie.getMessage());
+ }
+ }
+
+ @Test
+ public void testComponentCommandOrderBadComponent() throws Exception {
+ ConfTreeOperations resourcesGood = new ConfTreeOperations(new ConfTree());
+ resourcesGood.getOrAddComponent("A");
+ resourcesGood.getOrAddComponent("Z");
+ ConfTreeOperations resourcesBad = new ConfTreeOperations(new ConfTree());
+
+ CommandOrder co1 = new CommandOrder();
+ co1.setCommand("A-START");
+ co1.setRequires("Z-STARTED");
+ CommandOrder co2 = new CommandOrder();
+ co2.setCommand("Z-START");
+ co2.setRequires("A-STARTED");
+
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1), resourcesGood);
+ try {
+ cco = new ComponentCommandOrder(Arrays.asList(co1), resourcesBad);
+ Assert.fail("Instantiation should have failed.");
+ } catch (IllegalArgumentException ie) {
+ log.info(ie.getMessage());
+ }
+
+ cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesGood);
+ try {
+ cco = new ComponentCommandOrder(Arrays.asList(co2), resourcesBad);
Assert.fail("Instantiation should have failed.");
} catch (IllegalArgumentException ie) {
log.info(ie.getMessage());
}
}
+
+ @Test
+ public void testComponentCommandOrderPrefixes() throws Exception {
+ ConfTreeOperations resources = new ConfTreeOperations(new ConfTree());
+ resources.getOrAddComponent("a-A").put(ROLE_PREFIX, "a-");
+ resources.getOrAddComponent("b-B1").put(ROLE_PREFIX, "b-");
+ resources.getOrAddComponent("b-B2").put(ROLE_PREFIX, "b-");
+ resources.getOrAddComponent("c-C").put(ROLE_PREFIX, "c-");
+
+ CommandOrder co1 = new CommandOrder();
+ co1.setCommand("b-START");
+ co1.setRequires("a-STARTED");
+ CommandOrder co2 = new CommandOrder();
+ co2.setCommand("c-START");
+ co2.setRequires("b-STARTED");
+
+ ComponentCommandOrder cco = new ComponentCommandOrder(
+ Arrays.asList(co1, co2), resources);
+
+ ComponentInstanceState cisA = new ComponentInstanceState("a-A", containerId, "aid");
+ ComponentInstanceState cisB1 = new ComponentInstanceState("b-B1", containerId, "aid");
+ ComponentInstanceState cisB2 = new ComponentInstanceState("b-B2", containerId, "aid");
+ ComponentInstanceState cisC = new ComponentInstanceState("c-C", containerId, "aid");
+ cisA.setState(State.INSTALLED);
+ cisB1.setState(State.INSTALLED);
+ cisB2.setState(State.INSTALLED);
+ cisC.setState(State.INSTALLED);
+ List<ComponentInstanceState> states = Arrays.asList(cisA, cisB1, cisB2, cisC);
+ Assert.assertTrue(cco.canExecute("a-A", Command.START, states));
+ Assert.assertFalse(cco.canExecute("b-B1", Command.START, states));
+ Assert.assertFalse(cco.canExecute("b-B2", Command.START, states));
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisA.setState(State.STARTED);
+ Assert.assertTrue(cco.canExecute("b-B1", Command.START, states));
+ Assert.assertTrue(cco.canExecute("b-B2", Command.START, states));
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisB1.setState(State.STARTED);
+ Assert.assertFalse(cco.canExecute("c-C", Command.START, states));
+ cisB2.setState(State.STARTED);
+ Assert.assertTrue(cco.canExecute("c-C", Command.START, states));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
index 1cb6f0f..13919df 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@ -40,4 +40,8 @@ interface ResourcePaths {
String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json"
String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json"
+
+ String NESTED_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component_nested.json"
+ String NESTED_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json"
+ String NESTED_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json"
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a4dc574c/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
index b5e0270..292508a 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/misc/ExternalComponentIT.groovy
@@ -40,6 +40,7 @@ public class ExternalComponentIT extends AgentCommandTestBase
static String NAME = "test-external-component"
static String EXT_NAME = "test_sleep"
+ static String NESTED_NAME = "test-external-component-nested"
static String BUILD_APPCONFIG = ResourcePaths.SLEEP_APPCONFIG
static String BUILD_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
@@ -47,23 +48,30 @@ public class ExternalComponentIT extends AgentCommandTestBase
static String TEST_APPCONFIG = ResourcePaths.EXTERNAL_APPCONFIG
static String TEST_RESOURCES = ResourcePaths.EXTERNAL_RESOURCES
static String TEST_METAINFO = ResourcePaths.SLEEP_META
+ static String NEST_APPCONFIG = ResourcePaths.NESTED_APPCONFIG
+ static String NEST_RESOURCES = ResourcePaths.NESTED_RESOURCES
+ static String NEST_METAINFO = ResourcePaths.NESTED_META
public static final String SLEEP_100 = "SLEEP_100"
public static final String SLEEP_LONG = "SLEEP_LONG"
public static final String EXT_SLEEP_100 = EXT_NAME +
SliderKeys.COMPONENT_SEPARATOR + SLEEP_100
public static final String EXT_SLEEP_LONG = EXT_NAME +
SliderKeys.COMPONENT_SEPARATOR + SLEEP_LONG
+ public static final String NESTED_PREFIX = NAME +
+ SliderKeys.COMPONENT_SEPARATOR
@Before
public void prepareCluster() {
setupCluster(NAME)
setupCluster(EXT_NAME)
+ setupCluster(NESTED_NAME)
}
@After
public void destroyCluster() {
cleanup(NAME)
cleanup(EXT_NAME)
+ cleanup(NESTED_NAME)
}
@Test
@@ -129,5 +137,35 @@ public class ExternalComponentIT extends AgentCommandTestBase
expectLiveContainerCountReached(NAME, EXT_SLEEP_100, 0,
CONTAINER_LAUNCH_TIMEOUT)
+ cleanup(NAME)
+
+ describe NESTED_NAME
+
+ slider(0, [ACTION_BUILD, NAME, ARG_METAINFO, TEST_METAINFO,
+ ARG_TEMPLATE, TEST_APPCONFIG, ARG_RESOURCES, TEST_RESOURCES])
+
+ slider(0, [ACTION_CREATE, NESTED_NAME, ARG_METAINFO, NEST_METAINFO,
+ ARG_TEMPLATE, NEST_APPCONFIG, ARG_RESOURCES, NEST_RESOURCES])
+
+ ensureApplicationIsUp(NESTED_NAME)
+ status(0, NESTED_NAME)
+
+ cd = execStatus(NESTED_NAME)
+
+ assert 5 == cd.statistics.size()
+ assert cd.statistics.keySet().containsAll([SliderKeys.COMPONENT_AM,
+ NESTED_PREFIX + SLEEP_100,
+ NESTED_PREFIX + SLEEP_LONG,
+ NESTED_PREFIX + EXT_SLEEP_100,
+ NESTED_PREFIX + EXT_SLEEP_LONG])
+
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_LONG, 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
+ expectLiveContainerCountReached(NESTED_NAME, NESTED_PREFIX + EXT_SLEEP_100, 0,
+ CONTAINER_LAUNCH_TIMEOUT)
}
}
[4/6] incubator-slider git commit: Merge branch
'feature/SLIDER-875_uber_app' into develop
Posted by bi...@apache.org.
Merge branch 'feature/SLIDER-875_uber_app' into develop
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8863393f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8863393f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8863393f
Branch: refs/heads/develop
Commit: 8863393fad81865e136c9c6b86774babdddb08ad
Parents: 7428cae 2893d34
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Aug 10 08:33:16 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Aug 10 08:33:16 2016 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +
.../java/org/apache/slider/api/RoleKeys.java | 5 +
.../org/apache/slider/client/SliderClient.java | 13 +-
.../org/apache/slider/common/SliderKeys.java | 13 ++
.../apache/slider/common/tools/SliderUtils.java | 88 ++++++++
.../slider/core/build/InstanceBuilder.java | 205 +++++++++++++++++++
.../slider/core/conf/ConfTreeOperations.java | 50 +++++
.../providers/AbstractProviderService.java | 3 +-
.../slider/providers/ProviderService.java | 4 +-
.../providers/agent/AgentClientProvider.java | 29 +--
.../slider/providers/agent/AgentKeys.java | 3 +
.../providers/agent/AgentProviderService.java | 188 ++++++++++++-----
.../slider/providers/agent/AgentUtils.java | 16 ++
.../providers/agent/ComponentCommandOrder.java | 112 +++++++---
.../server/appmaster/SliderAppMaster.java | 3 +-
.../slider/server/appmaster/state/AppState.java | 5 +
.../appmaster/web/rest/agent/AgentResource.java | 4 +-
.../sleep_cmd/appConfig_external_component.json | 12 ++
.../appConfig_external_component_nested.json | 12 ++
.../test_min_pkg/sleep_cmd/metainfo.json | 6 +
.../metainfo_external_component_nested.json | 14 ++
.../sleep_cmd/resources_external_component.json | 22 ++
.../resources_external_component_nested.json | 12 ++
.../TestBuildExternalComponent.groovy | 138 +++++++++++++
.../slider/client/TestReplaceTokens.groovy | 5 +-
.../model/mock/MockProviderService.groovy | 3 +-
.../agent/TestAgentProviderService.java | 71 +++----
.../agent/TestComponentCommandOrder.java | 107 +++++++++-
.../apache/slider/funtest/ResourcePaths.groovy | 9 +-
.../funtest/misc/ExternalComponentIT.groovy | 171 ++++++++++++++++
30 files changed, 1165 insertions(+), 161 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 01a3f1a,8341af4..9ea984c
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@@ -103,7 -98,8 +104,9 @@@ public interface AgentKeys
String CERT_FILE_LOCALIZATION_PATH = INFRA_RUN_SECURITY_DIR + "ca.crt";
String KEY_CONTAINER_LAUNCH_DELAY = "container.launch.delay.sec";
String TEST_RELAX_VERIFICATION = "test.relax.validation";
+ String AM_CONFIG_GENERATION = "am.config.generation";
+
+ String DEFAULT_METAINFO_MAP_KEY = "DEFAULT_KEY";
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4ffae7c,452122f..bc362b5
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -170,10 -168,8 +172,10 @@@ public class AgentProviderService exten
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Metainfo metaInfo = null;
+ private volatile Map<String, MetainfoHolder> metaInfoMap = new HashMap<>();
+ private SliderFileSystem fileSystem = null;
+ private Map<String, DefaultConfig> defaultConfigs = null;
- private ComponentCommandOrder commandOrder = null;
+ private ComponentCommandOrder commandOrder = new ComponentCommandOrder();
private HeartbeatMonitor monitor;
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@@ -277,14 -285,23 +291,26 @@@
// Reads the metainfo.xml in the application package and loads it
private void buildMetainfo(AggregateConf instanceDefinition,
- SliderFileSystem fileSystem) throws IOException, SliderException {
- String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
- .getAppConfOperations());
+ SliderFileSystem fileSystem,
+ String roleGroup)
+ throws IOException, SliderException {
+ String mapKey = instanceDefinition.getAppConfOperations()
+ .getComponentOpt(roleGroup, ROLE_PREFIX, DEFAULT_METAINFO_MAP_KEY);
+ String appDef = SliderUtils.getApplicationDefinitionPath(
+ instanceDefinition.getAppConfOperations(), roleGroup);
+ MapOperations component = null;
+ if (roleGroup != null) {
+ component = instanceDefinition.getAppConfOperations().getComponent(roleGroup);
+ }
- if (metaInfo == null) {
+ MetainfoHolder metaInfoHolder = metaInfoMap.get(mapKey);
+ if (metaInfoHolder == null) {
synchronized (syncLock) {
- if (metaInfo == null) {
++ if (this.fileSystem == null) {
+ this.fileSystem = fileSystem;
++ }
+ metaInfoHolder = metaInfoMap.get(mapKey);
+ if (metaInfoHolder == null) {
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
@@@ -1764,11 -1642,9 +1828,11 @@@
log.info("Status report: {}", status.toString());
if (status.getConfigs() != null) {
- Application application = getMetaInfo().getApplication();
+ Application application = getMetaInfo(componentGroup).getApplication();
- if ((!canAnyMasterPublishConfig() || canPublishConfig(componentGroup)) &&
- if (canAnyMasterPublishConfig(componentGroup) == false || canPublishConfig(componentGroup)) {
++ if ((!canAnyMasterPublishConfig(componentGroup) || canPublishConfig(componentGroup)) &&
+ !getAmState().getAppConfSnapshot().getComponentOptBool(
+ componentGroup, AgentKeys.AM_CONFIG_GENERATION, false)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8863393f/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
----------------------------------------------------------------------
diff --cc slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
index c0aa06a,13919df..37503d9
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/ResourcePaths.groovy
@@@ -38,9 -38,10 +38,16 @@@ interface ResourcePaths
String SLEEP_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo.json"
String SLEEP_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig.json"
+ String AM_CONFIG_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_am_config/resources.json"
+ String AM_CONFIG_META = "$SLIDER_CORE_APP_PACKAGES/test_am_config/metainfo.json"
+ String AM_CONFIG_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_am_config/appConfig.json"
+
+ String UNIQUE_COMPONENT_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_command_log/resources_unique_names.json"
- }
++
+ String EXTERNAL_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component.json"
+ String EXTERNAL_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component.json"
+
+ String NESTED_RESOURCES = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/resources_external_component_nested.json"
+ String NESTED_META = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/metainfo_external_component_nested.json"
+ String NESTED_APPCONFIG = "$SLIDER_CORE_APP_PACKAGES/test_min_pkg/sleep_cmd/appConfig_external_component_nested.json"
-}
++}
[6/6] incubator-slider git commit: SLIDER-875 fix more funtest
failures
Posted by bi...@apache.org.
SLIDER-875 fix more funtest failures
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/50db94ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/50db94ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/50db94ce
Branch: refs/heads/develop
Commit: 50db94ce41e566b278ecb9c5079e97f7881fac63
Parents: 937b45b
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Aug 10 13:24:02 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Aug 11 10:07:40 2016 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloBasicIT.groovy | 3 +++
.../funtest/accumulo/AccumuloReadWriteIT.groovy | 2 +-
.../accumulo/AccumuloReadWriteSSLIT.groovy | 2 +-
.../funtest/accumulo/AccumuloScriptIT.groovy | 2 +-
.../funtest/accumulo/AccumuloScriptSSLIT.groovy | 2 +-
.../providers/agent/AgentProviderService.java | 18 ++++++++++++++++++
.../test_min_pkg/sleep_cmd/resources.json | 2 +-
7 files changed, 26 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
index 81d290a..260743f 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloBasicIT.groovy
@@ -47,6 +47,7 @@ import static org.apache.hadoop.registry.client.binding.RegistryUtils.servicePat
class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
protected static final String PROVIDER_PROPERTY = "site.accumulo-site." +
Property.GENERAL_SECURITY_CREDENTIAL_PROVIDER_PATHS
+ protected static final String INSTANCE_PROPERTY = "site.client.instance.name"
protected static final String KEY_PASS = "keypass"
protected static final String TRUST_PASS = "trustpass"
protected ConfTree tree
@@ -115,6 +116,8 @@ class AccumuloBasicIT extends AccumuloAgentCommandTestBase {
provider.flush()
assert clusterFS.exists(jksPath), "jks $jks not created"
log.info("Created credential provider $jks for test")
+ tree.global.put(INSTANCE_PROPERTY, tree.global.get(INSTANCE_PROPERTY)
+ .replaceAll(Pattern.quote('${CLUSTER_NAME}'), clusterName))
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
index 0ecf210..d5c85bc 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteIT.groovy
@@ -55,7 +55,7 @@ class AccumuloReadWriteIT extends AccumuloBasicIT {
ClientConfiguration configuration = new ClientConfiguration()
configuration.setProperty(ClientConfiguration.ClientProperty.INSTANCE_ZK_HOST, zookeepers)
- configuration.setProperty(ClientConfiguration.ClientProperty.INSTANCE_NAME, tree.global.get("site.client.instance.name"))
+ configuration.setProperty(ClientConfiguration.ClientProperty.INSTANCE_NAME, tree.global.get(INSTANCE_PROPERTY))
ZooKeeperInstance instance = new ZooKeeperInstance(configuration)
Connector connector = instance.getConnector(USER, new PasswordToken(PASSWORD))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
index b67cd5c..90f5fa1 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloReadWriteSSLIT.groovy
@@ -50,7 +50,7 @@ class AccumuloReadWriteSSLIT extends AccumuloSSLTestBase {
RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
ClientConfiguration conf = new ClientConfiguration()
- .withInstance(tree.global.get("site.client.instance.name"))
+ .withInstance(tree.global.get(INSTANCE_PROPERTY))
.withZkHosts(zookeepers)
.withSsl(true)
.withKeystore(clientKeyStoreFile.toString(), KEY_PASS, STORE_TYPE)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
index 24b6e22..8307104 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptIT.groovy
@@ -140,7 +140,7 @@ class AccumuloScriptIT extends AccumuloBasicIT {
String zookeepers = SLIDER_CONFIG.get(
RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
- String instance = tree.global.get("site.client.instance.name")
+ String instance = tree.global.get(INSTANCE_PROPERTY)
accumulo("shell -u $USER -p $PASSWORD -e \"createtable test1\"")
accumulo(InsertWithBatchWriter.class.getName() + " -i $instance -z " +
"$zookeepers -u $USER -p $PASSWORD -t test1")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptSSLIT.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptSSLIT.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptSSLIT.groovy
index 710812d..66ce13e 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptSSLIT.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloScriptSSLIT.groovy
@@ -141,7 +141,7 @@ class AccumuloScriptSSLIT extends AccumuloSSLTestBase {
String zookeepers = SLIDER_CONFIG.get(
RegistryConstants.KEY_REGISTRY_ZK_QUORUM,
FuntestProperties.DEFAULT_SLIDER_ZK_HOSTS)
- String instance = tree.global.get("site.client.instance.name")
+ String instance = tree.global.get(INSTANCE_PROPERTY)
accumulo("shell -u $USER -p $PASSWORD -e \"createtable test2\"")
accumulo(InsertWithBatchWriter.class.getName() + " -i $instance -z " +
"$zookeepers -u $USER -p $PASSWORD -t test2")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4d68ba2..7594d51 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -129,6 +129,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Scanner;
import java.util.Set;
import java.util.TreeMap;
@@ -136,6 +137,7 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.regex.Pattern;
import static org.apache.slider.api.RoleKeys.ROLE_PREFIX;
import static org.apache.slider.server.appmaster.web.rest.RestPaths.SLIDER_PATH_AGENTS;
@@ -663,6 +665,22 @@ public class AgentProviderService extends AbstractProviderService implements
AggregateConf instanceDefinition,
MapOperations compOps)
throws SliderException, IOException {
+ // substitute CLUSTER_NAME into credentials
+ Map<String,List<String>> newcred = new HashMap<>();
+ for (Entry<String,List<String>> entry : instanceDefinition.getAppConf().credentials.entrySet()) {
+ List<String> resultList = new ArrayList<>();
+ for (String v : entry.getValue()) {
+ resultList.add(v.replaceAll(Pattern.quote("${CLUSTER_NAME}"),
+ clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
+ clusterName));
+ }
+ newcred.put(entry.getKey().replaceAll(Pattern.quote("${CLUSTER_NAME}"),
+ clusterName).replaceAll(Pattern.quote("${CLUSTER}"),
+ clusterName),
+ resultList);
+ }
+ instanceDefinition.getAppConf().credentials = newcred;
+
// generate and localize security stores
SecurityStore[] stores = generateSecurityStores(container, role,
instanceDefinition, compOps);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50db94ce/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
index e0ed16a..b9eeb10 100644
--- a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
@@ -6,7 +6,7 @@
},
"components": {
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "384"
},
"SLEEP_100": {
"yarn.role.priority": "1",