You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2017/08/14 21:13:46 UTC
[08/15] hadoop git commit: YARN-6903. Yarn-native-service framework
core rewrite. Contributed by Jian He
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index 64783b6..235d3da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -16,7 +16,7 @@
*/
package org.apache.slider.common.tools;
-import org.apache.slider.common.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index 2e1236d..fc57c82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
@@ -50,10 +49,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.slider.Slider;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.types.ContainerInformation;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.params.Arguments;
-import org.apache.slider.common.params.SliderActions;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.client.params.Arguments;
+import org.apache.hadoop.yarn.service.client.params.SliderActions;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.BadCommandArgumentsException;
import org.apache.slider.core.exceptions.BadConfigException;
@@ -111,7 +110,7 @@ import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
-import static org.apache.slider.common.SliderKeys.COMPONENT_SEPARATOR;
+import static org.apache.hadoop.yarn.service.conf.SliderKeys.COMPONENT_SEPARATOR;
/**
* These are slider-specific Util methods
@@ -373,6 +372,13 @@ public final class SliderUtils {
private static final PatternValidator clusternamePattern
= new PatternValidator("[a-z][a-z0-9_-]*");
+ private static final PatternValidator compNamePattern
+ = new PatternValidator("[a-z][a-z0-9-]*");
+
+ public static void validateCompName(String compName) {
+ compNamePattern.validate(compName);
+ }
+
/**
* Normalize a cluster name then verify that it is valid
* @param name proposed cluster name
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
index 7f3134a..1430c5a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/exceptions/SliderException.java
@@ -18,7 +18,7 @@
package org.apache.slider.core.exceptions;
-import org.apache.slider.common.SliderExitCodes;
+import org.apache.hadoop.yarn.service.conf.SliderExitCodes;
import org.apache.slider.core.main.ServiceLaunchException;
public class SliderException extends ServiceLaunchException implements
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 8f61bf4..55ffbf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -19,30 +19,17 @@
package org.apache.slider.core.launch;
import com.google.common.base.Preconditions;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.Records;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.conf.MapOperations;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.lang.reflect.Method;
-import java.lang.reflect.InvocationTargetException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -51,12 +38,12 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import static org.apache.slider.providers.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
+import static org.apache.hadoop.yarn.service.provider.docker.DockerKeys.DEFAULT_DOCKER_NETWORK;
/**
* Launcher of applications: base class
*/
-public abstract class AbstractLauncher extends Configured {
+public class AbstractLauncher {
private static final Logger log =
LoggerFactory.getLogger(AbstractLauncher.class);
public static final String CLASSPATH = "CLASSPATH";
@@ -68,7 +55,6 @@ public abstract class AbstractLauncher extends Configured {
* Env vars; set up at final launch stage
*/
protected final Map<String, String> envVars = new HashMap<>();
- protected final MapOperations env = new MapOperations("env", envVars);
protected final ContainerLaunchContext containerLaunchContext =
Records.newRecord(ContainerLaunchContext.class);
protected final List<String> commands = new ArrayList<>(20);
@@ -77,36 +63,24 @@ public abstract class AbstractLauncher extends Configured {
private final Map<String, ByteBuffer> serviceData = new HashMap<>();
// security
protected final Credentials credentials;
- protected LogAggregationContext logAggregationContext;
protected boolean yarnDockerMode = false;
protected String dockerImage;
protected String dockerNetwork = DEFAULT_DOCKER_NETWORK;
protected String dockerHostname;
- protected String yarnContainerMountPoints;
protected String runPrivilegedContainer;
/**
* Create instance.
- * @param conf configuration
* @param coreFileSystem filesystem
* @param credentials initial set of credentials -null is permitted
*/
- protected AbstractLauncher(Configuration conf,
+ public AbstractLauncher(
CoreFileSystem coreFileSystem,
Credentials credentials) {
- super(conf);
this.coreFileSystem = coreFileSystem;
this.credentials = credentials != null ? credentials: new Credentials();
}
-
- /**
- * Get the container. Until "completed", this isn't valid to launch.
- * @return the container to launch
- */
- public ContainerLaunchContext getContainerLaunchContext() {
- return containerLaunchContext;
- }
public void setYarnDockerMode(boolean yarnDockerMode){
this.yarnDockerMode = yarnDockerMode;
@@ -116,8 +90,8 @@ public abstract class AbstractLauncher extends Configured {
* Get the env vars to work on
* @return env vars
*/
- public MapOperations getEnv() {
- return env;
+ public Map<String, String> getEnv() {
+ return envVars;
}
/**
@@ -128,14 +102,6 @@ public abstract class AbstractLauncher extends Configured {
return commands;
}
- /**
- * Get the map of local resources.
- * @return the live map of local resources.
- */
- public Map<String, LocalResource> getLocalResources() {
- return localResources;
- }
-
public void addLocalResource(String subPath, LocalResource resource) {
localResources.put(subPath, resource);
}
@@ -146,18 +112,6 @@ public abstract class AbstractLauncher extends Configured {
}
/**
- * Add a set of local resources
- * @param resourceMap map of name:resource to add
- */
- public void addLocalResources(Map<String, LocalResource> resourceMap) {
- localResources.putAll(resourceMap);
- }
-
- public Map<String, ByteBuffer> getServiceData() {
- return serviceData;
- }
-
- /**
* Accessor to the credentials
* @return the credentials associated with this launcher
*/
@@ -165,36 +119,12 @@ public abstract class AbstractLauncher extends Configured {
return credentials;
}
- /**
- * Add a command line. It is converted to a single command before being
- * added.
- * @param cmd
- */
- public void addCommandLine(CommandLineBuilder cmd) {
- commands.add(cmd.build());
- }
public void addCommand(String cmd) {
commands.add(cmd);
}
/**
- * Add a list of commands. Each element in the list becomes a single command
- * @param commandList list of commands
- */
- public void addCommands(List<String> commandList) {
- commands.addAll(commandList);
- }
-
- /**
- * Get all commands as a string, separated by ";". This is for diagnostics
- * @return a string description of the commands
- */
- public String getCommandsAsString() {
- return SliderUtils.join(getCommands(), "; ");
- }
-
- /**
* Complete the launch context (copy in env vars, etc).
* @return the container to launch
*/
@@ -211,7 +141,7 @@ public abstract class AbstractLauncher extends Configured {
log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
}
}
- containerLaunchContext.setEnvironment(env);
+ containerLaunchContext.setEnvironment(envVars);
//service data
if (log.isDebugEnabled()) {
@@ -281,120 +211,7 @@ public abstract class AbstractLauncher extends Configured {
protected void propagateUsernameInInsecureCluster() throws IOException {
//insecure cluster: propagate user name via env variable
String userName = UserGroupInformation.getCurrentUser().getUserName();
- env.put(SliderKeys.HADOOP_USER_NAME, userName);
- }
-
- /**
- * Extract any resource requirements from this component's settings.
- * All fields that are set will override the existing values -if
- * unset that resource field will be left unchanged.
- *
- * Important: the configuration must already be fully resolved
- * in order to pick up global options.
- * @param resource resource to configure
- * @param map map of options
- */
- public void extractResourceRequirements(Resource resource,
- Map<String, String> map) {
-
- if (map != null) {
- MapOperations options = new MapOperations("", map);
- resource.setMemory(options.getOptionInt(ResourceKeys.YARN_MEMORY,
- resource.getMemory()));
- resource.setVirtualCores(options.getOptionInt(ResourceKeys.YARN_CORES,
- resource.getVirtualCores()));
- }
- }
-
- public void extractLogAggregationContext(Map<String, String> map) {
- if (map != null) {
- String logPatternSepStr = "\\|";
- String logPatternJoinStr = "|";
- MapOperations options = new MapOperations("", map);
-
- List<String> logIncludePatterns = new ArrayList<>();
- String includePatternExpression = options.getOption(
- ResourceKeys.YARN_LOG_INCLUDE_PATTERNS, "").trim();
- if (!includePatternExpression.isEmpty()) {
- String[] includePatterns = includePatternExpression
- .split(logPatternSepStr);
- for (String includePattern : includePatterns) {
- String trimmedIncludePattern = includePattern.trim();
- if (!trimmedIncludePattern.isEmpty()) {
- logIncludePatterns.add(trimmedIncludePattern);
- }
- }
- }
- String logIncludePattern = StringUtils.join(logIncludePatterns,
- logPatternJoinStr);
- log.info("Log include patterns: {}", logIncludePattern);
-
- List<String> logExcludePatterns = new ArrayList<>();
- String excludePatternExpression = options.getOption(
- ResourceKeys.YARN_LOG_EXCLUDE_PATTERNS, "").trim();
- if (!excludePatternExpression.isEmpty()) {
- String[] excludePatterns = excludePatternExpression
- .split(logPatternSepStr);
- for (String excludePattern : excludePatterns) {
- String trimmedExcludePattern = excludePattern.trim();
- if (!trimmedExcludePattern.isEmpty()) {
- logExcludePatterns.add(trimmedExcludePattern);
- }
- }
- }
- String logExcludePattern = StringUtils.join(logExcludePatterns,
- logPatternJoinStr);
- log.info("Log exclude patterns: {}", logExcludePattern);
-
- // SLIDER-810/YARN-3154 - hadoop 2.7.0 onwards a new instance method has
- // been added for log aggregation for LRS. Existing newInstance method's
- // behavior has changed and is used for log aggregation only after the
- // application has finished. This forces Slider users to move to hadoop
- // 2.7.0+ just for log aggregation, which is not very desirable. So we
- // decided to use reflection here to find out if the new 2.7.0 newInstance
- // method is available. If yes, then we use it, so log aggregation will
- // work in hadoop 2.7.0+ env. If no, then we fallback to the pre-2.7.0
- // newInstance method, which means log aggregation will work as expected
- // in hadoop 2.6 as well.
- // TODO: At some point, say 2-3 Slider releases down, when most users are
- // running hadoop 2.7.0, we should get rid of the reflection code here.
- try {
- Method logAggregationContextMethod = LogAggregationContext.class
- .getMethod("newInstance", String.class, String.class, String.class,
- String.class);
- // Need to set include/exclude patterns appropriately since by default
- // rolled log aggregation is not done for any files, so defaults are
- // - include pattern set to ""
- // - exclude pattern set to "*"
- // For Slider we want all logs to be uploaded if include/exclude
- // patterns are left empty by the app owner in resources file
- if (StringUtils.isEmpty(logIncludePattern)
- && StringUtils.isEmpty(logExcludePattern)) {
- logIncludePattern = ".*";
- logExcludePattern = "";
- } else if (StringUtils.isEmpty(logIncludePattern)
- && StringUtils.isNotEmpty(logExcludePattern)) {
- logIncludePattern = ".*";
- } else if (StringUtils.isNotEmpty(logIncludePattern)
- && StringUtils.isEmpty(logExcludePattern)) {
- logExcludePattern = "";
- }
- log.debug("LogAggregationContext newInstance method for rolled logs "
- + "include/exclude patterns is available");
- log.info("Modified log include patterns: {}", logIncludePattern);
- log.info("Modified log exclude patterns: {}", logExcludePattern);
- logAggregationContext = (LogAggregationContext) logAggregationContextMethod
- .invoke(null, null, null, logIncludePattern, logExcludePattern);
- } catch (NoSuchMethodException | SecurityException
- | IllegalAccessException | IllegalArgumentException
- | InvocationTargetException e) {
- log.debug("LogAggregationContext newInstance method for rolled logs "
- + "include/exclude patterns is not available - fallback to old one");
- log.debug(e.toString());
- logAggregationContext = LogAggregationContext.newInstance(
- logIncludePattern, logExcludePattern);
- }
- }
+ envVars.put(SliderKeys.HADOOP_USER_NAME, userName);
}
/**
@@ -413,57 +230,15 @@ public abstract class AbstractLauncher extends Configured {
public void setEnv(String var, String value) {
Preconditions.checkArgument(var != null, "null variable name");
Preconditions.checkArgument(value != null, "null value");
- env.put(var, value);
+ envVars.put(var, value);
}
- /**
- * Set an environment variable if its value is non-null.
- * @param var variable name
- * @param value value (may be null)
- */
- public void maybeSetEnv(String var, String value) {
- if (value != null) {
- setEnv(var, value);
- }
- }
public void putEnv(Map<String, String> map) {
- env.putAll(map);
+ envVars.putAll(map);
}
- public String[] dumpEnvToString() {
-
- List<String> nodeEnv = new ArrayList<>();
-
- for (Map.Entry<String, String> entry : env.entrySet()) {
- String envElt = String.format("%s=\"%s\"",
- entry.getKey(),
- entry.getValue());
- log.debug(envElt);
- nodeEnv.add(envElt);
- }
- String[] envDescription = nodeEnv.toArray(new String[nodeEnv.size()]);
-
- return envDescription;
- }
-
- /**
- * Submit an entire directory
- * @param srcDir src path in filesystem
- * @param destRelativeDir relative path under destination local dir
- * @throws IOException IO problems
- */
- public void submitDirectory(Path srcDir, String destRelativeDir)
- throws IOException {
- //add the configuration resources
- Map<String, LocalResource> confResources;
- confResources = coreFileSystem.submitDirectory(
- srcDir,
- destRelativeDir);
- addLocalResources(confResources);
- }
-
public void setDockerImage(String dockerImage) {
this.dockerImage = dockerImage;
}
@@ -476,14 +251,6 @@ public abstract class AbstractLauncher extends Configured {
this.dockerHostname = dockerHostname;
}
- public void setYarnContainerMountPoints(String yarnContainerMountPoints) {
- this.yarnContainerMountPoints = yarnContainerMountPoints;
- }
-
- public void setRunPrivilegedContainer(String runPrivilegedContainer) {
- this.runPrivilegedContainer = runPrivilegedContainer;
- }
-
public void setRunPrivilegedContainer(boolean runPrivilegedContainer) {
if (runPrivilegedContainer) {
this.runPrivilegedContainer = Boolean.toString(true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index d220928..7e164e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -38,35 +38,11 @@ import java.net.InetSocketAddress;
public class ContainerLauncher extends AbstractLauncher {
private static final Logger log =
LoggerFactory.getLogger(ContainerLauncher.class);
- // Allocated container
- public final Container container;
public ContainerLauncher(Configuration conf,
CoreFileSystem coreFileSystem,
Container container,
Credentials credentials) {
- super(conf, coreFileSystem, credentials);
- this.container = container;
+ super(coreFileSystem, credentials);
}
-
- /**
- * This code is in the dist shell examples -it's been moved here
- * so that if it is needed, it's still here
- * @return a remote user with a token to access the container.
- */
- public UserGroupInformation setupUGI() {
- UserGroupInformation user =
- UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
-
- org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
- if (containerToken != null) {
- Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken, cmAddress);
- user.addToken(token);
- }
- return user;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
index 5357cc4..1fd49ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/launch/CredentialUtils.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.slider.common.SliderXmlConfKeys;
+import org.apache.hadoop.yarn.service.conf.SliderXmlConfKeys;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
index 37b36ea..ac8fca5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/SliderRegistryUtils.java
@@ -20,7 +20,7 @@ package org.apache.slider.core.registry;
import com.google.common.base.Preconditions;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.common.SliderKeys;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
/**
* Miscellaneous methods to assist slider registry work
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
deleted file mode 100644
index ea92ff7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ /dev/null
@@ -1,153 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.registry.client.api.RegistryOperations;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.SliderException;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.file.Paths;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-public abstract class AbstractClientProvider {
-
- public AbstractClientProvider() {
- }
-
- /**
- * Generates a fixed format of application tags given one or more of
- * application name, version and description. This allows subsequent query for
- * an application with a name only, version only or description only or any
- * combination of those as filters.
- *
- * @param appName name of the application
- * @param appVersion version of the application
- * @param appDescription brief description of the application
- * @return
- */
- public static final Set<String> createApplicationTags(String appName,
- String appVersion, String appDescription) {
- Set<String> tags = new HashSet<>();
- tags.add(SliderUtils.createNameTag(appName));
- if (appVersion != null) {
- tags.add(SliderUtils.createVersionTag(appVersion));
- }
- if (appDescription != null) {
- tags.add(SliderUtils.createDescriptionTag(appDescription));
- }
- return tags;
- }
-
- /**
- * Validate the artifact.
- * @param artifact
- */
- public abstract void validateArtifact(Artifact artifact, FileSystem
- fileSystem) throws IOException;
-
- protected abstract void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException;
-
- /**
- * Validate the config files.
- * @param configFiles config file list
- * @param fs file system
- */
- public void validateConfigFiles(List<ConfigFile> configFiles,
- FileSystem fs) throws IOException {
- Set<String> destFileSet = new HashSet<>();
-
- for (ConfigFile file : configFiles) {
- if (file.getType() == null) {
- throw new IllegalArgumentException("File type is empty");
- }
-
- if (file.getType().equals(ConfigFile.TypeEnum.TEMPLATE) && StringUtils
- .isEmpty(file.getSrcFile())) {
- throw new IllegalArgumentException(
- "Src_file is empty for " + ConfigFile.TypeEnum.TEMPLATE);
-
- }
- if (!StringUtils.isEmpty(file.getSrcFile())) {
- Path p = new Path(file.getSrcFile());
- if (!fs.exists(p)) {
- throw new IllegalArgumentException(
- "Src_file does not exist for config file: " + file
- .getSrcFile());
- }
- }
-
- if (StringUtils.isEmpty(file.getDestFile())) {
- throw new IllegalArgumentException("Dest_file is empty.");
- }
-
- if (destFileSet.contains(file.getDestFile())) {
- throw new IllegalArgumentException(
- "Duplicated ConfigFile exists: " + file.getDestFile());
- }
- destFileSet.add(file.getDestFile());
-
- java.nio.file.Path destPath = Paths.get(file.getDestFile());
- if (!destPath.isAbsolute() && destPath.getNameCount() > 1) {
- throw new IllegalArgumentException("Non-absolute dest_file has more " +
- "than one path element");
- }
-
- // provider-specific validation
- validateConfigFile(file, fs);
- }
- }
-
- /**
- * Process client operations for applications such as install, configure.
- * @param fileSystem
- * @param registryOperations
- * @param configuration
- * @param operation
- * @param clientInstallPath
- * @param clientPackage
- * @param clientConfig
- * @param name
- * @throws SliderException
- */
- public void processClientOperation(SliderFileSystem fileSystem,
- RegistryOperations registryOperations,
- Configuration configuration,
- String operation,
- File clientInstallPath,
- File clientPackage,
- JSONObject clientConfig,
- String name)
- throws SliderException {
- throw new SliderException("Provider does not support client operations.");
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e0d9402..1e1b1b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -23,10 +23,13 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.service.conf.SliderKeys;
+import org.apache.hadoop.yarn.service.provider.ProviderService;
+import org.apache.hadoop.yarn.service.provider.ProviderUtils;
+import org.apache.hadoop.yarn.service.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.api.resource.Application;
import org.apache.slider.api.resource.Component;
import org.apache.slider.api.resource.ContainerState;
-import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.SliderException;
@@ -35,7 +38,6 @@ import org.apache.slider.core.launch.ContainerLauncher;
import org.apache.slider.core.registry.docstore.PublishedConfiguration;
import org.apache.slider.server.appmaster.state.RoleInstance;
import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -44,14 +46,14 @@ import java.io.IOException;
import java.util.Map;
import java.util.Map.Entry;
-import static org.apache.slider.util.ServiceApiUtil.$;
+import static org.apache.hadoop.yarn.service.utils.ServiceApiUtil.$;
public abstract class AbstractProviderService extends AbstractService
implements ProviderService, SliderKeys {
protected static final Logger log =
LoggerFactory.getLogger(AbstractProviderService.class);
- private static final ProviderUtils providerUtils = new ProviderUtils(log);
+ private static final ProviderUtils providerUtils = new ProviderUtils();
protected StateAccessForProviders amState;
protected YarnRegistryViewForProviders yarnRegistry;
private ServiceTimelinePublisher serviceTimelinePublisher;
@@ -64,15 +66,6 @@ public abstract class AbstractProviderService extends AbstractService
Application application, RoleInstance roleInstance,
SliderFileSystem fileSystem) throws IOException;
- @Override
- public void setAMState(StateAccessForProviders stateAccessor) {
- this.amState = stateAccessor;
- }
-
- @Override
- public void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry) {
- this.yarnRegistry = yarnRegistry;
- }
public void buildContainerLaunchContext(ContainerLauncher launcher,
Application application, Container container, ProviderRole providerRole,
@@ -85,7 +78,7 @@ public abstract class AbstractProviderService extends AbstractService
// Get pre-defined tokens
Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
Map<String, String> tokensForSubstitution = providerUtils
- .initCompTokensForSubstitute(roleInstance);
+ .initCompTokensForSubstitute(null);
tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
launcher.putEnv(SliderUtils
@@ -105,8 +98,6 @@ public abstract class AbstractProviderService extends AbstractService
providerUtils.addComponentHostTokens(tokensForSubstitution, amState);
// create config file on hdfs and add local resource
- providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
- component, tokensForSubstitution, roleInstance, amState);
// substitute launch command
String launchCommand = ProviderUtils
@@ -141,15 +132,6 @@ public abstract class AbstractProviderService extends AbstractService
return false;
}
- try {
- providerUtils.updateServiceRecord(amState, yarnRegistry,
- containerId.toString(), instance.role, status.getIPs(), status.getHost());
- } catch (IOException e) {
- // could not write service record to ZK, log and retry
- log.warn("Error updating container {} service record in registry, " +
- "retrying", containerId, e);
- return true;
- }
// TODO publish ip and host
org.apache.slider.api.resource.Container container =
instance.providerRole.component.getContainer(containerId.toString());
@@ -163,8 +145,4 @@ public abstract class AbstractProviderService extends AbstractService
return false;
}
- @Override
- public void setServiceTimelinePublisher(ServiceTimelinePublisher publisher) {
- this.serviceTimelinePublisher = publisher;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
deleted file mode 100644
index 004e81f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultClientProvider.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-
-import java.io.IOException;
-import java.nio.file.Paths;
-
-public class DefaultClientProvider extends AbstractClientProvider {
-
- public DefaultClientProvider() {
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- // validate dest_file is not absolute
- if (Paths.get(configFile.getDestFile()).isAbsolute()) {
- throw new IllegalArgumentException(
- "Dest_file must not be absolute path: " + configFile.getDestFile());
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
deleted file mode 100644
index 09a1423..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderFactory.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-public final class DefaultProviderFactory extends SliderProviderFactory {
- private static final SliderProviderFactory FACTORY = new
- DefaultProviderFactory();
-
- private DefaultProviderFactory() {}
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DefaultClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DefaultProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static SliderProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
deleted file mode 100644
index 8d2725c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/DefaultProviderService.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers;
-
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-
-import java.io.IOException;
-
-public class DefaultProviderService extends AbstractProviderService {
-
- protected DefaultProviderService() {
- super(DefaultProviderService.class.getSimpleName());
- }
-
- @Override
- public void processArtifact(ContainerLauncher launcher, Application
- application, RoleInstance roleInstance, SliderFileSystem fileSystem)
- throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
deleted file mode 100644
index edd313b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.appmaster.timelineservice.ServiceTimelinePublisher;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-
-import java.io.IOException;
-
-public interface ProviderService extends Service {
-
- /**
- * Set up the entire container launch context
- */
- void buildContainerLaunchContext(ContainerLauncher containerLauncher,
- Application application, Container container, ProviderRole providerRole,
- SliderFileSystem sliderFileSystem, RoleInstance roleInstance)
- throws IOException, SliderException;
-
-
- void setAMState(StateAccessForProviders stateAccessForProviders);
-
- /**
- * Bind to the YARN registry
- * @param yarnRegistry YARN registry
- */
- void bindToYarnRegistry(YarnRegistryViewForProviders yarnRegistry);
-
- /**
- * Process container status
- * @return true if status needs to be requested again, false otherwise
- */
- boolean processContainerStatus(ContainerId containerId,
- ContainerStatus status);
-
- /**
- * Set service publisher.
- * @param serviceTimelinePublisher service publisher.
- */
- void setServiceTimelinePublisher(
- ServiceTimelinePublisher serviceTimelinePublisher);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
deleted file mode 100644
index ecc521f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ /dev/null
@@ -1,525 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
-import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
-import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.api.records.LocalResource;
-import org.apache.hadoop.yarn.api.records.LocalResourceType;
-import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.api.resource.Configuration;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.common.SliderXmlConfKeys;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.common.tools.SliderUtils;
-import org.apache.slider.core.exceptions.BadCommandArgumentsException;
-import org.apache.slider.core.exceptions.NoSuchNodeException;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.core.registry.docstore.ConfigFormat;
-import org.apache.slider.core.registry.docstore.PublishedConfiguration;
-import org.apache.slider.core.registry.docstore.PublishedConfigurationOutputter;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.apache.slider.server.appmaster.state.StateAccessForProviders;
-import org.apache.slider.server.services.yarnregistry.YarnRegistryViewForProviders;
-import org.slf4j.Logger;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.regex.Pattern;
-
-import static org.apache.slider.api.ServiceApiConstants.*;
-import static org.apache.slider.util.ServiceApiUtil.$;
-
-/**
- * This is a factoring out of methods handy for providers. It's bonded to a log
- * at construction time.
- */
-public class ProviderUtils implements RoleKeys, SliderKeys {
-
- protected final Logger log;
-
- /**
- * Create an instance
- * @param log log directory to use -usually the provider
- */
-
- public ProviderUtils(Logger log) {
- this.log = log;
- }
-
- /**
- * Add oneself to the classpath. This does not work
- * on minicluster test runs where the JAR is not built up.
- * @param providerResources map of provider resources to add these entries to
- * @param providerClass provider to add
- * @param jarName name of the jar to use
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libdir relative directory to place resources
- * @param miniClusterTestRun true if minicluster is being used
- * @return true if the class was found in a JAR
- *
- * @throws FileNotFoundException if the JAR was not found and this is NOT
- * a mini cluster test run
- * @throws IOException IO problems
- * @throws SliderException any Slider problem
- */
- public static boolean addProviderJar(
- Map<String, LocalResource> providerResources,
- Class providerClass,
- String jarName,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libdir,
- boolean miniClusterTestRun) throws
- IOException,
- SliderException {
- try {
- SliderUtils.putJar(providerResources,
- sliderFileSystem,
- providerClass,
- tempPath,
- libdir,
- jarName);
- return true;
- } catch (FileNotFoundException e) {
- if (miniClusterTestRun) {
- return false;
- } else {
- throw e;
- }
- }
- }
-
- /**
- * Loads all dependency jars from the default path.
- * @param providerResources map of provider resources to add these entries to
- * @param sliderFileSystem target filesystem
- * @param tempPath path in the cluster FS for temp files
- * @param libDir relative directory to place resources
- * @param libLocalSrcDir explicitly supplied local libs dir
- * @throws IOException trouble copying to HDFS
- * @throws SliderException trouble copying to HDFS
- */
- public static void addAllDependencyJars(
- Map<String, LocalResource> providerResources,
- SliderFileSystem sliderFileSystem,
- Path tempPath,
- String libDir,
- String libLocalSrcDir)
- throws IOException, SliderException {
- if (SliderUtils.isSet(libLocalSrcDir)) {
- File file = new File(libLocalSrcDir);
- if (!file.exists() || !file.isDirectory()) {
- throw new BadCommandArgumentsException(
- "Supplied lib src dir %s is not valid", libLocalSrcDir);
- }
- }
- SliderUtils.putAllJars(providerResources, sliderFileSystem, tempPath,
- libDir, libLocalSrcDir);
- }
-
- public static String substituteStrWithTokens(String content,
- Map<String, String> tokensForSubstitution) {
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- content =
- content.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- return content;
- }
-
- // configs will be substituted by corresponding env in tokenMap
- public void substituteMapWithTokens(Map<String, String> configs,
- Map<String, String> tokenMap) {
- for (Map.Entry<String, String> entry : configs.entrySet()) {
- String value = entry.getValue();
- if (tokenMap != null) {
- for (Map.Entry<String, String> token : tokenMap.entrySet()) {
- value =
- value.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- }
- }
- entry.setValue(value);
- }
- }
-
- /**
- * Get resource requirements from a String value. If value isn't specified,
- * use the default value. If value is greater than max, use the max value.
- * @param val string value
- * @param defVal default value
- * @param maxVal maximum value
- * @return int resource requirement
- */
- public int getRoleResourceRequirement(String val,
- int defVal,
- int maxVal) {
- if (val==null) {
- val = Integer.toString(defVal);
- }
- Integer intVal;
- if (ResourceKeys.YARN_RESOURCE_MAX.equals(val)) {
- intVal = maxVal;
- } else {
- intVal = Integer.decode(val);
- }
- return intVal;
- }
-
-
- /**
- * Localize the service keytabs for the application.
- * @param launcher container launcher
- * @param fileSystem file system
- * @throws IOException trouble uploading to HDFS
- */
- public void localizeServiceKeytabs(ContainerLauncher launcher,
- SliderFileSystem fileSystem, Application application) throws IOException {
-
- Configuration conf = application.getConfiguration();
- String keytabPathOnHost =
- conf.getProperty(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH);
- if (SliderUtils.isUnset(keytabPathOnHost)) {
- String amKeytabName =
- conf.getProperty(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME);
- String keytabDir =
- conf.getProperty(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR);
- // we need to localize the keytab files in the directory
- Path keytabDirPath = fileSystem.buildKeytabPath(keytabDir, null,
- application.getName());
- boolean serviceKeytabsDeployed = false;
- if (fileSystem.getFileSystem().exists(keytabDirPath)) {
- FileStatus[] keytabs = fileSystem.getFileSystem().listStatus(
- keytabDirPath);
- LocalResource keytabRes;
- for (FileStatus keytab : keytabs) {
- if (!amKeytabName.equals(keytab.getPath().getName())
- && keytab.getPath().getName().endsWith(".keytab")) {
- serviceKeytabsDeployed = true;
- log.info("Localizing keytab {}", keytab.getPath().getName());
- keytabRes = fileSystem.createAmResource(keytab.getPath(),
- LocalResourceType.FILE);
- launcher.addLocalResource(KEYTAB_DIR + "/" +
- keytab.getPath().getName(),
- keytabRes);
- }
- }
- }
- if (!serviceKeytabsDeployed) {
- log.warn("No service keytabs for the application have been localized. "
- + "If the application requires keytabs for secure operation, "
- + "please ensure that the required keytabs have been uploaded "
- + "to the folder {}", keytabDirPath);
- }
- }
- }
-
- // 1. Create all config files for a component on hdfs for localization
- // 2. Add the config file to localResource
- public synchronized void createConfigFileAndAddLocalResource(
- ContainerLauncher launcher, SliderFileSystem fs, Component component,
- Map<String, String> tokensForSubstitution, RoleInstance roleInstance,
- StateAccessForProviders appState) throws IOException {
- Path compDir =
- new Path(new Path(fs.getAppDir(), "components"), component.getName());
- Path compInstanceDir =
- new Path(compDir, roleInstance.getCompInstanceName());
- if (!fs.getFileSystem().exists(compInstanceDir)) {
- fs.getFileSystem().mkdirs(compInstanceDir,
- new FsPermission(FsAction.ALL, FsAction.NONE, FsAction.NONE));
- roleInstance.compInstanceDir = compInstanceDir;
- log.info("Creating component instance dir: " + compInstanceDir);
- } else {
- log.info("Component instance conf dir already exists: " + compInstanceDir);
- }
-
- log.info("Tokens substitution for component: " + roleInstance
- .getCompInstanceName() + System.lineSeparator()
- + tokensForSubstitution);
-
- for (ConfigFile originalFile : component.getConfiguration().getFiles()) {
- ConfigFile configFile = originalFile.copy();
- String fileName = new Path(configFile.getDestFile()).getName();
-
- // substitute file name
- for (Map.Entry<String, String> token : tokensForSubstitution.entrySet()) {
- configFile.setDestFile(configFile.getDestFile()
- .replaceAll(Pattern.quote(token.getKey()), token.getValue()));
- }
-
- Path remoteFile = new Path(compInstanceDir, fileName);
- if (!fs.getFileSystem().exists(remoteFile)) {
- log.info("Saving config file on hdfs for component " + roleInstance
- .getCompInstanceName() + ": " + configFile);
-
- if (configFile.getSrcFile() != null) {
- // Load config file template
- switch (configFile.getType()) {
- case HADOOP_XML:
- // Hadoop_xml_template
- resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, appState);
- break;
- case TEMPLATE:
- // plain-template
- resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, appState);
- break;
- default:
- log.info("Not supporting loading src_file for " + configFile);
- break;
- }
- } else {
- // non-template
- resolveNonTemplateConfigsAndSaveOnHdfs(fs, tokensForSubstitution,
- roleInstance, configFile, fileName, remoteFile);
- }
- }
-
- // Add resource for localization
- LocalResource configResource =
- fs.createAmResource(remoteFile, LocalResourceType.FILE);
- File destFile = new File(configFile.getDestFile());
- String symlink = APP_CONF_DIR + "/" + fileName;
- if (destFile.isAbsolute()) {
- launcher.addLocalResource(symlink, configResource,
- configFile.getDestFile());
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile() + ", dest mount path: "
- + configFile.getDestFile());
- } else {
- launcher.addLocalResource(symlink, configResource);
- log.info("Add config file for localization: " + symlink + " -> "
- + configResource.getResource().getFile());
- }
- }
- }
-
- private void resolveNonTemplateConfigsAndSaveOnHdfs(SliderFileSystem fs,
- Map<String, String> tokensForSubstitution, RoleInstance roleInstance,
- ConfigFile configFile, String fileName, Path remoteFile)
- throws IOException {
- // substitute non-template configs
- substituteMapWithTokens(configFile.getProps(), tokensForSubstitution);
-
- // write configs onto hdfs
- PublishedConfiguration publishedConfiguration =
- new PublishedConfiguration(fileName,
- configFile.getProps().entrySet());
- if (!fs.getFileSystem().exists(remoteFile)) {
- PublishedConfigurationOutputter configurationOutputter =
- PublishedConfigurationOutputter.createOutputter(
- ConfigFormat.resolve(configFile.getType().toString()),
- publishedConfiguration);
- try (FSDataOutputStream os = fs.getFileSystem().create(remoteFile)) {
- configurationOutputter.save(os);
- os.flush();
- }
- } else {
- log.info("Component instance = " + roleInstance.getCompInstanceName()
- + ", config file already exists: " + remoteFile);
- }
- }
-
- // 1. substitute config template - only handle hadoop_xml format
- // 2. save on hdfs
- @SuppressWarnings("unchecked")
- private void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, StateAccessForProviders appState) throws IOException {
- Map<String, String> conf;
- try {
- conf = (Map<String, String>) appState.getConfigFileCache()
- .get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // make a copy for substitution
- org.apache.hadoop.conf.Configuration confCopy =
- new org.apache.hadoop.conf.Configuration(false);
- for (Map.Entry<String, String> entry : conf.entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute properties
- for (Map.Entry<String, String> entry : configFile.getProps().entrySet()) {
- confCopy.set(entry.getKey(), entry.getValue());
- }
- // substitute env variables
- for (Map.Entry<String, String> entry : confCopy) {
- String val = entry.getValue();
- if (val != null) {
- for (Map.Entry<String, String> token : tokensForSubstitution
- .entrySet()) {
- val = val.replaceAll(Pattern.quote(token.getKey()), token.getValue());
- confCopy.set(entry.getKey(), val);
- }
- }
- }
- // save on hdfs
- try (OutputStream output = fs.create(remoteFile)) {
- confCopy.writeXml(output);
- log.info("Reading config from: " + configFile.getSrcFile()
- + ", writing to: " + remoteFile);
- }
- }
-
- // 1) read the template as a string
- // 2) do token substitution
- // 3) save on hdfs
- private void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
- Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, StateAccessForProviders appState) {
- String content;
- try {
- content = (String) appState.getConfigFileCache().get(configFile);
- } catch (ExecutionException e) {
- log.info("Failed to load config file: " + configFile, e);
- return;
- }
- // substitute tokens
- content = substituteStrWithTokens(content, tokensForSubstitution);
-
- try (OutputStream output = fs.create(remoteFile)) {
- org.apache.commons.io.IOUtils.write(content, output);
- } catch (IOException e) {
- log.info("Failed to create " + remoteFile);
- }
- }
-
- /**
- * Get initial component token map to be substituted into config values.
- * @param roleInstance role instance
- * @return tokens to replace
- */
- public Map<String, String> initCompTokensForSubstitute(
- RoleInstance roleInstance) {
- Map<String, String> tokens = new HashMap<>();
- tokens.put(COMPONENT_NAME, roleInstance.role);
- tokens.put(COMPONENT_NAME_LC, roleInstance.role.toLowerCase());
- tokens.put(COMPONENT_INSTANCE_NAME, roleInstance.getCompInstanceName());
- tokens.put(CONTAINER_ID, roleInstance.getContainerId().toString());
- tokens.put(COMPONENT_ID, String.valueOf(roleInstance.componentId));
- return tokens;
- }
-
- /**
- * Add ROLE_HOST tokens for substitution into config values.
- * @param tokens existing tokens
- * @param amState access to AM state
- */
- public void addComponentHostTokens(Map<String, String> tokens,
- StateAccessForProviders amState) {
- if (amState == null) {
- return;
- }
- for (Map.Entry<String, Map<String, ClusterNode>> entry :
- amState.getRoleClusterNodeMapping().entrySet()) {
- String tokenName = entry.getKey().toUpperCase(Locale.ENGLISH) + "_HOST";
- String hosts = StringUtils .join(",",
- getHostsList(entry.getValue().values(), true));
- tokens.put($(tokenName), hosts);
- }
- }
-
- /**
- * Return a list of hosts based on current ClusterNodes.
- * @param values cluster nodes
- * @param hostOnly whether host or host/server name will be added to list
- * @return list of hosts
- */
- public Iterable<String> getHostsList(Collection<ClusterNode> values,
- boolean hostOnly) {
- List<String> hosts = new ArrayList<>();
- for (ClusterNode cn : values) {
- hosts.add(hostOnly ? cn.host : cn.host + "/" + cn.name);
- }
- return hosts;
- }
-
- /**
- * Update ServiceRecord in Registry with IP and hostname.
- * @param amState access to AM state
- * @param yarnRegistry acces to YARN registry
- * @param containerId container ID
- * @param roleName component name
- * @param ip list of IPs
- * @param hostname hostname
- */
- public void updateServiceRecord(StateAccessForProviders amState,
- YarnRegistryViewForProviders yarnRegistry,
- String containerId, String roleName, List<String> ip, String hostname)
- throws IOException {
- try {
- RoleInstance role = null;
- if(ip != null && !ip.isEmpty()){
- role = amState.getOwnedContainer(containerId);
- role.ip = ip.get(0);
- }
- if(hostname != null && !hostname.isEmpty()){
- role = amState.getOwnedContainer(containerId);
- role.hostname = hostname;
- }
- if (role != null) {
- // create and publish updated service record (including hostname & ip)
- ServiceRecord record = new ServiceRecord();
- record.set(YarnRegistryAttributes.YARN_ID, containerId);
- record.description = role.getCompInstanceName();
- record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
- PersistencePolicies.CONTAINER);
- // TODO: use constants from YarnRegistryAttributes
- if (role.ip != null) {
- record.set("yarn:ip", role.ip);
- }
- if (role.hostname != null) {
- record.set("yarn:hostname", role.hostname);
- }
- yarnRegistry.putComponent(
- RegistryPathUtils.encodeYarnID(containerId), record);
- }
- } catch (NoSuchNodeException e) {
- // ignore - there is nothing to do if we don't find a container
- log.warn("Owned container {} not found - {}", containerId, e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
deleted file mode 100644
index 5ecc374..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/SliderProviderFactory.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers;
-
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.core.exceptions.SliderException;
-import org.apache.slider.providers.docker.DockerProviderFactory;
-import org.apache.slider.providers.tarball.TarballProviderFactory;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-/**
- * Base class for factories.
- */
-public abstract class SliderProviderFactory {
- protected static final Logger LOG =
- LoggerFactory.getLogger(SliderProviderFactory.class);
-
- protected SliderProviderFactory() {}
-
- public abstract AbstractClientProvider createClientProvider();
-
- public abstract ProviderService createServerProvider();
-
- public static synchronized ProviderService getProviderService(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createServerProvider();
- }
-
- public static synchronized AbstractClientProvider getClientProvider(Artifact
- artifact) {
- return createSliderProviderFactory(artifact).createClientProvider();
- }
-
- /**
- * Create a provider for a specific application
- * @param artifact artifact
- * @return provider factory
- * @throws SliderException on any instantiation problem
- */
- public static synchronized SliderProviderFactory createSliderProviderFactory(
- Artifact artifact) {
- if (artifact == null || artifact.getType() == null) {
- LOG.debug("Loading service provider type default");
- return DefaultProviderFactory.getInstance();
- }
- LOG.debug("Loading service provider type {}", artifact.getType());
- switch (artifact.getType()) {
- // TODO add handling for custom types?
- // TODO handle application
- case DOCKER:
- return DockerProviderFactory.getInstance();
- case TARBALL:
- return TarballProviderFactory.getInstance();
- default:
- throw new IllegalArgumentException(String.format("Resolution error, " +
- "%s should not be passed to createSliderProviderFactory",
- artifact.getType()));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
deleted file mode 100644
index 4773327..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerClientProvider.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.slider.api.resource.Artifact;
-import org.apache.slider.api.resource.ConfigFile;
-import org.apache.slider.common.SliderKeys;
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.util.RestApiErrorMessages;
-
-import java.io.IOException;
-
-public class DockerClientProvider extends AbstractClientProvider
- implements SliderKeys {
-
- public DockerClientProvider() {
- super();
- }
-
- @Override
- public void validateArtifact(Artifact artifact, FileSystem fileSystem) {
- if (artifact == null) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_INVALID);
- }
- if (StringUtils.isEmpty(artifact.getId())) {
- throw new IllegalArgumentException(
- RestApiErrorMessages.ERROR_ARTIFACT_ID_INVALID);
- }
- }
-
- @Override
- protected void validateConfigFile(ConfigFile configFile, FileSystem
- fileSystem) throws IOException {
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
deleted file mode 100644
index 7413ebb..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerKeys.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-public interface DockerKeys {
- String PROVIDER_DOCKER = "docker";
- String DOCKER_PREFIX = "docker.";
- String DOCKER_IMAGE = DOCKER_PREFIX + "image";
- String DOCKER_NETWORK = DOCKER_PREFIX + "network";
- String DOCKER_USE_PRIVILEGED = DOCKER_PREFIX + "usePrivileged";
- String DOCKER_START_COMMAND = DOCKER_PREFIX + "startCommand";
-
- String DEFAULT_DOCKER_NETWORK = "bridge";
- Boolean DEFAULT_DOCKER_USE_PRIVILEGED = false;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
deleted file mode 100644
index 6977e41..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderFactory.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.slider.providers.AbstractClientProvider;
-import org.apache.slider.providers.ProviderService;
-import org.apache.slider.providers.SliderProviderFactory;
-
-public class DockerProviderFactory extends SliderProviderFactory {
- private static final SliderProviderFactory FACTORY = new
- DockerProviderFactory();
-
- private DockerProviderFactory() {
- }
-
- private static class Client {
- static final AbstractClientProvider PROVIDER = new DockerClientProvider();
- }
-
- private static class Server {
- static final ProviderService PROVIDER = new DockerProviderService();
- }
-
- @Override
- public AbstractClientProvider createClientProvider() {
- return Client.PROVIDER;
- }
-
- @Override
- public ProviderService createServerProvider() {
- return Server.PROVIDER;
- }
-
- public static SliderProviderFactory getInstance() {
- return FACTORY;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/164c0c4c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
deleted file mode 100644
index 73783af..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.slider.providers.docker;
-
-import org.apache.hadoop.registry.client.api.RegistryConstants;
-import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.slider.api.resource.Application;
-import org.apache.slider.api.resource.Component;
-import org.apache.slider.common.tools.SliderFileSystem;
-import org.apache.slider.core.launch.ContainerLauncher;
-import org.apache.slider.providers.AbstractProviderService;
-import org.apache.slider.server.appmaster.state.RoleInstance;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.text.MessageFormat;
-
-public class DockerProviderService extends AbstractProviderService
- implements DockerKeys {
-
- protected static final Logger log =
- LoggerFactory.getLogger(DockerProviderService.class);
-
- protected DockerProviderService() {
- super(DockerProviderService.class.getSimpleName());
- }
-
- public void processArtifact(ContainerLauncher launcher, Application
- application, RoleInstance roleInstance, SliderFileSystem fileSystem)
- throws IOException {
- Component component = roleInstance.providerRole.component;
- launcher.setYarnDockerMode(true);
- launcher.setDockerImage(component.getArtifact().getId());
- launcher.setDockerNetwork(component.getConfiguration()
- .getProperty(DOCKER_NETWORK, DEFAULT_DOCKER_NETWORK));
- String domain = getConfig().get(RegistryConstants.KEY_DNS_DOMAIN);
- String hostname;
- if (domain == null || domain.isEmpty()) {
- hostname = MessageFormat.format("{0}.{1}.{2}", roleInstance
- .getCompInstanceName(), application.getName(), RegistryUtils
- .currentUser());
- } else {
- hostname = MessageFormat.format("{0}.{1}.{2}.{3}", roleInstance
- .getCompInstanceName(), application.getName(), RegistryUtils
- .currentUser(), domain);
- }
- launcher.setDockerHostname(hostname);
- launcher.setRunPrivilegedContainer(component.getRunPrivilegedContainer());
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org