You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2017/07/21 18:39:46 UTC
[44/52] [abbrv] hadoop git commit: YARN-6545. Followup fix for
YARN-6405. Contributed by Jian He
YARN-6545. Followup fix for YARN-6405. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3147883
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3147883
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3147883
Branch: refs/heads/yarn-native-services
Commit: c3147883abbf9216ce4f48bd309ae6127b0c6b41
Parents: 636cbba
Author: Billie Rinaldi <bi...@apache.org>
Authored: Tue May 9 09:26:00 2017 -0700
Committer: Jian He <ji...@apache.org>
Committed: Fri Jul 21 11:38:14 2017 -0700
----------------------------------------------------------------------
.../apache/slider/api/ServiceApiConstants.java | 19 +++++++
.../slider/api/resource/Configuration.java | 26 +++-------
.../org/apache/slider/client/SliderClient.java | 26 +++++-----
.../slider/common/tools/CoreFileSystem.java | 47 ------------------
.../core/registry/docstore/ConfigFormat.java | 2 +-
.../apache/slider/core/zk/ZKIntegration.java | 2 +-
.../apache/slider/providers/ProviderRole.java | 14 ++----
.../apache/slider/providers/ProviderUtils.java | 43 ++++++----------
.../providers/docker/DockerProviderService.java | 8 +--
.../server/appmaster/RoleLaunchService.java | 4 +-
.../server/appmaster/SliderAppMaster.java | 1 +
.../slider/server/appmaster/state/AppState.java | 52 ++++++++++++++++----
.../appmaster/state/AppStateBindingInfo.java | 2 +
.../appmaster/state/ProviderAppState.java | 12 +++++
.../server/appmaster/state/RoleInstance.java | 8 +--
.../state/StateAccessForProviders.java | 12 +++++
.../appstate/TestMockAppStateRMOperations.java | 48 ++++++++++++++++++
.../appstate/TestMockAppStateUniqueNames.java | 2 +-
.../model/mock/BaseMockAppStateTest.java | 6 +--
.../runtime/DockerLinuxContainerRuntime.java | 2 +
20 files changed, 194 insertions(+), 142 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
index 5f76f19..da87e3a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/ServiceApiConstants.java
@@ -36,6 +36,10 @@ public interface ServiceApiConstants {
String SERVICE_NAME_LC = $("SERVICE_NAME.lc");
+ String USER = $("USER");
+
+ String DOMAIN = $("DOMAIN");
+
// Constants for component
String COMPONENT_NAME = $("COMPONENT_NAME");
@@ -47,4 +51,19 @@ public interface ServiceApiConstants {
String COMPONENT_ID = $("COMPONENT_ID");
String CONTAINER_ID = $("CONTAINER_ID");
+
+ // Constants for default cluster ZK
+ String CLUSTER_ZK_QUORUM = $("CLUSTER_ZK_QUORUM");
+
+ // URI for the default cluster fs
+ String CLUSTER_FS_URI = $("CLUSTER_FS_URI");
+
+ // the host component of the cluster fs UI
+ String CLUSTER_FS_HOST = $("CLUSTER_FS_HOST");
+
+ // Path in zookeeper for a specific service
+ String SERVICE_ZK_PATH = $("SERVICE_ZK_PATH");
+
+ // Constants for service specific hdfs dir
+ String SERVICE_HDFS_DIR = $("SERVICE_HDFS_DIR");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
index 0df586c..37d1a40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/api/resource/Configuration.java
@@ -105,10 +105,7 @@ public class Configuration implements Serializable {
}
public long getPropertyLong(String name, long defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = properties.get(name.trim());
+ String value = getProperty(name);
if (StringUtils.isEmpty(value)) {
return defaultValue;
}
@@ -116,10 +113,7 @@ public class Configuration implements Serializable {
}
public int getPropertyInt(String name, int defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = properties.get(name.trim());
+ String value = getProperty(name);
if (StringUtils.isEmpty(value)) {
return defaultValue;
}
@@ -127,10 +121,7 @@ public class Configuration implements Serializable {
}
public boolean getPropertyBool(String name, boolean defaultValue) {
- if (name == null) {
- return defaultValue;
- }
- String value = properties.get(name.trim());
+ String value = getProperty(name);
if (StringUtils.isEmpty(value)) {
return defaultValue;
}
@@ -138,10 +129,11 @@ public class Configuration implements Serializable {
}
public String getProperty(String name, String defaultValue) {
- if (name == null) {
+ String value = getProperty(name);
+ if (StringUtils.isEmpty(value)) {
return defaultValue;
}
- return properties.get(name.trim());
+ return value;
}
public void setProperty(String name, String value) {
@@ -149,16 +141,10 @@ public class Configuration implements Serializable {
}
public String getProperty(String name) {
- if (name == null) {
- return null;
- }
return properties.get(name.trim());
}
public String getEnv(String name) {
- if (name == null) {
- return null;
- }
return env.get(name.trim());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 7241374..83b4841 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1112,18 +1112,20 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
"not a directory", folder);
}
- for (File f : files) {
- srcFile = new Path(f.toURI());
-
- Path fileInFs = new Path(pkgPath, srcFile.getName());
- log.info("Installing file {} at {} and overwrite is {}.",
- srcFile, fileInFs, resourceInfo.overwrite);
- require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite),
- "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
-
- sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs);
- sfs.setPermission(fileInFs,
- new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ if (files != null) {
+ for (File f : files) {
+ srcFile = new Path(f.toURI());
+
+ Path fileInFs = new Path(pkgPath, srcFile.getName());
+ log.info("Installing file {} at {} and overwrite is {}.",
+ srcFile, fileInFs, resourceInfo.overwrite);
+ require(!(sfs.exists(fileInFs) && !resourceInfo.overwrite),
+ "File exists at %s. Use --overwrite to overwrite.", fileInFs.toUri());
+
+ sfs.copyFromLocalFile(false, resourceInfo.overwrite, srcFile, fileInFs);
+ sfs.setPermission(fileInFs,
+ new FsPermission(FsAction.READ_WRITE, FsAction.NONE, FsAction.NONE));
+ }
}
return EXIT_SUCCESS;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index 5f5e611..0c249d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -597,53 +597,6 @@ public class CoreFileSystem {
providerResources.put(SliderKeys.SLIDER_DEPENDENCY_LOCALIZED_DIR_LINK, lc);
}
- /**
- * Copy local file(s) to destination HDFS directory. If {@code localPath} is a
- * local directory then all files matching the {@code filenameFilter}
- * (optional) are copied, otherwise {@code filenameFilter} is ignored.
- *
- * @param localPath
- * a local file or directory path
- * @param filenameFilter
- * if {@code localPath} is a directory then filenameFilter is used as
- * a filter (if specified)
- * @param destDir
- * the destination HDFS directory where the file(s) should be copied
- * @param fp
- * file permissions of all the directories and files that will be
- * created in this api
- * @throws IOException
- */
- public void copyLocalFilesToHdfs(File localPath,
- FilenameFilter filenameFilter, Path destDir, FsPermission fp)
- throws IOException {
- if (localPath == null || destDir == null) {
- throw new IOException("Either localPath or destDir is null");
- }
- fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
- "000");
- fileSystem.mkdirs(destDir, fp);
- if (localPath.isDirectory()) {
- // copy all local files under localPath to destDir (honoring filename
- // filter if provided
- File[] localFiles = localPath.listFiles(filenameFilter);
- Path[] localFilePaths = new Path[localFiles.length];
- int i = 0;
- for (File localFile : localFiles) {
- localFilePaths[i++] = new Path(localFile.getPath());
- }
- log.info("Copying {} files from {} to {}", i, localPath.toURI(),
- destDir.toUri());
- fileSystem.copyFromLocalFile(false, true, localFilePaths, destDir);
- } else {
- log.info("Copying file {} to {}", localPath.toURI(), destDir.toUri());
- fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
- destDir);
- }
- // set permissions for all the files created in the destDir
- fileSystem.setPermission(destDir, fp);
- }
-
public void copyLocalFileToHdfs(File localPath,
Path destPath, FsPermission fp)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
index 723b975..081688b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/registry/docstore/ConfigFormat.java
@@ -25,7 +25,7 @@ public enum ConfigFormat {
JSON("json"),
PROPERTIES("properties"),
XML("xml"),
- HADOOP_XML("hadoop-xml"),
+ HADOOP_XML("hadoop_xml"),
ENV("env"),
TEMPLATE("template"),
YAML("yaml"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
index 4302530..519cd16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/core/zk/ZKIntegration.java
@@ -52,7 +52,7 @@ public class ZKIntegration implements Watcher, Closeable {
public static final String SVC_SLIDER = "/" + ZK_SERVICES + "/" + ZK_SLIDER;
public static final String SVC_SLIDER_USERS = SVC_SLIDER + "/" + ZK_USERS;
- public static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
+ private static final List<String> ZK_USERS_PATH_LIST = new ArrayList<String>();
static {
ZK_USERS_PATH_LIST.add(ZK_SERVICES);
ZK_USERS_PATH_LIST.add(ZK_SLIDER);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 9cc48e1..182e956 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -20,10 +20,8 @@ package org.apache.slider.providers;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.resource.Component;
-import org.apache.slider.server.appmaster.state.AppState;
+import org.apache.slider.server.appmaster.state.RoleInstance;
-import java.util.LinkedList;
-import java.util.List;
import java.util.Queue;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicLong;
@@ -44,8 +42,7 @@ public final class ProviderRole {
public final String labelExpression;
public final Component component;
public AtomicLong componentIdCounter = null;
- public AppState appState;
- public Queue<String> failedInstanceName = new ConcurrentLinkedQueue<String>();
+ public Queue<RoleInstance> failedInstances = new ConcurrentLinkedQueue<>();
public ProviderRole(String name, int id) {
this(name,
id,
@@ -78,7 +75,7 @@ public final class ProviderRole {
nodeFailureThreshold,
placementTimeoutSeconds,
labelExpression,
- new Component().name(name).numberOfContainers(0L), null);
+ new Component().name(name).numberOfContainers(0L));
}
/**
@@ -88,13 +85,13 @@ public final class ProviderRole {
* @param id ID. This becomes the YARN priority
* @param policy placement policy
* @param nodeFailureThreshold threshold for node failures (within a reset interval)
-* after which a node failure is considered an app failure
+ * after which a node failure is considered an app failure
* @param placementTimeoutSeconds for lax placement, timeout in seconds before
* @param labelExpression label expression for requests; may be null
*/
public ProviderRole(String name, String group, int id, int policy,
int nodeFailureThreshold, long placementTimeoutSeconds,
- String labelExpression, Component component, AppState state) {
+ String labelExpression, Component component) {
this.name = name;
if (group == null) {
this.group = name;
@@ -110,7 +107,6 @@ public final class ProviderRole {
if(component.getUniqueComponentSupport()) {
componentIdCounter = new AtomicLong(0);
}
- this.appState = state;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
index d384585..beeaa55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/ProviderUtils.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterNode;
-import org.apache.slider.api.OptionKeys;
import org.apache.slider.api.ResourceKeys;
import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.resource.Application;
@@ -59,7 +58,6 @@ import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.OutputStream;
-import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -271,8 +269,8 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
// 2. Add the config file to localResource
public synchronized void createConfigFileAndAddLocalResource(
ContainerLauncher launcher, SliderFileSystem fs, Component component,
- Map<String, String> tokensForSubstitution, RoleInstance roleInstance)
- throws IOException {
+ Map<String, String> tokensForSubstitution, RoleInstance roleInstance,
+ StateAccessForProviders appState) throws IOException {
Path compDir =
new Path(new Path(fs.getAppDir(), "components"), component.getName());
Path compInstanceDir =
@@ -315,12 +313,12 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
case HADOOP_XML:
// Hadoop_xml_template
resolveHadoopXmlTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, roleInstance);
+ tokensForSubstitution, configFile, remoteFile, appState);
break;
case TEMPLATE:
// plain-template
resolvePlainTemplateAndSaveOnHdfs(fs.getFileSystem(),
- tokensForSubstitution, configFile, remoteFile, roleInstance);
+ tokensForSubstitution, configFile, remoteFile, appState);
break;
default:
log.info("Not supporting loading src_file for " + configFile);
@@ -383,11 +381,11 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
@SuppressWarnings("unchecked")
private void resolveHadoopXmlTemplateAndSaveOnHdfs(FileSystem fs,
Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, RoleInstance roleInstance) throws IOException {
+ Path remoteFile, StateAccessForProviders appState) throws IOException {
Map<String, String> conf;
try {
- conf = (Map<String, String>) roleInstance.providerRole.
- appState.configFileCache.get(configFile);
+ conf = (Map<String, String>) appState.getConfigFileCache()
+ .get(configFile);
} catch (ExecutionException e) {
log.info("Failed to load config file: " + configFile, e);
return;
@@ -426,17 +424,16 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
// 3) save on hdfs
private void resolvePlainTemplateAndSaveOnHdfs(FileSystem fs,
Map<String, String> tokensForSubstitution, ConfigFile configFile,
- Path remoteFile, RoleInstance roleInstance) {
+ Path remoteFile, StateAccessForProviders appState) {
String content;
try {
- content = (String) roleInstance.providerRole.appState.configFileCache
- .get(configFile);
+ content = (String) appState.getConfigFileCache().get(configFile);
} catch (ExecutionException e) {
log.info("Failed to load config file: " + configFile, e);
return;
}
// substitute tokens
- substituteStrWithTokens(content, tokensForSubstitution);
+ content = substituteStrWithTokens(content, tokensForSubstitution);
try (OutputStream output = fs.create(remoteFile)) {
org.apache.commons.io.IOUtils.write(content, output);
@@ -446,25 +443,13 @@ public class ProviderUtils implements RoleKeys, SliderKeys {
}
/**
- * Get initial token map to be substituted into config values.
- * @param appConf app configurations
- * @param clusterName app name
+ * Get initial component token map to be substituted into config values.
+ * @param roleInstance role instance
* @return tokens to replace
*/
- public Map<String, String> getStandardTokenMap(Configuration appConf,
- RoleInstance roleInstance, String clusterName) {
-
+ public Map<String, String> initCompTokensForSubstitute(
+ RoleInstance roleInstance) {
Map<String, String> tokens = new HashMap<>();
-
- String nnuri = appConf.getProperty("fs.defaultFS");
- if (nnuri != null && !nnuri.isEmpty()) {
- tokens.put("${NN_URI}", nnuri);
- tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
- }
- tokens.put("${ZK_HOST}", appConf.getProperty(OptionKeys.ZOOKEEPER_HOSTS));
- tokens.put("${DEFAULT_ZK_PATH}", appConf.getProperty(OptionKeys.ZOOKEEPER_PATH));
- tokens.put(SERVICE_NAME_LC, clusterName.toLowerCase());
- tokens.put(SERVICE_NAME, clusterName);
tokens.put(COMPONENT_NAME, roleInstance.role);
tokens.put(COMPONENT_NAME_LC, roleInstance.role.toLowerCase());
tokens.put(COMPONENT_INSTANCE_NAME, roleInstance.getCompInstanceName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
index 482bb27..12c2b04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/providers/docker/DockerProviderService.java
@@ -88,10 +88,10 @@ public class DockerProviderService extends AbstractService
// Generate tokens (key-value pair) for config substitution.
// Get pre-defined tokens
+ Map<String, String> globalTokens = amState.getGlobalSubstitutionTokens();
Map<String, String> tokensForSubstitution = providerUtils
- .getStandardTokenMap(application.getConfiguration(), roleInstance,
- application.getName());
-
+ .initCompTokensForSubstitute(roleInstance);
+ tokensForSubstitution.putAll(globalTokens);
// Set the environment variables in launcher
launcher.putEnv(SliderUtils
.buildEnvMap(component.getConfiguration(), tokensForSubstitution));
@@ -111,7 +111,7 @@ public class DockerProviderService extends AbstractService
// create config file on hdfs and add local resource
providerUtils.createConfigFileAndAddLocalResource(launcher, fileSystem,
- component, tokensForSubstitution, roleInstance);
+ component, tokensForSubstitution, roleInstance, amState);
// substitute launch command
String launchCommand = ProviderUtils
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index c53349f..7c096c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -150,10 +150,10 @@ public class RoleLaunchService
containerLauncher.setupUGI();
containerLauncher.putEnv(envVars);
- String failedInstance = role.failedInstanceName.poll();
+ RoleInstance failedInstance = role.failedInstances.poll();
RoleInstance instance;
if (failedInstance != null) {
- instance = new RoleInstance(container, role, failedInstance);
+ instance = new RoleInstance(container, failedInstance);
} else {
instance = new RoleInstance(container, role);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index ffa07fb..ae03b45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -819,6 +819,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
binding.releaseSelector = new MostRecentContainerReleaseSelector();
binding.nodeReports = nodeReports;
binding.application = application;
+ binding.serviceHdfsDir = fs.buildClusterDirPath(appName).toString();
appState.buildInstance(binding);
// build up environment variables that the AM wants set in every container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f7b4a8..1e1b377 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -24,11 +24,11 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.metrics2.lib.MutableGaugeInt;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
+import org.apache.slider.api.ServiceApiConstants;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.proto.Messages;
import org.apache.slider.api.proto.Messages.ComponentCountProto;
@@ -61,6 +62,7 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
+import org.apache.slider.core.zk.ZKIntegration;
import org.apache.slider.providers.PlacementPolicy;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.appmaster.management.MetricsAndMonitoring;
@@ -75,6 +77,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
@@ -89,7 +92,12 @@ import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import static org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.DEFAULT_REGISTRY_ZK_QUORUM;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_DNS_DOMAIN;
+import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
import static org.apache.slider.api.ResourceKeys.*;
+import static org.apache.slider.api.ServiceApiConstants.*;
import static org.apache.slider.api.StateValues.*;
import static org.apache.slider.api.resource.ApplicationState.STARTED;
@@ -193,14 +201,13 @@ public class AppState {
private int containerMinMemory;
private RoleHistory roleHistory;
- private Configuration publishedProviderConf;
private long startTimeThreshold;
private int failureThreshold = 10;
private int nodeFailureThreshold = 3;
private String logServerURL = "";
-
+ public Map<String, String> globalTokens = new HashMap<>();
/**
* Selector of containers to release; application wide.
*/
@@ -335,6 +342,7 @@ public class AppState {
DEFAULT_CONTAINER_FAILURE_THRESHOLD);
nodeFailureThreshold = conf.getPropertyInt(NODE_FAILURE_THRESHOLD,
DEFAULT_NODE_FAILURE_THRESHOLD);
+ initGlobalTokensForSubstitute(binding);
//build the initial component list
int priority = 1;
@@ -367,6 +375,34 @@ public class AppState {
createConfigFileCache(binding.fs);
}
+ private void initGlobalTokensForSubstitute(AppStateBindingInfo binding)
+ throws IOException {
+ // ZK
+ globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM,
+ binding.serviceConfig
+ .getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
+ String user = UserGroupInformation.getCurrentUser().getShortUserName();
+ globalTokens
+ .put(SERVICE_ZK_PATH, ZKIntegration.mkClusterPath(user, app.getName()));
+
+ globalTokens.put(ServiceApiConstants.USER, user);
+ String dnsDomain = binding.serviceConfig.getTrimmed(KEY_DNS_DOMAIN);
+ if (dnsDomain != null && !dnsDomain.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.DOMAIN, dnsDomain);
+ }
+ // HDFS
+ String clusterFs = binding.serviceConfig.getTrimmed(FS_DEFAULT_NAME_KEY);
+ if (clusterFs != null && !clusterFs.isEmpty()) {
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_URI, clusterFs);
+ globalTokens.put(ServiceApiConstants.CLUSTER_FS_HOST,
+ URI.create(clusterFs).getHost());
+ }
+ globalTokens.put(SERVICE_HDFS_DIR, binding.serviceHdfsDir);
+ // service name
+ globalTokens.put(SERVICE_NAME_LC, app.getName().toLowerCase());
+ globalTokens.put(SERVICE_NAME, app.getName());
+ }
+
private void createConfigFileCache(final FileSystem fileSystem) {
this.configFileCache =
CacheBuilder.newBuilder().expireAfterAccess(10, TimeUnit.MINUTES)
@@ -411,7 +447,7 @@ public class AppState {
DEF_YARN_LABEL_EXPRESSION);
ProviderRole newRole =
new ProviderRole(name, group, priority, (int)placementPolicy, threshold,
- placementTimeout, label, component, this);
+ placementTimeout, label, component);
buildRole(newRole, component);
log.info("Created a new role " + newRole);
return newRole;
@@ -1300,8 +1336,7 @@ public class AppState {
try {
RoleStatus roleStatus = lookupRoleStatus(roleInstance.roleId);
decRunningContainers(roleStatus);
- roleStatus.getProviderRole().failedInstanceName
- .offer(roleInstance.compInstanceName);
+ roleStatus.getProviderRole().failedInstances.offer(roleInstance);
boolean shortLived = isShortLived(roleInstance);
String message;
Container failedContainer = roleInstance.container;
@@ -1742,8 +1777,7 @@ public class AppState {
for (RoleInstance possible : finalCandidates) {
log.info("Targeting for release: {}", possible);
containerReleaseSubmitted(possible.container);
- role.getProviderRole().failedInstanceName
- .offer(possible.compInstanceName);
+ role.getProviderRole().failedInstances.offer(possible);
operations.add(new ContainerReleaseOperation(possible.getContainerId()));
}
}
@@ -1862,7 +1896,6 @@ public class AppState {
//get the role
final ContainerId cid = container.getId();
final RoleStatus role = lookupRoleStatus(container);
- decRequestedContainers(role);
//inc allocated count -this may need to be dropped in a moment,
// but us needed to update the logic below
@@ -1888,6 +1921,7 @@ public class AppState {
role.getComponentMetrics().surplusContainers.incr();
containersRunning.decr();
} else {
+ decRequestedContainers(role);
log.info("Assigning role {} to container" + " {}," + " on {}:{},",
roleName, cid, nodeId.getHost(), nodeId.getPort());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
index 2dfded8..ac9b8eb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.common.tools.CoreFileSystem;
import org.apache.slider.providers.ProviderRole;
import java.util.ArrayList;
@@ -45,6 +46,7 @@ public class AppStateBindingInfo {
public Path historyPath;
public List<Container> liveContainers = new ArrayList<>(0);
public ContainerReleaseSelector releaseSelector = new SimpleReleaseSelector();
+ public String serviceHdfsDir = "";
/** node reports off the RM. */
public List<NodeReport> nodeReports = new ArrayList<>(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
index 8046472..8fc08b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/ProviderAppState.java
@@ -18,11 +18,13 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.cache.LoadingCache;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
@@ -262,4 +264,14 @@ public class ProviderAppState implements StateAccessForProviders {
public RoleStatistics getRoleStatistics() {
return appState.getRoleStatistics();
}
+
+ @Override
+ public Map<String, String> getGlobalSubstitutionTokens() {
+ return appState.globalTokens;
+ }
+
+ @Override
+ public LoadingCache<ConfigFile, Object> getConfigFileCache() {
+ return appState.configFileCache;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 736dfd1..9ac26b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -128,11 +128,11 @@ public final class RoleInstance implements Cloneable {
this.providerRole = role;
}
- public RoleInstance(Container container, ProviderRole role,
- String compInstanceName) {
+ public RoleInstance(Container container, RoleInstance failedInstance) {
this(container);
- this.compInstanceName = compInstanceName;
- this.providerRole = role;
+ this.componentId = failedInstance.componentId;
+ this.compInstanceName = failedInstance.compInstanceName;
+ this.providerRole = failedInstance.providerRole;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
index 5bc6dce..90221cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/main/java/org/apache/slider/server/appmaster/state/StateAccessForProviders.java
@@ -18,12 +18,14 @@
package org.apache.slider.server.appmaster.state;
+import com.google.common.cache.LoadingCache;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.resource.Application;
+import org.apache.slider.api.resource.ConfigFile;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
import org.apache.slider.api.types.NodeInformation;
@@ -260,4 +262,14 @@ public interface StateAccessForProviders {
* @return role statistics
*/
RoleStatistics getRoleStatistics();
+
+ /**
+ * Get global substitution tokens.
+ */
+ Map<String, String> getGlobalSubstitutionTokens();
+
+ /**
+ * Get config file cache.
+ */
+ LoadingCache<ConfigFile, Object> getConfigFileCache();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
index 2d87be6..363c551 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateRMOperations.java
@@ -379,4 +379,52 @@ public class TestMockAppStateRMOperations extends BaseMockAppStateTest
assertNull(ri3);
}
+ @Test
+ public void testDoubleAllocate() throws Throwable {
+ getRole0Status().setDesired(1);
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes();
+ ContainerRequestOperation operation = (ContainerRequestOperation)ops.get(0);
+ AMRMClient.ContainerRequest request = operation.getRequest();
+ Container cont = engine.allocateContainer(request);
+ List<Container> allocated = new ArrayList<>();
+ allocated.add(cont);
+ List<ContainerAssignment> assignments = new ArrayList<>();
+ List<AbstractRMOperation> operations = new ArrayList<>();
+ assertEquals(0L, getRole0Status().getRunning());
+ assertEquals(1L, getRole0Status().getRequested());
+ appState.onContainersAllocated(allocated, assignments, operations);
+
+ assertListLength(ops, 1);
+ assertListLength(assignments, 1);
+ ContainerAssignment assigned = assignments.get(0);
+ Container target = assigned.container;
+ assertEquals(target.getId(), cont.getId());
+ int roleId = assigned.role.getPriority();
+ assertEquals(roleId, extractRole(request.getPriority()));
+ assertEquals(assigned.role.getName(), ROLE0);
+ RoleInstance ri = roleInstance(assigned);
+ //tell the app it arrived
+ appState.containerStartSubmitted(target, ri);
+ appState.innerOnNodeManagerContainerStarted(target.getId());
+ assertEquals(1L, getRole0Status().getRunning());
+ assertEquals(0L, getRole0Status().getRequested());
+
+ // now get an extra allocation that should be released
+ cont = engine.allocateContainer(request);
+ allocated = new ArrayList<>();
+ allocated.add(cont);
+ assignments = new ArrayList<>();
+ operations = new ArrayList<>();
+ appState.onContainersAllocated(allocated, assignments, operations);
+
+ assertListLength(operations, 1);
+ assertTrue(operations.get(0) instanceof ContainerReleaseOperation);
+ ContainerReleaseOperation release = (ContainerReleaseOperation)
+ operations.get(0);
+ assertEquals(release.getContainerId(), cont.getId());
+
+ assertEquals(1L, getRole0Status().getRunning());
+ assertEquals(0L, getRole0Status().getRequested());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
index 54ffe17..b7e967f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.java
@@ -99,6 +99,7 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assertEquals(roles[i], entry.getKey());
RoleInstance instance = entry.getValue();
assertEquals(roles[i], instance.compInstanceName);
+ assertEquals(i, instance.componentId);
assertEquals(group, instance.role);
assertEquals(group, instance.providerRole.name);
assertEquals(group, instance.providerRole.group);
@@ -129,7 +130,6 @@ public class TestMockAppStateUniqueNames extends BaseMockAppStateTest
createAndStartNodes();
instances = appState.cloneOwnedContainerList();
verifyInstances(instances, "group1", "group10", "group11", "group12");
- // fails because the names continue at N+1, with group12, group13, group14
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
index 4352959..69abccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-slider/hadoop-yarn-slider-core/src/test/java/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.java
@@ -176,11 +176,11 @@ public abstract class BaseMockAppStateTest extends SliderTestBase implements
*/
public RoleInstance roleInstance(ContainerAssignment assigned) {
Container target = assigned.container;
- String failedInstance =
- assigned.role.getProviderRole().failedInstanceName.poll();
+ RoleInstance failedInstance =
+ assigned.role.getProviderRole().failedInstances.poll();
RoleInstance ri;
if (failedInstance != null) {
- ri = new RoleInstance(target, assigned.role.getProviderRole(), failedInstance);
+ ri = new RoleInstance(target, failedInstance);
} else {
ri = new RoleInstance(target, assigned.role.getProviderRole());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3147883/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index e058d6e..57dadb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -627,6 +627,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOG.error("Incorrect format for ip and host");
return null;
}
+ // strip off quotes if any
+ output = output.replaceAll("['\"]", "");
String ips = output.substring(0, index).trim();
String host = output.substring(index+1).trim();
String[] ipAndHost = new String[2];
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org