You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2018/01/24 22:36:08 UTC
[39/50] [abbrv] hadoop git commit: YARN-7766. Introduce a new config
property for YARN Service dependency tarball location. Contributed by Gour
Saha
YARN-7766. Introduce a new config property for YARN Service dependency tarball location. Contributed by Gour Saha
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a72cdcc4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a72cdcc4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a72cdcc4
Branch: refs/heads/HDFS-7240
Commit: a72cdcc47a1b3a4fcadbc9e12bd1bae2bbf04d83
Parents: f63d13f
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 23 10:53:27 2018 -0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Jan 23 10:53:27 2018 -0800
----------------------------------------------------------------------
.../yarn/service/client/ServiceClient.java | 35 +++++++++++++++-----
.../yarn/service/conf/YarnServiceConf.java | 6 ++++
.../tarball/TarballProviderService.java | 4 +--
.../yarn/service/utils/CoreFileSystem.java | 32 ++++++++----------
.../hadoop/yarn/client/api/AppAdminClient.java | 12 ++++---
.../hadoop/yarn/client/cli/ApplicationCLI.java | 13 +++++---
.../hadoop/yarn/client/cli/TestYarnCLI.java | 5 ++-
7 files changed, 68 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index c224089..b8034b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.api.records.*;
import org.apache.hadoop.yarn.client.api.AppAdminClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.client.cli.ApplicationCLI;
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -718,14 +719,20 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
libPath, "lib", false);
Path dependencyLibTarGzip = fs.getDependencyTarGzip();
if (fs.isFile(dependencyLibTarGzip)) {
- LOG.debug("Loading lib tar from " + fs.getFileSystem().getScheme() + ":/"
- + dependencyLibTarGzip);
+ LOG.info("Loading lib tar from " + dependencyLibTarGzip);
fs.submitTarGzipAndUpdate(localResources);
} else {
+ if (dependencyLibTarGzip != null) {
+ LOG.warn("Property {} has a value {}, but is not a valid file",
+ YarnServiceConf.DEPENDENCY_TARBALL_PATH, dependencyLibTarGzip);
+ }
String[] libs = ServiceUtils.getLibDirs();
- LOG.info("Uploading all dependency jars to HDFS. For faster submission of" +
- " apps, pre-upload dependency jars to HDFS "
- + "using command: yarn app -enableFastLaunch");
+ LOG.info("Uploading all dependency jars to HDFS. For faster submission of"
+ + " apps, set config property {} to the dependency tarball location."
+ + " Dependency tarball can be uploaded to any HDFS path directly"
+ + " or by using command: yarn app -{} [<Destination Folder>]",
+ YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+ ApplicationCLI.ENABLE_FAST_LAUNCH);
for (String libDirProp : libs) {
ProviderUtils.addAllDependencyJars(localResources, fs, libPath, "lib",
libDirProp);
@@ -988,16 +995,23 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return this.yarnClient;
}
- public int enableFastLaunch() throws IOException, YarnException {
- return actionDependency(true);
+ public int enableFastLaunch(String destinationFolder)
+ throws IOException, YarnException {
+ return actionDependency(destinationFolder, true);
}
- public int actionDependency(boolean overwrite)
+ public int actionDependency(String destinationFolder, boolean overwrite)
throws IOException, YarnException {
String currentUser = RegistryUtils.currentUser();
LOG.info("Running command as user {}", currentUser);
- Path dependencyLibTarGzip = fs.getDependencyTarGzip();
+ if (destinationFolder == null) {
+ destinationFolder = String.format(YarnServiceConstants.DEPENDENCY_DIR,
+ VersionInfo.getVersion());
+ }
+ Path dependencyLibTarGzip = new Path(destinationFolder,
+ YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
+ + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
// Check if dependency has already been uploaded, in which case log
// appropriately and exit success (unless overwrite has been requested)
@@ -1019,6 +1033,9 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
LOG.info("Version Info: " + VersionInfo.getBuildVersion());
fs.copyLocalFileToHdfs(tempLibTarGzipFile, dependencyLibTarGzip,
new FsPermission(YarnServiceConstants.DEPENDENCY_DIR_PERMISSIONS));
+ LOG.info("To let apps use this tarball, in yarn-site set config property "
+ + "{} to {}", YarnServiceConf.DEPENDENCY_TARBALL_PATH,
+ dependencyLibTarGzip);
return EXIT_SUCCESS;
} else {
return EXIT_FALSE;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 22926e2..21470d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -94,6 +94,12 @@ public class YarnServiceConf {
public static final int DEFAULT_CONTAINER_RECOVERY_TIMEOUT_MS = 120000;
/**
+ * The dependency tarball file location.
+ */
+ public static final String DEPENDENCY_TARBALL_PATH = YARN_SERVICE_PREFIX
+ + "framework.path";
+
+ /**
* Get long value for the property. First get from the userConf, if not
* present, get from systemConf.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
index 9f29c8b..7f29d1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/tarball/TarballProviderService.java
@@ -38,9 +38,9 @@ public class TarballProviderService extends AbstractProviderService {
Path artifact = new Path(instance.getCompSpec().getArtifact().getId());
if (!fileSystem.isFile(artifact)) {
throw new IOException(
- "Package doesn't exist as a resource: " + artifact.toString());
+ "Package doesn't exist as a resource: " + artifact);
}
- log.info("Adding resource {}", artifact.toString());
+ log.info("Adding resource {}", artifact);
LocalResourceType type = LocalResourceType.ARCHIVE;
LocalResource packageResource = fileSystem.createAmResource(artifact, type);
launcher.addLocalResource(APP_LIB_DIR, packageResource);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 6b9d4d5..284825e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -295,6 +295,9 @@ public class CoreFileSystem {
* reasons including if file check throws IOException
*/
public boolean isFile(Path path) {
+ if (path == null) {
+ return false;
+ }
boolean isFile = false;
try {
FileStatus status = fileSystem.getFileStatus(path);
@@ -321,26 +324,18 @@ public class CoreFileSystem {
}
/**
- * Get slider dependency parent dir in HDFS
+ * Get service dependency absolute filepath in HDFS used for application
+ * submission.
*
- * @return the parent dir path of slider.tar.gz in HDFS
- */
- public Path getDependencyPath() {
- String parentDir = YarnServiceConstants.DEPENDENCY_DIR;
- return new Path(String.format(parentDir, VersionInfo.getVersion()));
- }
-
- /**
- * Get slider.tar.gz absolute filepath in HDFS
- *
- * @return the absolute path to slider.tar.gz in HDFS
+ * @return the absolute path to service dependency tarball in HDFS
*/
public Path getDependencyTarGzip() {
- Path dependencyLibAmPath = getDependencyPath();
- Path dependencyLibTarGzip = new Path(
- dependencyLibAmPath.toUri().toString(),
- YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_NAME
- + YarnServiceConstants.DEPENDENCY_TAR_GZ_FILE_EXT);
+ Path dependencyLibTarGzip = null;
+ String configuredDependencyTarballPath = configuration
+ .get(YarnServiceConf.DEPENDENCY_TARBALL_PATH);
+ if (configuredDependencyTarballPath != null) {
+ dependencyLibTarGzip = new Path(configuredDependencyTarballPath);
+ }
return dependencyLibTarGzip;
}
@@ -467,8 +462,7 @@ public class CoreFileSystem {
fileSystem.getConf().set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY,
"000");
fileSystem.mkdirs(destPath.getParent(), fp);
- log.info("Copying file {} to {}", localPath.toURI(),
- fileSystem.getScheme() + ":/" + destPath.toUri());
+ log.info("Copying file {} to {}", localPath.toURI(), destPath);
fileSystem.copyFromLocalFile(false, true, new Path(localPath.getPath()),
destPath);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index 6aba91a..55be13b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -196,14 +196,18 @@ public abstract class AppAdminClient extends CompositeService {
* faster since the dependencies do not have to be uploaded on each launch.
* </p>
*
+ * @param destinationFolder
+ * an optional HDFS folder where dependency tarball will be uploaded
* @return exit code
- * @throws IOException IOException
- * @throws YarnException exception in client or server
+ * @throws IOException
+ * IOException
+ * @throws YarnException
+ * exception in client or server
*/
@Public
@Unstable
- public abstract int enableFastLaunch() throws IOException,
- YarnException;
+ public abstract int enableFastLaunch(String destinationFolder)
+ throws IOException, YarnException;
/**
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index c751f79..471b4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -232,9 +232,10 @@ public class ApplicationCLI extends YarnCLI {
"the number of components/containers running for an application / " +
"long-running service. Supports absolute or relative changes, such " +
"as +1, 2, or -3.");
- opts.addOption(ENABLE_FAST_LAUNCH, false, "Uploads AM dependencies " +
- "to HDFS to make future launches faster. Supports -appTypes option" +
- " to specify which client implementation to use.");
+ opts.addOption(ENABLE_FAST_LAUNCH, true, "Uploads AM dependencies " +
+ "to HDFS to make future launches faster. Supports -appTypes option " +
+ "to specify which client implementation to use. Optionally a " +
+ "destination folder for the tarball can be specified.");
opts.getOption(LAUNCH_CMD).setArgName("Application Name> <File Name");
opts.getOption(LAUNCH_CMD).setArgs(2);
opts.getOption(START_CMD).setArgName("Application Name");
@@ -245,6 +246,8 @@ public class ApplicationCLI extends YarnCLI {
opts.getOption(FLEX_CMD).setArgName("Application Name or ID");
opts.getOption(COMPONENT).setArgName("Component Name> <Count");
opts.getOption(COMPONENT).setArgs(2);
+ opts.getOption(ENABLE_FAST_LAUNCH).setOptionalArg(true);
+ opts.getOption(ENABLE_FAST_LAUNCH).setArgName("Destination Folder");
} else if (title != null && title.equalsIgnoreCase(APPLICATION_ATTEMPT)) {
opts.addOption(STATUS_CMD, true,
"Prints the status of the application attempt.");
@@ -513,13 +516,15 @@ public class ApplicationCLI extends YarnCLI {
.actionFlex(appNameAndType[0], counts);
} else if (cliParser.hasOption(ENABLE_FAST_LAUNCH)) {
String appType = getSingleAppTypeFromCLI(cliParser);
+ String uploadDestinationFolder = cliParser
+ .getOptionValue(ENABLE_FAST_LAUNCH);
if (hasAnyOtherCLIOptions(cliParser, opts, ENABLE_FAST_LAUNCH,
APP_TYPE_CMD)) {
printUsage(title, opts);
return exitCode;
}
return AppAdminClient.createAppAdminClient(appType, getConf())
- .enableFastLaunch();
+ .enableFastLaunch(uploadDestinationFolder);
} else if (cliParser.hasOption(UPDATE_LIFETIME)) {
if (!cliParser.hasOption(APP_ID)) {
printUsage(title, opts);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a72cdcc4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 25b426d..fdd3fc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -2064,11 +2064,14 @@ public class TestYarnCLI {
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
- pw.println(" -enableFastLaunch Uploads AM dependencies to HDFS");
+ pw.println(" -enableFastLaunch <Destination Folder> Uploads AM dependencies to HDFS");
pw.println(" to make future launches faster.");
pw.println(" Supports -appTypes option to");
pw.println(" specify which client");
pw.println(" implementation to use.");
+ pw.println(" Optionally a destination folder");
+ pw.println(" for the tarball can be");
+ pw.println(" specified.");
pw.println(" -flex <Application Name or ID> Changes number of running");
pw.println(" containers for a component of an");
pw.println(" application / long-running");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org