You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2015/07/30 21:58:48 UTC
incubator-twill git commit: (TWILL-144) No need to rename resource
name when localization to YARN
Repository: incubator-twill
Updated Branches:
refs/heads/master 0c20804c2 -> bb0212d41
(TWILL-144) No need to rename resource name when localization to YARN
This closes #57 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/bb0212d4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/bb0212d4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/bb0212d4
Branch: refs/heads/master
Commit: bb0212d415be891b53b40e3ec928f742198762fd
Parents: 0c20804
Author: Terence Yim <ch...@apache.org>
Authored: Mon Jul 27 21:15:46 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jul 30 12:58:43 2015 -0700
----------------------------------------------------------------------
.../ApplicationMasterProcessLauncher.java | 5 -----
.../internal/container/TwillContainerMain.java | 13 -------------
.../yarn/AbstractYarnProcessLauncher.java | 20 +-------------------
3 files changed, 1 insertion(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
index 4112080..126ff97 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
@@ -43,11 +43,6 @@ public final class ApplicationMasterProcessLauncher extends AbstractYarnProcessL
}
@Override
- protected boolean useArchiveSuffix() {
- return false;
- }
-
- @Override
@SuppressWarnings("unchecked")
protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
final ApplicationId appId = getContainerInfo();
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
index e04c8f9..51837a7 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -86,7 +86,6 @@ public final class TwillContainerMain extends ServiceMain {
ZKClient appRunZkClient = getAppRunZKClient(zkClientService, appRunId);
TwillSpecification twillSpec = loadTwillSpec(twillSpecFile);
- renameLocalFiles(twillSpec.getRunnables().get(runnableName));
TwillRunnableSpecification runnableSpec = twillSpec.getRunnables().get(runnableName).getRunnableSpecification();
ContainerInfo containerInfo = new EnvContainerInfo();
@@ -135,18 +134,6 @@ public final class TwillContainerMain extends ServiceMain {
}
}
- private static void renameLocalFiles(RuntimeSpecification runtimeSpec) {
- for (LocalFile file : runtimeSpec.getLocalFiles()) {
- if (file.isArchive()) {
- String path = file.getURI().toString();
- String name = file.getName() + (path.endsWith(".tar.gz") ? ".tar.gz" : path.substring(path.lastIndexOf('.')));
- Preconditions.checkState(new File(name).renameTo(new File(file.getName())),
- "Fail to rename file from %s to %s.",
- name, file.getName());
- }
- }
- }
-
/**
* Returns a {@link ZKClient} that namespaced under the given run id.
*/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/bb0212d4/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
index ebbb559..2023b0e 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.twill.api.LocalFile;
import org.apache.twill.internal.ProcessController;
import org.apache.twill.internal.ProcessLauncher;
-import org.apache.twill.internal.utils.Paths;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -65,13 +64,6 @@ public abstract class AbstractYarnProcessLauncher<T> implements ProcessLauncher<
}
/**
- * Tells whether to append suffix to localize resource name for archive file type. Default is true.
- */
- protected boolean useArchiveSuffix() {
- return true;
- }
-
- /**
* For children class to override to perform actual process launching.
*/
protected abstract <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext);
@@ -100,17 +92,7 @@ public abstract class AbstractYarnProcessLauncher<T> implements ProcessLauncher<
}
private void addLocalFile(LocalFile localFile) {
- String name = localFile.getName();
- // Always append the file extension as the resource name so that archive expansion by Yarn could work.
- // Renaming would happen by the Container Launcher.
- if (localFile.isArchive() && useArchiveSuffix()) {
- String path = localFile.getURI().toString();
- String suffix = Paths.getExtension(path);
- if (!suffix.isEmpty()) {
- name += '.' + suffix;
- }
- }
- localResources.put(name, YarnUtils.createLocalResource(localFile));
+ localResources.put(localFile.getName(), YarnUtils.createLocalResource(localFile));
}
@Override