You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2017/04/08 18:08:39 UTC
[16/24] twill git commit: (TWILL-228) Remove the optimization when
building application jar
(TWILL-228) Remove the optimization when building application jar
- The optimization can leading skipping certain jars in the application jar
This closes #50 on Github.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/twill/commit/2a316a60
Tree: http://git-wip-us.apache.org/repos/asf/twill/tree/2a316a60
Diff: http://git-wip-us.apache.org/repos/asf/twill/diff/2a316a60
Branch: refs/heads/site
Commit: 2a316a60b041f39bff95bfcd45f308f7c89d4bfa
Parents: e61be38
Author: Terence Yim <ch...@apache.org>
Authored: Fri Mar 31 15:39:20 2017 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Mar 31 16:15:14 2017 -0700
----------------------------------------------------------------------
.../apache/twill/yarn/YarnTwillPreparer.java | 23 +++-----------------
.../twill/yarn/YarnTwillRunnerService.java | 21 +-----------------
2 files changed, 4 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/twill/blob/2a316a60/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 4846fe3..c8abf4f 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -146,7 +146,6 @@ final class YarnTwillPreparer implements TwillPreparer {
private final Credentials credentials;
private final Map<String, Map<String, String>> logLevels = Maps.newHashMap();
private final LocationCache locationCache;
- private final Set<URL> twillClassPaths;
private String schedulerQueue;
private String extraOptions;
private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG;
@@ -154,8 +153,8 @@ final class YarnTwillPreparer implements TwillPreparer {
private final Map<String, Integer> maxRetries = Maps.newHashMap();
YarnTwillPreparer(Configuration config, TwillSpecification twillSpec, RunId runId,
- String zkConnectString, Location appLocation, Set<URL> twillClassPaths,
- String extraOptions, LocationCache locationCache, YarnTwillControllerFactory controllerFactory) {
+ String zkConnectString, Location appLocation, String extraOptions,
+ LocationCache locationCache, YarnTwillControllerFactory controllerFactory) {
this.config = config;
this.twillSpec = twillSpec;
this.runId = runId;
@@ -166,7 +165,6 @@ final class YarnTwillPreparer implements TwillPreparer {
this.extraOptions = extraOptions;
this.classAcceptor = new ClassAcceptor();
this.locationCache = locationCache;
- this.twillClassPaths = twillClassPaths;
}
private void confirmRunnableName(String runnableName) {
@@ -372,7 +370,7 @@ final class YarnTwillPreparer implements TwillPreparer {
createLauncherJar(localFiles);
createTwillJar(createBundler(classAcceptor), yarnAppClient, localFiles);
- createApplicationJar(createApplicationJarBundler(classAcceptor), localFiles);
+ createApplicationJar(createBundler(classAcceptor), localFiles);
createResourcesJar(createBundler(classAcceptor), localFiles);
Path runtimeConfigDir = Files.createTempDirectory(getLocalStagingDir().toPath(),
@@ -835,19 +833,4 @@ final class YarnTwillPreparer implements TwillPreparer {
private ApplicationBundler createBundler(ClassAcceptor classAcceptor) {
return new ApplicationBundler(classAcceptor).setTempDir(getLocalStagingDir());
}
-
- /**
- * Creates a {@link ApplicationBundler} for building application jar. The bundler will include classes
- * accepted by the given {@link ClassAcceptor}, as long as it is not a twill class.
- */
- private ApplicationBundler createApplicationJarBundler(final ClassAcceptor classAcceptor) {
- // Accept classes based on the classAcceptor and also excluding all twill classes as they are already
- // in the twill.jar
- return createBundler(new ClassAcceptor() {
- @Override
- public boolean accept(String className, URL classUrl, URL classPathUrl) {
- return !twillClassPaths.contains(classPathUrl) && classAcceptor.accept(className, classUrl, classPathUrl);
- }
- });
- }
}
http://git-wip-us.apache.org/repos/asf/twill/blob/2a316a60/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 269ffdf..20627e2 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -44,7 +44,6 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.twill.api.ClassAcceptor;
import org.apache.twill.api.Configs;
import org.apache.twill.api.LocalFile;
import org.apache.twill.api.ResourceSpecification;
@@ -73,7 +72,6 @@ import org.apache.twill.internal.appmaster.ApplicationMasterLiveNodeData;
import org.apache.twill.internal.io.BasicLocationCache;
import org.apache.twill.internal.io.LocationCache;
import org.apache.twill.internal.io.NoCachingLocationCache;
-import org.apache.twill.internal.utils.Dependencies;
import org.apache.twill.internal.yarn.VersionDetectYarnAppClientFactory;
import org.apache.twill.internal.yarn.YarnAppClient;
import org.apache.twill.internal.yarn.YarnApplicationReport;
@@ -96,7 +94,6 @@ import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.URL;
import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
@@ -134,7 +131,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
private final ZKClientService zkClientService;
private final LocationFactory locationFactory;
private final Table<String, RunId, YarnTwillController> controllers;
- private final Set<URL> twillClassPaths;
// A Guava service to help the state transition.
private final Service serviceDelegate;
private LocationCache locationCache;
@@ -169,7 +165,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
this.locationFactory = locationFactory;
this.zkClientService = getZKClientService(zkConnect);
this.controllers = HashBasedTable.create();
- this.twillClassPaths = new HashSet<>();
this.serviceDelegate = new AbstractIdleService() {
@Override
protected void startUp() throws Exception {
@@ -308,8 +303,7 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
Configuration config = new Configuration(yarnConfig);
return new YarnTwillPreparer(config, twillSpec, runId, zkClientService.getConnectString(),
- appLocation, twillClassPaths, jvmOptions,
- locationCache, new YarnTwillControllerFactory() {
+ appLocation, jvmOptions, locationCache, new YarnTwillControllerFactory() {
@Override
public YarnTwillController create(RunId runId, boolean logCollectionEnabled, Iterable<LogHandler> logHandlers,
Callable<ProcessController<YarnApplicationReport>> startUp,
@@ -355,19 +349,6 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
private void startUp() throws Exception {
zkClientService.startAndWait();
- // Find all the classpaths for Twill classes. It is used for class filtering when building application jar
- // in the YarnTwillPreparer
- Dependencies.findClassDependencies(getClass().getClassLoader(), new ClassAcceptor() {
- @Override
- public boolean accept(String className, URL classUrl, URL classPathUrl) {
- if (!className.startsWith("org.apache.twill.")) {
- return false;
- }
- twillClassPaths.add(classPathUrl);
- return true;
- }
- }, getClass().getName());
-
// Create the root node, so that the namespace root would get created if it is missing
// If the exception is caused by node exists, then it's ok. Otherwise propagate the exception.
ZKOperations.ignoreError(zkClientService.create("/", null, CreateMode.PERSISTENT),