You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2016/01/26 20:16:23 UTC
[05/22] incubator-twill git commit: (TWILL-154) Remove hardcoded check for “hdfs” or “maprfs” URI prefix
(TWILL-154) Remove hardcoded check for “hdfs” or “maprfs” URI prefix
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ef8b1eae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ef8b1eae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ef8b1eae
Branch: refs/heads/site
Commit: ef8b1eae804db155dd05b20163e507521694647b
Parents: f88e18f
Author: Terence Yim <ch...@apache.org>
Authored: Fri Oct 9 13:42:22 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Oct 9 15:36:40 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/internal/ServiceMain.java | 21 +++++++++-----------
.../apache/twill/yarn/YarnTwillPreparer.java | 5 +++--
2 files changed, 12 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ef8b1eae/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
index f7bea24..a6af3d3 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -136,20 +136,17 @@ public abstract class ServiceMain {
return new LocalLocationFactory().create(appDir);
}
- if ("hdfs".equals(appDir.getScheme()) || "maprfs".equals(appDir.getScheme())) {
- if (UserGroupInformation.isSecurityEnabled()) {
- return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir);
- }
-
- String fsUser = System.getenv(EnvKeys.TWILL_FS_USER);
- if (fsUser == null) {
- throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER);
- }
- return new HDFSLocationFactory(FileSystem.get(appDir, conf, fsUser)).create(appDir);
+ // If not file, assuming it is a FileSystem, hence construct with HDFSLocationFactory which wraps
+ // a FileSystem created from the Configuration
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir);
}
- LOG.warn("Unsupported location type {}.", appDir);
- throw new IllegalArgumentException("Unsupported location type " + appDir);
+ String fsUser = System.getenv(EnvKeys.TWILL_FS_USER);
+ if (fsUser == null) {
+ throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER);
+ }
+ return new HDFSLocationFactory(FileSystem.get(appDir, conf, fsUser)).create(appDir);
} catch (Exception e) {
LOG.error("Failed to create application location for {}.", appDir);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ef8b1eae/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index a444dda..ea77116 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -52,6 +52,7 @@ import org.apache.twill.api.TwillPreparer;
import org.apache.twill.api.TwillSpecification;
import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.filesystem.HDFSLocationFactory;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
import org.apache.twill.internal.ApplicationBundler;
@@ -451,8 +452,8 @@ final class YarnTwillPreparer implements TwillPreparer {
Location location;
URI uri = localFile.getURI();
- if ("hdfs".equals(uri.getScheme())) {
- // Assuming the location factory is HDFS one. If it is not, it will failed, which is the correct behavior.
+ if (locationFactory.getHomeLocation().toURI().getScheme().equals(uri.getScheme())) {
+ // If the source file location is having the same scheme as the target location, no need to copy
location = locationFactory.create(uri);
} else {
URL url = uri.toURL();