You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2016/06/21 23:23:38 UTC
[1/5] incubator-twill git commit: Unnecessary new line for assignment.
Repository: incubator-twill
Updated Branches:
refs/heads/site c7c3f774c -> 6c7809d19
Unnecessary new line for assignment.
Add close message to close obsolete PR.
This closes #60
This closes #80 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/94995e2a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/94995e2a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/94995e2a
Branch: refs/heads/site
Commit: 94995e2af0a7a734c3dcbd31a78f330d048eda96
Parents: 1b6b42c
Author: Henry Saputra <hs...@apache.org>
Authored: Fri May 6 11:13:17 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue May 17 10:51:26 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/twill/example/yarn/HelloWorld.java | 7 ++-----
1 file changed, 2 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/94995e2a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
index ff1b83c..1a573da 100644
--- a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
+++ b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
@@ -65,9 +65,7 @@ public class HelloWorld {
String zkStr = args[0];
YarnConfiguration yarnConfiguration = new YarnConfiguration();
- final TwillRunnerService twillRunner =
- new YarnTwillRunnerService(
- yarnConfiguration, zkStr);
+ final TwillRunnerService twillRunner = new YarnTwillRunnerService(yarnConfiguration, zkStr);
twillRunner.start();
String yarnClasspath =
@@ -75,8 +73,7 @@ public class HelloWorld {
Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
List<String> applicationClassPaths = Lists.newArrayList();
Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
- final TwillController controller =
- twillRunner.prepare(new HelloWorldRunnable())
+ final TwillController controller = twillRunner.prepare(new HelloWorldRunnable())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
.withApplicationClassPaths(applicationClassPaths)
.withBundlerClassAcceptor(new HadoopClassExcluder())
[5/5] incubator-twill git commit: Moved site generation to it’s own profile.
Posted by ch...@apache.org.
Moved site generation to it\u2019s own profile.
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/6c7809d1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/6c7809d1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/6c7809d1
Branch: refs/heads/site
Commit: 6c7809d19f14235525c7dc6ff92e33c5cad92a8d
Parents: 85d5db1
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jun 21 16:23:33 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jun 21 16:23:33 2016 -0700
----------------------------------------------------------------------
pom.xml | 118 +++++++++++++++++++++++++++++++++++++++++++++++++
src/site/site.xml | 5 +++
2 files changed, 123 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6c7809d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aa6cf53..e65930f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -734,6 +734,124 @@
<module>twill-java8-test</module>
</modules>
</profile>
+ <profile>
+ <id>site</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.4</version>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-core</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.maven.doxia</groupId>
+ <artifactId>doxia-module-markdown</artifactId>
+ <version>1.6</version>
+ </dependency>
+ <dependency>
+ <groupId>lt.velykis.maven.skins</groupId>
+ <artifactId>reflow-velocity-tools</artifactId>
+ <version>1.1.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.velocity</groupId>
+ <artifactId>velocity</artifactId>
+ <version>1.7</version>
+ </dependency>
+ </dependencies>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>0.11</version>
+ <configuration>
+ <excludes>
+ <exclude>.git/**/*</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>target/**/*</exclude>
+ <exclude>**/README</exclude>
+ <exclude>src/test/resources/header.txt</exclude>
+ <exclude>**/zookeeper.out</exclude>
+ <exclude>twill-java8-test/target/**</exclude>
+ <exclude>src/site/resources/**</exclude>
+ </excludes>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.7</version>
+ <reportSets>
+ <reportSet>
+ <reports/>
+ </reportSet>
+ <reportSet>
+ <id>aggregate</id>
+ <inherited>false</inherited>
+ <reports>
+ <report>index</report>
+ <report>mailing-list</report>
+ <report>scm</report>
+ <report>issue-tracking</report>
+ <report>project-team</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9.1</version>
+ <configuration>
+ <failOnError>false</failOnError>
+ <excludePackageNames>*.internal.*:echo:*.example.*</excludePackageNames>
+ <links>
+ <link>http://download.oracle.com/javase/6/docs/api/</link>
+ </links>
+ <bottom>
+ <![CDATA[Copyright © 2013-2016 <a href="http://www.apache.org">The Apache Software Foundation</a>. All rights reserved.]]>
+ </bottom>
+ </configuration>
+ <reportSets>
+ <reportSet>
+ <reports>
+ <report>javadoc</report>
+ </reports>
+ </reportSet>
+ <reportSet>
+ <id>aggregate</id>
+ <inherited>false</inherited>
+ <reports>
+ <report>aggregate</report>
+ </reports>
+ </reportSet>
+ </reportSets>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>2.14.1</version>
+ </plugin>
+ </plugins>
+ </reporting>
+ </profile>
</profiles>
<dependencyManagement>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/6c7809d1/src/site/site.xml
----------------------------------------------------------------------
diff --git a/src/site/site.xml b/src/site/site.xml
index bed9695..ca9370a 100644
--- a/src/site/site.xml
+++ b/src/site/site.xml
@@ -46,6 +46,9 @@
</skin>
<body>
+ <head>
+ <link rel="stylesheet" href="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/styles/default.min.css"/>
+ </head>
<breadcrumbs position="left">
<item name="Apache Twill" href="http://twill.incubator.apache.org/index.html"/>
</breadcrumbs>
@@ -88,6 +91,7 @@
and the Apache Twill project logos are trademarks of The Apache Software Foundation.
All other marks mentioned may be trademarks or registered trademarks of their respective owners.
</div>
+ <script src="http://cdnjs.cloudflare.com/ajax/libs/highlight.js/9.4.0/highlight.min.js"></script>
</footer>
</body>
@@ -95,6 +99,7 @@
<reflowSkin>
<theme>bootswatch-flatly</theme>
<smoothScroll>true</smoothScroll>
+ <markPageHeader>false</markPageHeader>
<bottomNav maxSpan="9" >
<column>Documentation</column>
<column>Releases</column>
[3/5] incubator-twill git commit: - Upgrade rat plugin to 0.11 - Move
site generation logic into a different branch
Posted by ch...@apache.org.
- Upgrade rat plugin to 0.11
- Move site generation logic into a different branch
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/c1a6f8af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/c1a6f8af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/c1a6f8af
Branch: refs/heads/site
Commit: c1a6f8af261ee3b48834d91a3b48a26aa26382f4
Parents: 16f47ea
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jun 21 16:01:02 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jun 21 16:01:02 2016 -0700
----------------------------------------------------------------------
pom.xml | 76 ++----------------------------------------------------------
1 file changed, 2 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/c1a6f8af/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 839214c..aa6cf53 100644
--- a/pom.xml
+++ b/pom.xml
@@ -279,7 +279,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
- <version>0.10</version>
+ <version>0.11</version>
<configuration>
<excludes>
<exclude>.git/**/*</exclude>
@@ -320,7 +320,7 @@
<plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
- <version>0.10</version>
+ <version>0.11</version>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
@@ -342,18 +342,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-site-plugin</artifactId>
- <version>3.3</version>
- <dependencies>
- <dependency>
- <groupId>org.apache.maven.doxia</groupId>
- <artifactId>doxia-module-markdown</artifactId>
- <version>1.3</version>
- </dependency>
- </dependencies>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.14.1</version>
<configuration>
@@ -1042,64 +1030,4 @@
</dependency>
</dependencies>
</dependencyManagement>
-
- <reporting>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-project-info-reports-plugin</artifactId>
- <version>2.7</version>
- <reportSets>
- <reportSet>
- <reports/>
- </reportSet>
- <reportSet>
- <id>aggregate</id>
- <inherited>false</inherited>
- <reports>
- <report>index</report>
- <report>mailing-list</report>
- <report>scm</report>
- <report>issue-tracking</report>
- <report>project-team</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-javadoc-plugin</artifactId>
- <version>2.9.1</version>
- <configuration>
- <failOnError>false</failOnError>
- <excludePackageNames>*.internal.*:echo:*.example.*</excludePackageNames>
- <links>
- <link>http://download.oracle.com/javase/6/docs/api/</link>
- </links>
- <bottom>
- <![CDATA[Copyright © 2013 <a href="http://www.apache.org">The Apache Software Foundation</a>. All rights reserved.]]>
- </bottom>
- </configuration>
- <reportSets>
- <reportSet>
- <reports>
- <report>javadoc</report>
- </reports>
- </reportSet>
- <reportSet>
- <id>aggregate</id>
- <inherited>false</inherited>
- <reports>
- <report>aggregate</report>
- </reports>
- </reportSet>
- </reportSets>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-surefire-report-plugin</artifactId>
- <version>2.14.1</version>
- </plugin>
- </plugins>
- </reporting>
</project>
[2/5] incubator-twill git commit: (TWILL-163) Fix
FileContextLocationFactory to honor UGI for home location
Posted by ch...@apache.org.
(TWILL-163) Fix FileContextLocationFactory to honor UGI for home location
- Also include removal on usage of the deprecated HDFSLocationFactory
This closes #78 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/16f47ea4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/16f47ea4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/16f47ea4
Branch: refs/heads/site
Commit: 16f47ea490c5a9df7ab78240a0b0039df1acb043
Parents: 94995e2
Author: Terence Yim <ch...@apache.org>
Authored: Thu Jan 28 12:36:14 2016 -0800
Committer: Terence Yim <ch...@apache.org>
Committed: Tue May 17 11:01:20 2016 -0700
----------------------------------------------------------------------
.../filesystem/FileContextLocationFactory.java | 17 +++++-
.../org/apache/twill/internal/ServiceMain.java | 38 ++++++++-----
.../apache/twill/internal/yarn/YarnUtils.java | 58 ++++++++++----------
.../apache/twill/yarn/YarnTwillPreparer.java | 5 --
.../twill/yarn/YarnTwillRunnerService.java | 10 ++--
.../twill/filesystem/LocalLocationTest.java | 8 +++
.../twill/filesystem/LocationTestBase.java | 22 +++++++-
.../org/apache/twill/yarn/BaseYarnTest.java | 20 +++++++
.../apache/twill/yarn/EchoServerTestRun.java | 18 +++++-
9 files changed, 138 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationFactory.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationFactory.java b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationFactory.java
index d64be71..b8453bc 100644
--- a/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationFactory.java
+++ b/twill-yarn/src/main/java/org/apache/twill/filesystem/FileContextLocationFactory.java
@@ -50,8 +50,19 @@ public class FileContextLocationFactory implements LocationFactory {
* @param pathBase base path for all non-absolute location created through this {@link LocationFactory}.
*/
public FileContextLocationFactory(Configuration configuration, String pathBase) {
+ this(configuration, createFileContext(configuration), pathBase);
+ }
+
+ /**
+ * Creates a new instance with the given {@link FileContext} created from the given {@link Configuration}.
+ *
+ * @param configuration the hadoop configuration
+ * @param fc {@link FileContext} instance created from the given configuration
+ * @param pathBase base path for all non-absolute location created through this (@link LocationFactory}.
+ */
+ public FileContextLocationFactory(Configuration configuration, FileContext fc, String pathBase) {
this.configuration = configuration;
- this.fc = createFileContext(configuration);
+ this.fc = fc;
this.pathBase = new Path(pathBase.startsWith("/") ? pathBase : "/" + pathBase);
}
@@ -92,7 +103,9 @@ public class FileContextLocationFactory implements LocationFactory {
@Override
public Location getHomeLocation() {
- return new FileContextLocation(this, fc, fc.getHomeDirectory());
+ // Fix for TWILL-163. FileContext.getHomeDirectory() uses System.getProperty("user.name") instead of UGI
+ return new FileContextLocation(this, fc,
+ new Path(fc.getHomeDirectory().getParent(), fc.getUgi().getShortUserName()));
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
index cafd375..a6d9132 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -27,10 +27,12 @@ import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.Service;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.twill.api.RunId;
-import org.apache.twill.filesystem.HDFSLocationFactory;
+import org.apache.twill.filesystem.FileContextLocationFactory;
import org.apache.twill.filesystem.LocalLocationFactory;
import org.apache.twill.filesystem.Location;
import org.apache.twill.internal.logging.KafkaAppender;
@@ -50,6 +52,8 @@ import org.xml.sax.InputSource;
import java.io.File;
import java.io.StringReader;
import java.net.URI;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
@@ -127,27 +131,35 @@ public abstract class ServiceMain {
/**
* Returns the {@link Location} for the application based on the env {@link EnvKeys#TWILL_APP_DIR}.
*/
- protected static Location createAppLocation(Configuration conf) {
+ protected static Location createAppLocation(final Configuration conf) {
// Note: It's a little bit hacky based on the uri schema to create the LocationFactory, refactor it later.
- URI appDir = URI.create(System.getenv(EnvKeys.TWILL_APP_DIR));
+ final URI appDir = URI.create(System.getenv(EnvKeys.TWILL_APP_DIR));
try {
if ("file".equals(appDir.getScheme())) {
return new LocalLocationFactory().create(appDir);
}
- // If not file, assuming it is a FileSystem, hence construct with HDFSLocationFactory which wraps
- // a FileSystem created from the Configuration
+ // If not file, assuming it is a FileSystem, hence construct with FileContextLocationFactory
+ UserGroupInformation ugi;
if (UserGroupInformation.isSecurityEnabled()) {
- return new HDFSLocationFactory(FileSystem.get(appDir, conf)).create(appDir);
- }
-
- String fsUser = System.getenv(EnvKeys.TWILL_FS_USER);
- if (fsUser == null) {
- throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER);
+ ugi = UserGroupInformation.getCurrentUser();
+ } else {
+ String fsUser = System.getenv(EnvKeys.TWILL_FS_USER);
+ if (fsUser == null) {
+ throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER);
+ }
+ ugi = UserGroupInformation.createRemoteUser(fsUser);
}
- return new HDFSLocationFactory(FileSystem.get(appDir, conf, fsUser)).create(appDir);
-
+ return ugi.doAs(new PrivilegedExceptionAction<Location>() {
+ @Override
+ public Location run() throws Exception {
+ Configuration hConf = new Configuration(conf);
+ URI defaultURI = new URI(appDir.getScheme(), appDir.getAuthority(), null, null, null);
+ hConf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultURI.toString());
+ return new FileContextLocationFactory(hConf).create(appDir);
+ }
+ });
} catch (Exception e) {
LOG.error("Failed to create application location for {}.", appDir);
throw Throwables.propagate(e);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
index 1c65591..e63deed 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
@@ -17,14 +17,13 @@
*/
package org.apache.twill.internal.yarn;
-import com.google.common.base.Function;
import com.google.common.base.Preconditions;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.security.Credentials;
@@ -39,6 +38,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.twill.api.LocalFile;
+import org.apache.twill.filesystem.FileContextLocationFactory;
import org.apache.twill.filesystem.ForwardingLocationFactory;
import org.apache.twill.filesystem.HDFSLocationFactory;
import org.apache.twill.filesystem.LocationFactory;
@@ -50,7 +50,6 @@ import java.lang.reflect.Method;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -68,7 +67,7 @@ public class YarnUtils {
}
private static final Logger LOG = LoggerFactory.getLogger(YarnUtils.class);
- private static final AtomicReference<HadoopVersions> HADOOP_VERSION = new AtomicReference<HadoopVersions>();
+ private static final AtomicReference<HadoopVersions> HADOOP_VERSION = new AtomicReference<>();
public static YarnLocalResource createLocalResource(LocalFile localFile) {
Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0.");
@@ -155,19 +154,31 @@ public class YarnUtils {
return ImmutableList.of();
}
- FileSystem fileSystem = getFileSystem(locationFactory);
+ LocationFactory factory = unwrap(locationFactory);
+ String renewer = getYarnTokenRenewer(config);
+ List<Token<?>> tokens = ImmutableList.of();
- if (fileSystem == null) {
- LOG.debug("LocationFactory is not HDFS");
- return ImmutableList.of();
+ if (factory instanceof HDFSLocationFactory) {
+ FileSystem fs = ((HDFSLocationFactory) factory).getFileSystem();
+ Token<?>[] fsTokens = fs.addDelegationTokens(renewer, credentials);
+ if (fsTokens != null) {
+ tokens = ImmutableList.copyOf(fsTokens);
+ }
+ } else if (factory instanceof FileContextLocationFactory) {
+ FileContext fc = ((FileContextLocationFactory) locationFactory).getFileContext();
+ tokens = fc.getDelegationTokens(new Path(locationFactory.create("/").toURI()), renewer);
}
- String renewer = getYarnTokenRenewer(config);
+ for (Token<?> token : tokens) {
+ credentials.addToken(token.getService(), token);
+ }
- Token<?>[] tokens = fileSystem.addDelegationTokens(renewer, credentials);
- return tokens == null ? ImmutableList.<Token<?>>of() : ImmutableList.copyOf(tokens);
+ return ImmutableList.copyOf(tokens);
}
+ /**
+ * Encodes the given {@link Credentials} as bytes.
+ */
public static ByteBuffer encodeCredentials(Credentials credentials) {
try {
DataOutputBuffer out = new DataOutputBuffer();
@@ -268,26 +279,15 @@ public class YarnUtils {
return localResource;
}
- private static <T> Map<String, T> transformResource(Map<String, YarnLocalResource> from) {
- return Maps.transformValues(from, new Function<YarnLocalResource, T>() {
- @Override
- public T apply(YarnLocalResource resource) {
- return resource.getLocalResource();
- }
- });
- }
-
/**
- * Gets the Hadoop FileSystem from LocationFactory.
+ * Unwraps the given {@link LocationFactory} and returns the inner most {@link LocationFactory} which is not
+ * a {@link ForwardingLocationFactory}.
*/
- private static FileSystem getFileSystem(LocationFactory locationFactory) {
- if (locationFactory instanceof HDFSLocationFactory) {
- return ((HDFSLocationFactory) locationFactory).getFileSystem();
- }
- if (locationFactory instanceof ForwardingLocationFactory) {
- return getFileSystem(((ForwardingLocationFactory) locationFactory).getDelegate());
+ private static LocationFactory unwrap(LocationFactory locationFactory) {
+ while (locationFactory instanceof ForwardingLocationFactory) {
+ locationFactory = ((ForwardingLocationFactory) locationFactory).getDelegate();
}
- return null;
+ return locationFactory;
}
private YarnUtils() {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index d04cdab..f7cb388 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -83,8 +83,6 @@ import org.apache.twill.internal.yarn.YarnApplicationReport;
import org.apache.twill.internal.yarn.YarnUtils;
import org.apache.twill.launcher.FindFreePort;
import org.apache.twill.launcher.TwillLauncher;
-import org.apache.twill.zookeeper.ZKClient;
-import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -131,7 +129,6 @@ final class YarnTwillPreparer implements TwillPreparer {
private final List<String> applicationClassPaths = Lists.newArrayList();
private final Credentials credentials;
private final int reservedMemory;
- private String user;
private String schedulerQueue;
private String extraOptions;
private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG;
@@ -152,7 +149,6 @@ final class YarnTwillPreparer implements TwillPreparer {
this.credentials = createCredentials();
this.reservedMemory = yarnConfig.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB,
Configs.Defaults.JAVA_RESERVED_MEMORY_MB);
- this.user = System.getProperty("user.name");
this.extraOptions = extraOptions;
this.logLevel = logLevel;
this.classAcceptor = new ClassAcceptor();
@@ -166,7 +162,6 @@ final class YarnTwillPreparer implements TwillPreparer {
@Override
public TwillPreparer setUser(String user) {
- this.user = user;
return this;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index c5853d6..67ee2ac 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -42,7 +42,7 @@ import com.google.gson.Gson;
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -62,7 +62,7 @@ import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.common.Cancellable;
import org.apache.twill.common.Threads;
-import org.apache.twill.filesystem.HDFSLocationFactory;
+import org.apache.twill.filesystem.FileContextLocationFactory;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
import org.apache.twill.internal.Constants;
@@ -139,7 +139,7 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
private volatile String jvmOptions = null;
/**
- * Creates an instance with a {@link HDFSLocationFactory} created base on the given configuration with the
+ * Creates an instance with a {@link FileContextLocationFactory} created base on the given configuration with the
* user home directory as the location factory namespace.
*
* @param config Configuration of the yarn cluster
@@ -612,8 +612,8 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
private static LocationFactory createDefaultLocationFactory(Configuration configuration) {
try {
- FileSystem fs = FileSystem.get(configuration);
- return new HDFSLocationFactory(fs, fs.getHomeDirectory().toUri().getPath());
+ FileContext fc = FileContext.getFileContext(configuration);
+ return new FileContextLocationFactory(configuration, fc, fc.getHomeDirectory().toUri().getPath());
} catch (IOException e) {
throw Throwables.propagate(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java b/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
index ba21beb..6bdba27 100644
--- a/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
+++ b/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.twill.filesystem;
+import org.junit.Assert;
+
import java.io.File;
/**
@@ -30,4 +32,10 @@ public class LocalLocationTest extends LocationTestBase {
basePath.mkdirs();
return new LocalLocationFactory(basePath);
}
+
+ @Override
+ public void testHomeLocation() throws Exception {
+ // For Local location, UGI won't take an effect.
+ Assert.assertEquals(System.getProperty("user.name"), createLocationFactory("/").getHomeLocation().getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/test/java/org/apache/twill/filesystem/LocationTestBase.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/filesystem/LocationTestBase.java b/twill-yarn/src/test/java/org/apache/twill/filesystem/LocationTestBase.java
index e01115b..af485dc 100644
--- a/twill-yarn/src/test/java/org/apache/twill/filesystem/LocationTestBase.java
+++ b/twill-yarn/src/test/java/org/apache/twill/filesystem/LocationTestBase.java
@@ -22,6 +22,7 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.io.CharStreams;
+import org.apache.hadoop.security.UserGroupInformation;
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
@@ -33,7 +34,7 @@ import java.io.OutputStreamWriter;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
-import java.net.URISyntaxException;
+import java.security.PrivilegedExceptionAction;
import java.util.List;
/**
@@ -96,6 +97,25 @@ public abstract class LocationTestBase {
}
@Test
+ public void testHomeLocation() throws Exception {
+ LocationFactory locationFactory = createLocationFactory("/");
+
+ // Without UGI, the home location should be the same as the user
+ Assert.assertEquals(System.getProperty("user.name"), locationFactory.getHomeLocation().getName());
+
+ // With UGI, the home location should be based on the UGI current user
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(System.getProperty("user.name") + "1");
+ locationFactory = ugi.doAs(new PrivilegedExceptionAction<LocationFactory>() {
+ @Override
+ public LocationFactory run() throws Exception {
+ return createLocationFactory("/");
+ }
+ });
+
+ Assert.assertEquals(ugi.getUserName(), locationFactory.getHomeLocation().getName());
+ }
+
+ @Test
public void testDelete() throws IOException {
LocationFactory factory = locationFactoryCache.getUnchecked("delete");
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java b/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
index a9cf2ed..5d67dfa 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/BaseYarnTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.twill.yarn;
+import com.google.common.base.Stopwatch;
import com.google.common.collect.Iterables;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.twill.api.TwillController;
@@ -28,6 +29,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.List;
+import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -101,6 +103,24 @@ public abstract class BaseYarnTest {
return trial < limit;
}
+ /**
+ * Waits for a task returns the expected value.
+ *
+ * @param expected the expected value
+ * @param callable the task to execute
+ * @param timeout timeout of the wait
+ * @param delay delay between calls to the task to poll for the latest value
+ * @param unit unit for the timeout and delay
+ * @param <T> type of the expected value
+ * @throws Exception if the task through exception or timeout.
+ */
+ public <T> void waitFor(T expected, Callable<T> callable, long timeout, long delay, TimeUnit unit) throws Exception {
+ Stopwatch stopwatch = new Stopwatch().start();
+ while (callable.call() != expected && stopwatch.elapsedTime(unit) < timeout) {
+ unit.sleep(delay);
+ }
+ }
+
@SuppressWarnings("unchecked")
public <T extends TwillRunner> T getTwillRunner() {
return (T) TWILL_TESTER.getTwillRunner();
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/16f47ea4/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
index 13c07b1..02075a7 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -32,6 +32,7 @@ import org.apache.twill.api.logging.PrinterLogHandler;
import org.apache.twill.common.Threads;
import org.apache.twill.discovery.Discoverable;
import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.zookeeper.data.Stat;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -45,6 +46,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.annotation.Nullable;
@@ -158,7 +160,7 @@ public final class EchoServerTestRun extends BaseYarnTest {
@Test
public void testZKCleanup() throws Exception {
- ZKClientService zkClient = ZKClientService.Builder.of(getZKConnectionString() + "/twill").build();
+ final ZKClientService zkClient = ZKClientService.Builder.of(getZKConnectionString() + "/twill").build();
zkClient.startAndWait();
try {
@@ -177,7 +179,12 @@ public final class EchoServerTestRun extends BaseYarnTest {
controller.terminate().get();
// Verify the ZK node gets cleanup
- Assert.assertNull(zkClient.exists("/EchoServer").get());
+ waitFor(null, new Callable<Stat>() {
+ @Override
+ public Stat call() throws Exception {
+ return zkClient.exists("/EchoServer").get();
+ }
+ }, 10000, 100, TimeUnit.MILLISECONDS);
// Start two instances of the application and stop one of it
List<TwillController> controllers = new ArrayList<>();
@@ -207,7 +214,12 @@ public final class EchoServerTestRun extends BaseYarnTest {
controllers.get(1).terminate().get();
// Verify the ZK node gets cleanup
- Assert.assertNull(zkClient.exists("/EchoServer").get());
+ waitFor(null, new Callable<Stat>() {
+ @Override
+ public Stat call() throws Exception {
+ return zkClient.exists("/EchoServer").get();
+ }
+ }, 10000, 100, TimeUnit.MILLISECONDS);
} finally {
zkClient.stopAndWait();
[4/5] incubator-twill git commit: Merge branch 'master' into site
Posted by ch...@apache.org.
Merge branch 'master' into site
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/85d5db1d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/85d5db1d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/85d5db1d
Branch: refs/heads/site
Commit: 85d5db1d72334bacee5f6d194aed24b9ebae8db4
Parents: c7c3f77 c1a6f8a
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jun 21 16:02:17 2016 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jun 21 16:02:17 2016 -0700
----------------------------------------------------------------------
pom.xml | 86 +-------------------
.../apache/twill/example/yarn/HelloWorld.java | 7 +-
.../filesystem/FileContextLocationFactory.java | 17 +++-
.../org/apache/twill/internal/ServiceMain.java | 38 ++++++---
.../apache/twill/internal/yarn/YarnUtils.java | 58 ++++++-------
.../apache/twill/yarn/YarnTwillPreparer.java | 5 --
.../twill/yarn/YarnTwillRunnerService.java | 10 +--
.../twill/filesystem/LocalLocationTest.java | 8 ++
.../twill/filesystem/LocationTestBase.java | 22 ++++-
.../org/apache/twill/yarn/BaseYarnTest.java | 20 +++++
.../apache/twill/yarn/EchoServerTestRun.java | 18 +++-
11 files changed, 142 insertions(+), 147 deletions(-)
----------------------------------------------------------------------