You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2015/07/24 20:29:35 UTC
[01/25] incubator-twill git commit: Remove javadoc stylesheet
Repository: incubator-twill
Updated Branches:
refs/heads/site 5c6f5788c -> 1fb6c78da
Remove javadoc stylesheet
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/f24896b9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/f24896b9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/f24896b9
Branch: refs/heads/site
Commit: f24896b95854e518b00909a544bd21bf7380f4d5
Parents: 6896257
Author: Terence Yim <ch...@apache.org>
Authored: Fri May 22 17:46:10 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri May 22 17:46:41 2015 -0700
----------------------------------------------------------------------
pom.xml | 1 -
src/site/javadoc-stylesheet.css | 99 ------------------------------------
2 files changed, 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/f24896b9/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e079273..f41e075 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1063,7 +1063,6 @@
<configuration>
<failOnError>false</failOnError>
<excludePackageNames>*.internal.*:echo:*.example.*</excludePackageNames>
- <stylesheetfile>${basedir}/src/site/javadoc-stylesheet.css</stylesheetfile>
<links>
<link>http://download.oracle.com/javase/6/docs/api/</link>
</links>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/f24896b9/src/site/javadoc-stylesheet.css
----------------------------------------------------------------------
diff --git a/src/site/javadoc-stylesheet.css b/src/site/javadoc-stylesheet.css
deleted file mode 100644
index 0e4fe3d..0000000
--- a/src/site/javadoc-stylesheet.css
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/* Javadoc style sheet */
-
-/* Define colors, fonts and other style attributes here to override the defaults */
-
-/* Page background color */
-body { background-color: #FFFFFF; color:#111; font-size: 100%; }
-
-body { font-size: 1em; line-height: 1.286em; font-family: "Trebuchet MS", "Helvetica", "Arial", sans-serif; }
-
-code { line-height: 1.286em; font-family: "Consolas", "Lucida Console", "Droid Sans Mono", "Andale Mono", "Monaco", "Lucida Sans Typewriter"; }
-
-a { text-decoration: none; color: #162d50; /* also try #2E85ED, #0033FF, #6C93C6, #1D7BBE, #1D8DD2 */ }
-a:hover { text-decoration: underline; }
-a:visited { color: #2f4e7c; }
-a:visited:hover { text-decoration: underline; }
-
-table[border="1"] { border: 1px solid #ddd;
-border-width: 1px 0px 0px 1px;
-border-radius: 3px;
-}
-table[border="1"] .TableRowColor td, table[border="1"] th { border: 1px solid #ddd; border-width: 0px 1px 1px 0px; }
-table[border="1"] th { font-size: 12px; padding: 8px 12px; }
-table[cellpadding="3"] td { padding: 0.5em; }
-
-font[size="-1"] { font-size: 0.9em; line-height: 1.5em; }
-font[size="-2"] { font-size: 0.8em; }
-font[size="+2"] { font-size: 1.4em; line-height: 1.3em; padding: 0.4em 0; }
-
-hr {
- border: 0;
- border-bottom: 1px dashed #ddd;
-}
-
-/* Headings */
-h1 { font-size: 1.5em; line-height: 1.286em;}
-
-h2 {
- font-weight: 500;
-}
-h2 b {
- font-weight: 500;
-}
-
-/* Table colors */
-.TableHeadingColor { background-color:#ddd; } /* Dark mauve */
-.TableSubHeadingColor { background: #ddd; color:#444; } /* Light mauve */
-.TableRowColor { background: #FFFFFF; color:#333; font-size: 0.95em; } /* White */
-.TableRowColor a { }
-.TableRowColor code { color:#000; } /* White */
-
-/* Font used in left-hand frame lists */
-.FrameTitleFont { font-size: 100%; }
-.FrameHeadingFont { font-size: 90%; }
-.FrameItemFont { font-size: 0.9em; line-height: 1.3em;
-}
-/* Java Interfaces */
-.FrameItemFont a i {
- color: #2f4e7c;
-}
-.FrameItemFont a:hover i {
-
-}
-
-/* Navigation bar fonts and colors */
-.NavBarCell1 { padding: 4px 8px; background-color:#162d50; color: #FFFFFF; } /* Light mauve */
-.NavBarCell1Rev { padding: 4px 8px; background-color:#49b1d2; color:#FFFFFF} /* Dark Blue */
-.NavBarFont1 { color: #FFFFFF;}
-.NavBarFont1Rev { color:#FFFFFF; }
-
-.NavBarCell2 { background-color:#FFFFFF; color:#000000}
-.NavBarCell3 { background-color:#FFFFFF; color:#000000}
-
-li {
- margin-left: 25px;
-}
-
-pre {
-font-size: 1.1em;
-padding: 8px 16px;
-border: 1px solid #e0e0e0;
-border-radius: 3px;
-}
\ No newline at end of file
[07/25] incubator-twill git commit: Add more missing JavaDoc class
header information.
Posted by ch...@apache.org.
Add more missing JavaDoc class header information.
This closes #37 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ca33245c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ca33245c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ca33245c
Branch: refs/heads/site
Commit: ca33245cb086d9598479f1317d59e5dc26bf0ca0
Parents: d8462a4
Author: hsaputra <hs...@apache.org>
Authored: Mon Jun 1 21:37:34 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jun 12 18:04:48 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/twill/internal/BasicTwillContext.java | 3 ++-
.../java/org/apache/twill/internal/ProcessLauncher.java | 2 +-
.../org/apache/twill/internal/TwillContainerLauncher.java | 10 ++++++++++
.../org/apache/twill/internal/logging/KafkaAppender.java | 2 +-
.../apache/twill/internal/yarn/Hadoop21YarnAMClient.java | 2 +-
.../apache/twill/internal/yarn/Hadoop21YarnNMClient.java | 2 +-
.../internal/appmaster/ApplicationMasterService.java | 2 +-
.../twill/internal/container/TwillContainerMain.java | 2 +-
8 files changed, 18 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/BasicTwillContext.java b/twill-core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
index 94e6a69..370c3d3 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
@@ -20,6 +20,7 @@ package org.apache.twill.internal;
import org.apache.twill.api.ElectionHandler;
import org.apache.twill.api.RunId;
import org.apache.twill.api.TwillContext;
+import org.apache.twill.api.TwillRunnable;
import org.apache.twill.api.TwillRunnableSpecification;
import org.apache.twill.common.Cancellable;
import org.apache.twill.discovery.Discoverable;
@@ -34,7 +35,7 @@ import java.net.InetSocketAddress;
import java.util.concurrent.locks.Lock;
/**
- *
+ * Implementation of {@link TwillContext} that provides the basic runtime information of a {@link TwillRunnable}.
*/
public final class BasicTwillContext implements TwillContext {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ProcessLauncher.java b/twill-core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
index 098b90e..d0f289b 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
@@ -45,7 +45,7 @@ public interface ProcessLauncher<T> {
interface PrepareLaunchContext {
/**
- * Adds list of files to be localized for the container
+ * Adds list of files to be localized for the container.
*/
PrepareLaunchContext addResources(LocalFile...localFiles);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java b/twill-core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
index 02606cd..0667bb4 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
@@ -68,6 +68,16 @@ public final class TwillContainerLauncher {
this.secureStoreLocation = secureStoreLocation;
}
+ /**
+ * Start execution run for a class in a container. Will return instance of {@link TwillContainerController}
+ * that allows sending messages directly to the container.
+ *
+ * @param runId Use to represent unique id of the container run.
+ * @param instanceId The Twill instance Id.
+ * @param mainClass The main class to run in the container.
+ * @param classPath The class path to load classes for the container.
+ * @return instance of {@link TwillContainerController} to control the container run.
+ */
public TwillContainerController start(RunId runId, int instanceId, Class<?> mainClass, String classPath) {
// Clean up zookeeper path in case this is a retry and there are old messages and state there.
Futures.getUnchecked(ZKOperations.ignoreError(
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java b/twill-core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
index ff74677..1f8d02d 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
@@ -31,8 +31,8 @@ import com.google.common.util.concurrent.Service;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.twill.api.logging.LogThrowable;
-import org.apache.twill.internal.Services;
import org.apache.twill.common.Threads;
+import org.apache.twill.internal.Services;
import org.apache.twill.internal.json.ILoggingEventSerializer;
import org.apache.twill.internal.json.LogThrowableCodec;
import org.apache.twill.internal.json.StackTraceElementCodec;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
index c5ac458..82f428a 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
@@ -40,7 +40,7 @@ import java.util.List;
import javax.annotation.Nullable;
/**
- *
+ * Wrapper class for {@link AMRMClient} for Hadoop version 2.1 or greater.
*/
public class Hadoop21YarnAMClient extends AbstractYarnAMClient<AMRMClient.ContainerRequest> {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
index b8c9088..8c5f0fc 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
@@ -30,7 +30,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- *
+ * Wrapper class for {@link NMClient} for Hadoop version 2.1 or greater.
*/
public final class Hadoop21YarnNMClient extends AbstractIdleService implements YarnNMClient {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index f248750..b85ce97 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -94,7 +94,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
/**
- *
+ * The class that acts as {@code ApplicationMaster} for Twill applications.
*/
public final class ApplicationMasterService extends AbstractYarnTwillService implements Supplier<ResourceReport> {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ca33245c/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
index a29b27c..e04c8f9 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -58,7 +58,7 @@ import java.io.IOException;
import java.io.Reader;
/**
- *
+ * The main class for launching a {@link TwillContainerService}.
*/
public final class TwillContainerMain extends ServiceMain {
[14/25] incubator-twill git commit: Remove final modifier in
ResourceSpecification#BASIC since all fields in an interface are implicitly
public, static and final.
Posted by ch...@apache.org.
Remove final modifier in ResourceSpecification#BASIC since all fields in an interface are implicitly public, static and final.
This closes #50 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/43b080d9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/43b080d9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/43b080d9
Branch: refs/heads/site
Commit: 43b080d95fbff9efcceaab6df016889d0be9a9e1
Parents: d864ed1
Author: hsaputra <hs...@apache.org>
Authored: Wed Jun 24 10:08:31 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Wed Jun 24 14:34:26 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/twill/api/ResourceSpecification.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/43b080d9/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java b/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
index 2865caf..50d4ed4 100644
--- a/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
+++ b/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
@@ -25,7 +25,7 @@ import org.apache.twill.internal.DefaultResourceSpecification;
*/
public interface ResourceSpecification {
- final ResourceSpecification BASIC = Builder.with().setVirtualCores(1).setMemory(512, SizeUnit.MEGA).build();
+ ResourceSpecification BASIC = Builder.with().setVirtualCores(1).setMemory(512, SizeUnit.MEGA).build();
/**
* Unit for specifying memory size.
[18/25] incubator-twill git commit: Cleanup on Apache Twill code.
Posted by ch...@apache.org.
Cleanup on Apache Twill code.
When doing previous PR I saw some small cleanups opportunity:
1. Remove unnecessary public meodifiers in the interfaces method definition because they automatically public.
2. All field members declared in an interface are by default public, static and final so it is redundant to say it again.
3. Remove unused imports.
4. Remove private modifier in enum constructors bc they inherently private.
5. Remove static modifier for inner classes for interface bc they inherently static.
This closes #53 on GitHub.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/3a0f6e1a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/3a0f6e1a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/3a0f6e1a
Branch: refs/heads/site
Commit: 3a0f6e1a1645b79f6d8008f6ba3c7d91edd4447c
Parents: d4a1508
Author: hsaputra <hs...@apache.org>
Authored: Tue Jul 14 10:11:55 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 14 18:03:10 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/twill/api/ResourceReport.java | 10 +++++-----
.../java/org/apache/twill/api/ResourceSpecification.java | 4 ++--
.../main/java/org/apache/twill/filesystem/Location.java | 2 +-
.../java/org/apache/twill/kafka/client/Compression.java | 2 +-
.../org/apache/twill/kafka/client/KafkaPublisher.java | 2 +-
.../java/org/apache/twill/example/yarn/HelloWorld.java | 5 +----
.../internal/appmaster/RunnableContainerRequest.java | 1 -
.../apache/twill/internal/appmaster/TrackerService.java | 1 -
8 files changed, 11 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-api/src/main/java/org/apache/twill/api/ResourceReport.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/ResourceReport.java b/twill-api/src/main/java/org/apache/twill/api/ResourceReport.java
index 750376a..75271c7 100644
--- a/twill-api/src/main/java/org/apache/twill/api/ResourceReport.java
+++ b/twill-api/src/main/java/org/apache/twill/api/ResourceReport.java
@@ -32,33 +32,33 @@ public interface ResourceReport {
* @param runnableName the runnable name.
* @return resources being used by all instances of the runnable.
*/
- public Collection<TwillRunResources> getRunnableResources(String runnableName);
+ Collection<TwillRunResources> getRunnableResources(String runnableName);
/**
* Get all the run resources being used across all runnables.
*
* @return all run resources used by all instances of all runnables.
*/
- public Map<String, Collection<TwillRunResources>> getResources();
+ Map<String, Collection<TwillRunResources>> getResources();
/**
* Get the resources application master is using.
*
* @return resources being used by the application master.
*/
- public TwillRunResources getAppMasterResources();
+ TwillRunResources getAppMasterResources();
/**
* Get the id of the application master.
*
* @return id of the application master.
*/
- public String getApplicationId();
+ String getApplicationId();
/**
* Get the list of services of the application master.
*
* @return list of services of the application master.
*/
- public List<String> getServices();
+ List<String> getServices();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java b/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
index 50d4ed4..d8d873f 100644
--- a/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
+++ b/twill-api/src/main/java/org/apache/twill/api/ResourceSpecification.java
@@ -36,7 +36,7 @@ public interface ResourceSpecification {
private final int multiplier;
- private SizeUnit(int multiplier) {
+ SizeUnit(int multiplier) {
this.multiplier = multiplier;
}
}
@@ -81,7 +81,7 @@ public interface ResourceSpecification {
/**
* Builder for creating {@link ResourceSpecification}.
*/
- static final class Builder {
+ final class Builder {
private int cores;
private int memory;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-common/src/main/java/org/apache/twill/filesystem/Location.java
----------------------------------------------------------------------
diff --git a/twill-common/src/main/java/org/apache/twill/filesystem/Location.java b/twill-common/src/main/java/org/apache/twill/filesystem/Location.java
index 04edd27..2560e55 100644
--- a/twill-common/src/main/java/org/apache/twill/filesystem/Location.java
+++ b/twill-common/src/main/java/org/apache/twill/filesystem/Location.java
@@ -34,7 +34,7 @@ public interface Location {
/**
* Suffix added to every temp file name generated with {@link #getTempFile(String)}.
*/
- static final String TEMP_FILE_SUFFIX = ".tmp";
+ String TEMP_FILE_SUFFIX = ".tmp";
/**
* Checks if the this location exists.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-core/src/main/java/org/apache/twill/kafka/client/Compression.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/kafka/client/Compression.java b/twill-core/src/main/java/org/apache/twill/kafka/client/Compression.java
index 8597d3b..d3e51bf 100644
--- a/twill-core/src/main/java/org/apache/twill/kafka/client/Compression.java
+++ b/twill-core/src/main/java/org/apache/twill/kafka/client/Compression.java
@@ -28,7 +28,7 @@ public enum Compression {
private final String codec;
- private Compression() {
+ Compression() {
this.codec = name().toLowerCase();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-core/src/main/java/org/apache/twill/kafka/client/KafkaPublisher.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/kafka/client/KafkaPublisher.java b/twill-core/src/main/java/org/apache/twill/kafka/client/KafkaPublisher.java
index 0be45cb..f096375 100644
--- a/twill-core/src/main/java/org/apache/twill/kafka/client/KafkaPublisher.java
+++ b/twill-core/src/main/java/org/apache/twill/kafka/client/KafkaPublisher.java
@@ -73,7 +73,7 @@ public interface KafkaPublisher {
private final int ack;
- private Ack(int ack) {
+ Ack(int ack) {
this.ack = ack;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
index a9cbba8..ff1b83c 100644
--- a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
+++ b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
@@ -104,10 +104,7 @@ public class HelloWorld {
@Override
public boolean accept(String className, URL classUrl, URL classPathUrl) {
// exclude hadoop but not hbase package
- if (className.startsWith("org.apache.hadoop") && !className.startsWith("org.apache.hadoop.hbase")) {
- return false;
- }
- return true;
+ return !(className.startsWith("org.apache.hadoop") && !className.startsWith("org.apache.hadoop.hbase"));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
index f065380..2105629 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
@@ -26,7 +26,6 @@ import org.apache.twill.api.TwillSpecification;
import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
/**
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/3a0f6e1a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
index 7c09c58..d3f7c86 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
@@ -64,7 +64,6 @@ import java.net.URL;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
-import javax.ws.rs.GET;
/**
* Webservice that the Application Master will register back to the resource manager
[21/25] incubator-twill git commit: Merge branch 'branch-0.6.0'
Posted by ch...@apache.org.
Merge branch 'branch-0.6.0'
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/c1d977b4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/c1d977b4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/c1d977b4
Branch: refs/heads/site
Commit: c1d977b482b6caa2a149f609db8dd2f7ca3bbb68
Parents: 9495e30 518cc1a
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jul 14 22:40:51 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 14 22:40:51 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
twill-api/pom.xml | 2 +-
twill-common/pom.xml | 2 +-
twill-core/pom.xml | 2 +-
twill-discovery-api/pom.xml | 2 +-
twill-discovery-core/pom.xml | 2 +-
twill-examples/echo/pom.xml | 2 +-
twill-examples/pom.xml | 2 +-
twill-examples/yarn/pom.xml | 2 +-
twill-ext/pom.xml | 2 +-
twill-java8-test/pom.xml | 2 +-
twill-yarn/pom.xml | 2 +-
twill-zookeeper/pom.xml | 2 +-
13 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[20/25] incubator-twill git commit: Prepare for releasing
0.6.0-incubating
Posted by ch...@apache.org.
Prepare for releasing 0.6.0-incubating
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/518cc1a2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/518cc1a2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/518cc1a2
Branch: refs/heads/site
Commit: 518cc1a24c1ae29e46b58debe0f1a177d1589321
Parents: 9495e30
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jul 14 20:04:05 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 14 20:04:05 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
twill-api/pom.xml | 2 +-
twill-common/pom.xml | 2 +-
twill-core/pom.xml | 2 +-
twill-discovery-api/pom.xml | 2 +-
twill-discovery-core/pom.xml | 2 +-
twill-examples/echo/pom.xml | 2 +-
twill-examples/pom.xml | 2 +-
twill-examples/yarn/pom.xml | 2 +-
twill-ext/pom.xml | 2 +-
twill-java8-test/pom.xml | 2 +-
twill-yarn/pom.xml | 2 +-
twill-zookeeper/pom.xml | 2 +-
13 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1556534..e026296 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
<packaging>pom</packaging>
<name>Apache Twill</name>
<url>http://twill.incubator.apache.org</url>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-api/pom.xml b/twill-api/pom.xml
index f36444e..102c1d0 100644
--- a/twill-api/pom.xml
+++ b/twill-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<artifactId>twill-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-common/pom.xml
----------------------------------------------------------------------
diff --git a/twill-common/pom.xml b/twill-common/pom.xml
index 0d1f338..cc3030b 100644
--- a/twill-common/pom.xml
+++ b/twill-common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-core/pom.xml b/twill-core/pom.xml
index 075630b..8f1de96 100644
--- a/twill-core/pom.xml
+++ b/twill-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-discovery-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-api/pom.xml b/twill-discovery-api/pom.xml
index b07d960..589022d 100644
--- a/twill-discovery-api/pom.xml
+++ b/twill-discovery-api/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-discovery-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-core/pom.xml b/twill-discovery-core/pom.xml
index b29af09..8159250 100644
--- a/twill-discovery-core/pom.xml
+++ b/twill-discovery-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-examples/echo/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/echo/pom.xml b/twill-examples/echo/pom.xml
index 51db27b..634a1ac 100644
--- a/twill-examples/echo/pom.xml
+++ b/twill-examples/echo/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<name>Apache Twill examples: Echo</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-examples/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/pom.xml b/twill-examples/pom.xml
index 30a8bd9..ce968bb 100644
--- a/twill-examples/pom.xml
+++ b/twill-examples/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<artifactId>twill-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-examples/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/pom.xml b/twill-examples/yarn/pom.xml
index 6f3c805..17e7a33 100644
--- a/twill-examples/yarn/pom.xml
+++ b/twill-examples/yarn/pom.xml
@@ -24,7 +24,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<name>Apache Twill examples: YARN</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-ext/pom.xml
----------------------------------------------------------------------
diff --git a/twill-ext/pom.xml b/twill-ext/pom.xml
index 976828f..0f1d674 100644
--- a/twill-ext/pom.xml
+++ b/twill-ext/pom.xml
@@ -22,7 +22,7 @@ limitations under the License.
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-java8-test/pom.xml
----------------------------------------------------------------------
diff --git a/twill-java8-test/pom.xml b/twill-java8-test/pom.xml
index 9f020f8..8f45842 100644
--- a/twill-java8-test/pom.xml
+++ b/twill-java8-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<artifactId>twill-java8-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-yarn/pom.xml b/twill-yarn/pom.xml
index 4c6abff..117a505 100644
--- a/twill-yarn/pom.xml
+++ b/twill-yarn/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/518cc1a2/twill-zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/twill-zookeeper/pom.xml b/twill-zookeeper/pom.xml
index 0d2edda..5a98f95 100644
--- a/twill-zookeeper/pom.xml
+++ b/twill-zookeeper/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating-SNAPSHOT</version>
+ <version>0.6.0-incubating</version>
</parent>
<modelVersion>4.0.0</modelVersion>
[03/25] incubator-twill git commit: Add JavaDoc class header to give
short description.
Posted by ch...@apache.org.
Add JavaDoc class header to give short description.
Adding JavaDoc class header for several classes to help describe why and what they exist.
This closes #36 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/70b68588
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/70b68588
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/70b68588
Branch: refs/heads/site
Commit: 70b6858857e763336e2bcf9a11110f34962db735
Parents: 2175c66
Author: hsaputra <hs...@apache.org>
Authored: Thu May 28 13:18:50 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Thu May 28 13:56:19 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/twill/internal/state/Message.java | 2 +-
.../main/java/org/apache/twill/internal/state/MessageCallback.java | 2 +-
.../main/java/org/apache/twill/internal/state/SimpleMessage.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/70b68588/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/state/Message.java b/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
index 6c3e719..c6944fd 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
@@ -20,7 +20,7 @@ package org.apache.twill.internal.state;
import org.apache.twill.api.Command;
/**
- *
+ * Represent message objects to communicate between components in Twill.
*/
public interface Message {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/70b68588/twill-core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/state/MessageCallback.java b/twill-core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
index f94eaa3..4413a21 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
@@ -20,7 +20,7 @@ package org.apache.twill.internal.state;
import com.google.common.util.concurrent.ListenableFuture;
/**
- *
+ * Define interface that could provide override-able callback method when a message is received.
*/
public interface MessageCallback {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/70b68588/twill-core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java b/twill-core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
index 9cfbe29..0465d46 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
@@ -21,7 +21,7 @@ import com.google.common.base.Objects;
import org.apache.twill.api.Command;
/**
- *
+ * Implementation of {@code Message} interface to pass information about {@code Command} to execute.
*/
final class SimpleMessage implements Message {
[17/25] incubator-twill git commit: TWILL-116 Add TwillRunnable
instances lifecycle management.
Posted by ch...@apache.org.
TWILL-116 Add TwillRunnable instances lifecycle management.
Add support to restart certain or all instances of runnables in a Twill application.
See proposed design attached to the JIRA: https://issues.apache.org/jira/browse/TWILL-116
Summary of changes:
1. Add new APIs to TwillController to restart all or certain instances of runnables.
2. Support message handler in ApplicationMasterService for restart instances.
3. Modify RunningContainers to launch new container using same instance id to support restart for a runnable.
4. Add validation for instance ids to be restarted.
This closes #52 on GitHub.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d4a1508e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d4a1508e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d4a1508e
Branch: refs/heads/site
Commit: d4a1508eeacbebdff5263b1ecad182775f27162a
Parents: d181b7c
Author: hsaputra <hs...@apache.org>
Authored: Fri Jun 26 22:19:24 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jul 13 18:33:30 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/TwillController.java | 28 +++++++
.../twill/internal/AbstractTwillController.java | 81 +++++++++++++++++++-
.../internal/AbstractZKServiceController.java | 1 +
.../org/apache/twill/internal/Constants.java | 5 ++
.../org/apache/twill/internal/ZKMessages.java | 2 +-
.../apache/twill/internal/state/Message.java | 3 +-
.../twill/internal/state/SystemMessages.java | 27 ++++++-
.../yarn/Hadoop21YarnContainerInfo.java | 2 +-
.../org/apache/twill/internal/ServiceMain.java | 2 +-
.../appmaster/ApplicationMasterService.java | 79 +++++++++++++++++++
.../appmaster/RunnableContainerRequest.java | 1 +
.../internal/appmaster/RunningContainers.java | 65 ++++++++++------
.../apache/twill/yarn/EchoServerTestRun.java | 70 +++++++++++++++++
13 files changed, 338 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-api/src/main/java/org/apache/twill/api/TwillController.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillController.java b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
index b84f817..08206f5 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillController.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
@@ -21,6 +21,8 @@ import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.discovery.Discoverable;
import org.apache.twill.discovery.ServiceDiscovered;
+import java.util.Map;
+import java.util.Set;
import java.util.concurrent.Future;
import javax.annotation.Nullable;
@@ -61,4 +63,30 @@ public interface TwillController extends ServiceController {
*/
@Nullable
ResourceReport getResourceReport();
+
+ /**
+ * Restart all instances of a particular {@link TwillRunnable}.
+ *
+ * @param runnable The name of the runnable to restart.
+ * @return A {@link Future} that will be completed when the restart operation has been done.
+ */
+ Future<String> restartAllInstances(String runnable);
+
+ /**
+ * Restart instances of some {@link TwillRunnable}.
+ *
+ * @param runnableToInstanceIds A map of runnable ID to list of instance IDs to be restarted.
+ * @return A {@link Future} that will be completed when the restart operation has been done.
+ */
+ Future<Set<String>> restartInstances(Map<String, ? extends Set<Integer>> runnableToInstanceIds);
+
+ /**
+ * Restart instances of some {@link TwillRunnable}.
+ *
+ * @param runnable The name of the runnable to restart.
+ * @param instanceId The main instance id to be restarted.
+ * @param moreInstanceIds The optional instance ids.
+ * @return A {@link Future} that will be completed when the restart operation has been done.
+ */
+ Future<String> restartInstances(String runnable, int instanceId, int... moreInstanceIds);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/AbstractTwillController.java b/twill-core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
index bf20616..41a044b 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
@@ -18,12 +18,22 @@
package org.apache.twill.internal;
import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.reflect.TypeToken;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.RunId;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.api.logging.LogThrowable;
@@ -35,6 +45,7 @@ import org.apache.twill.internal.json.LogEntryDecoder;
import org.apache.twill.internal.json.LogThrowableCodec;
import org.apache.twill.internal.json.StackTraceElementCodec;
import org.apache.twill.internal.kafka.client.ZKKafkaClientService;
+import org.apache.twill.internal.state.Message;
import org.apache.twill.internal.state.SystemMessages;
import org.apache.twill.kafka.client.FetchedMessage;
import org.apache.twill.kafka.client.KafkaClientService;
@@ -44,8 +55,11 @@ import org.apache.twill.zookeeper.ZKClients;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.Collection;
import java.util.Iterator;
+import java.util.Map;
import java.util.Queue;
+import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
/**
@@ -55,6 +69,7 @@ import java.util.concurrent.ConcurrentLinkedQueue;
public abstract class AbstractTwillController extends AbstractZKServiceController implements TwillController {
private static final Logger LOG = LoggerFactory.getLogger(AbstractTwillController.class);
+ private static final Gson GSON = new Gson();
private final Queue<LogHandler> logHandlers;
private final KafkaClientService kafkaClient;
@@ -63,7 +78,7 @@ public abstract class AbstractTwillController extends AbstractZKServiceControlle
public AbstractTwillController(RunId runId, ZKClient zkClient, Iterable<LogHandler> logHandlers) {
super(runId, zkClient);
- this.logHandlers = new ConcurrentLinkedQueue<LogHandler>();
+ this.logHandlers = new ConcurrentLinkedQueue<>();
this.kafkaClient = new ZKKafkaClientService(ZKClients.namespace(zkClient, "/" + runId.getId() + "/kafka"));
this.discoveryServiceClient = new ZKDiscoveryService(zkClient);
Iterables.addAll(this.logHandlers, logHandlers);
@@ -109,6 +124,70 @@ public abstract class AbstractTwillController extends AbstractZKServiceControlle
return sendMessage(SystemMessages.setInstances(runnable, newCount), newCount);
}
+ @Override
+ public final ListenableFuture<String> restartAllInstances(String runnableName) {
+ Command updateStateCommand = Command.Builder.of(Constants.RESTART_ALL_RUNNABLE_INSTANCES).
+ build();
+ Message message = SystemMessages.updateRunnableInstances(updateStateCommand, runnableName);
+ return sendMessage(message, updateStateCommand.getCommand());
+ }
+
+ @Override
+ public final ListenableFuture<Set<String>> restartInstances(Map<String,
+ ? extends Set<Integer>> runnableToInstanceIds) {
+ Map<String, String> runnableToStringInstanceIds =
+ Maps.transformEntries(runnableToInstanceIds, new Maps.EntryTransformer<String, Set<Integer>, String>() {
+ @Override
+ public String transformEntry(String runnableName, Set<Integer> instanceIds) {
+ validateInstanceIds(runnableName, instanceIds);
+ return GSON.toJson(instanceIds, new TypeToken<Set<Integer>>() {}.getType());
+ }
+ });
+ Command updateStateCommand = Command.Builder.of(Constants.RESTART_RUNNABLES_INSTANCES)
+ .addOptions(runnableToStringInstanceIds)
+ .build();
+ Message message = SystemMessages.updateRunnablesInstances(updateStateCommand);
+
+ return sendMessage(message, runnableToInstanceIds.keySet());
+ }
+
+ @Override
+ public ListenableFuture<String> restartInstances(final String runnable, int instanceId, int... moreInstanceIds) {
+ Set<Integer> instanceIds = Sets.newLinkedHashSet();
+ instanceIds.add(instanceId);
+ for (int id : moreInstanceIds) {
+ instanceIds.add(id);
+ }
+
+ return Futures.transform(restartInstances(ImmutableMap.of(runnable, instanceIds)),
+ new Function<Set<String>, String>() {
+ public String apply(Set<String> input) {
+ return runnable;
+ }
+ });
+ }
+
+ private void validateInstanceIds(String runnable, Set<Integer> instanceIds) {
+ ResourceReport resourceReport = getResourceReport();
+ if (resourceReport == null) {
+ throw new IllegalStateException("Unable to get resource report since application has not started.");
+ }
+ Collection<TwillRunResources> runnableResources = resourceReport.getRunnableResources(runnable);
+ if (runnableResources == null) {
+ throw new RuntimeException("Unable to verify run resources for runnable " + runnable);
+ }
+ Set<Integer> existingInstanceIds = Sets.newHashSet();
+ for (TwillRunResources twillRunResources : runnableResources) {
+ existingInstanceIds.add(twillRunResources.getInstanceId());
+ }
+ LOG.info("Existing instance ids: {}", existingInstanceIds);
+ for (int instanceId : instanceIds) {
+ if (!existingInstanceIds.contains(instanceId)) {
+ throw new IllegalArgumentException("Unable to find instance id " + instanceId + " for " + runnable);
+ }
+ }
+ }
+
private static final class LogMessageCallback implements KafkaConsumer.MessageCallback {
private static final Gson GSON = new GsonBuilder()
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java b/twill-core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
index 6d95009..0cf92ea 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
@@ -21,6 +21,7 @@ import com.google.common.collect.Lists;
import com.google.common.util.concurrent.FutureCallback;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
+
import org.apache.twill.api.Command;
import org.apache.twill.api.RunId;
import org.apache.twill.api.ServiceController;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/Constants.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/Constants.java b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
index 64b029d..e3a2194 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/Constants.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
@@ -47,6 +47,11 @@ public final class Constants {
public static final String CLASSPATH = "classpath";
public static final String APPLICATION_CLASSPATH = "application-classpath";
+
+ /** For runnables instance lifecycle ZK path */
+ public static final String RESTART_ALL_RUNNABLE_INSTANCES = "restartAllRunnableInstances";
+ public static final String RESTART_RUNNABLES_INSTANCES = "restartRunnablesInstances";
+
/**
* Constants for names of internal files that are shared between client, AM and containers.
*/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/ZKMessages.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ZKMessages.java b/twill-core/src/main/java/org/apache/twill/internal/ZKMessages.java
index b7905d9..cfcc7e7 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ZKMessages.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ZKMessages.java
@@ -28,7 +28,7 @@ import org.apache.twill.zookeeper.ZKOperations;
import org.apache.zookeeper.CreateMode;
/**
- *
+ * Helper class to send messages to remote instances using Apache Zookeeper watch mechanism.
*/
public final class ZKMessages {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/state/Message.java b/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
index c6944fd..1c758c1 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/state/Message.java
@@ -38,7 +38,8 @@ public interface Message {
enum Scope {
APPLICATION,
ALL_RUNNABLE,
- RUNNABLE
+ RUNNABLE,
+ RUNNABLES
}
Type getType();
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/state/SystemMessages.java b/twill-core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
index 73683bd..4c4bb8b 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
@@ -17,9 +17,10 @@
*/
package org.apache.twill.internal.state;
-import com.google.common.base.Preconditions;
import org.apache.twill.api.Command;
+import com.google.common.base.Preconditions;
+
/**
* Collection of predefined system messages.
*/
@@ -43,6 +44,30 @@ public final class SystemMessages {
Command.Builder.of("instances").addOption("count", Integer.toString(instances)).build());
}
+ /**
+ * Helper method to get System {@link Message} for update instances for a runnable.
+ *
+ * @param updateCommand The {@link Command} to be added to the message.
+ * @param runnableName The name of the runnable to be restarted.
+ * @return An instance of System {@link Message} to restart runnable instances.
+ */
+ public static Message updateRunnableInstances(Command updateCommand, String runnableName) {
+ Preconditions.checkNotNull(updateCommand);
+ Preconditions.checkNotNull(runnableName);
+ return new SimpleMessage(Message.Type.SYSTEM, Message.Scope.RUNNABLE, runnableName, updateCommand);
+ }
+
+ /**
+ * Helper method to get System {@link Message} for restarting certain instances from certain runnables.
+ *
+ * @param updateCommand The {@link Command} to be added to the message.
+ * @return An instance of System {@link Message} to restart runnables' instances.
+ */
+ public static Message updateRunnablesInstances(Command updateCommand) {
+ Preconditions.checkNotNull(updateCommand);
+ return new SimpleMessage(Message.Type.SYSTEM, Message.Scope.RUNNABLES, null, updateCommand);
+ }
+
private SystemMessages() {
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
index 86903c1..8571933 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
@@ -24,7 +24,7 @@ import java.net.InetAddress;
import java.net.UnknownHostException;
/**
- *
+ * Implementation of {@link YarnContainerInfo} for Apache Hadoop 2.1.0 or above.
*/
public final class Hadoop21YarnContainerInfo implements YarnContainerInfo {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
index d7ef72b..f7bea24 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -228,7 +228,7 @@ public abstract class ServiceMain {
}
/**
- * Override to return the right log level for the service
+ * Override to return the right log level for the service.
*
* @param logger the {@link Logger} instance of the service context.
* @return String of log level based on {@code slf4j} log levels.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index 5c89a79..f76cd0b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -20,14 +20,17 @@ package org.apache.twill.internal.appmaster;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
+import com.google.common.base.Strings;
import com.google.common.base.Supplier;
import com.google.common.base.Throwables;
+import com.google.common.collect.DiscreteDomains;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multiset;
+import com.google.common.collect.Ranges;
import com.google.common.collect.Sets;
import com.google.common.io.Files;
import com.google.common.io.InputSupplier;
@@ -35,6 +38,7 @@ import com.google.common.reflect.TypeToken;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.SettableFuture;
+import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -92,6 +96,7 @@ import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import javax.annotation.Nullable;
/**
* The class that acts as {@code ApplicationMaster} for Twill applications.
@@ -99,6 +104,7 @@ import java.util.concurrent.TimeUnit;
public final class ApplicationMasterService extends AbstractYarnTwillService implements Supplier<ResourceReport> {
private static final Logger LOG = LoggerFactory.getLogger(ApplicationMasterService.class);
+ private static final Gson GSON = new Gson();
// Copied from org.apache.hadoop.yarn.security.AMRMTokenIdentifier.KIND_NAME since it's missing in Hadoop-2.0
private static final Text AMRM_TOKEN_KIND_NAME = new Text("YARN_AM_RM_TOKEN");
@@ -288,6 +294,10 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
return result;
}
+ if (handleRestartRunnablesInstances(message, completion)) {
+ return result;
+ }
+
// Replicate messages to all runnables
if (message.getScope() == Message.Scope.ALL_RUNNABLE) {
runningContainers.sendToAll(message, completion);
@@ -787,4 +797,73 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
capability.setMemory(resourceSpec.getMemorySize());
return capability;
}
+
+ /**
+ * Attempt to restart some instances from a runnable or some runnables.
+ * @return {@code true} if the message requests restarting some instances and {@code false} otherwise.
+ */
+ private boolean handleRestartRunnablesInstances(final Message message, final Runnable completion) {
+ LOG.debug("Check if it should process a restart runnable instances.");
+
+ if (message.getType() != Message.Type.SYSTEM) {
+ return false;
+ }
+
+ Message.Scope messageScope = message.getScope();
+ if (messageScope != Message.Scope.RUNNABLE && messageScope != Message.Scope.RUNNABLES) {
+ return false;
+ }
+
+ Command requestCommand = message.getCommand();
+ if (!Constants.RESTART_ALL_RUNNABLE_INSTANCES.equals(requestCommand.getCommand()) &&
+ !Constants.RESTART_RUNNABLES_INSTANCES.equals(requestCommand.getCommand())) {
+ return false;
+ }
+
+ LOG.debug("Processing restart runnable instances message {}.", message);
+
+ if (!Strings.isNullOrEmpty(message.getRunnableName()) && message.getScope() == Message.Scope.RUNNABLE) {
+ // ... for a runnable ...
+ String runnableName = message.getRunnableName();
+ LOG.debug("Start restarting all runnable {} instances.", runnableName);
+ restartRunnableInstances(runnableName, null);
+ } else {
+ // ... or maybe some runnables
+ for (Map.Entry<String, String> option : requestCommand.getOptions().entrySet()) {
+ String runnableName = option.getKey();
+ Set<Integer> restartedInstanceIds = GSON.fromJson(option.getValue(),
+ new TypeToken<Set<Integer>>() {}.getType());
+
+ LOG.debug("Start restarting runnable {} instances {}", runnableName, restartedInstanceIds);
+ restartRunnableInstances(runnableName, restartedInstanceIds);
+ }
+ }
+
+ completion.run();
+ return true;
+ }
+
+ /**
+ * Helper method to restart instances of runnables.
+ */
+ private void restartRunnableInstances(String runnableName, @Nullable Set<Integer> instanceIds) {
+ LOG.debug("Begin restart runnable {} instances.", runnableName);
+
+ Set<Integer> instancesToRemove = instanceIds;
+ if (instancesToRemove == null) {
+ instancesToRemove = Ranges.closedOpen(0, runningContainers.count(runnableName)).asSet(DiscreteDomains.integers());
+ }
+
+ for (int instanceId : instancesToRemove) {
+ LOG.debug("Remove instance {} for runnable {}", instanceId, runnableName);
+ try {
+ runningContainers.removeById(runnableName, instanceId);
+ } catch (Exception ex) {
+ // could be thrown if the container already stopped.
+ LOG.info("Exception thrown when stopping instance {} probably already stopped.", instanceId);
+ }
+ }
+ LOG.info("Restarting instances {} for runnable {}", instancesToRemove, runnableName);
+ runnableContainerRequests.add(createRunnableContainerRequest(runnableName, instancesToRemove.size()));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
index 2105629..f065380 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
@@ -26,6 +26,7 @@ import org.apache.twill.api.TwillSpecification;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
/**
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
index 4a56229..d957768 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
@@ -121,6 +121,9 @@ final class RunningContainers {
}
}
+ /**
+ * Start a container for a runnable.
+ */
void start(String runnableName, ContainerInfo containerInfo, TwillContainerLauncher launcher) {
containerLock.lock();
try {
@@ -186,32 +189,47 @@ final class RunningContainers {
LOG.warn("No running container found for {}", runnableName);
return;
}
+ removeInstanceById(runnableName, maxInstanceId);
+ } finally {
+ containerLock.unlock();
+ }
+ }
- String lastContainerId = null;
- TwillContainerController lastController = null;
+ /**
+ * Stop and remove a container for a runnable on an id.
+ */
+ void removeById(String runnableName, int instanceId) {
+ containerLock.lock();
+ try {
+ removeInstanceById(runnableName, instanceId);
+ } finally {
+ containerLock.unlock();
+ }
+ }
- // Find the controller with the maxInstanceId
- for (Map.Entry<String, TwillContainerController> entry : containers.row(runnableName).entrySet()) {
- if (getInstanceId(entry.getValue().getRunId()) == maxInstanceId) {
- lastContainerId = entry.getKey();
- lastController = entry.getValue();
- break;
- }
+ private void removeInstanceById(String runnableName, int instanceId) {
+ String containerId = null;
+ TwillContainerController controller = null;
+
+ // Find the controller with particular instance id.
+ for (Map.Entry<String, TwillContainerController> entry : containers.row(runnableName).entrySet()) {
+ if (getInstanceId(entry.getValue().getRunId()) == instanceId) {
+ containerId = entry.getKey();
+ controller = entry.getValue();
+ break;
}
+ }
- Preconditions.checkState(lastContainerId != null,
- "No container found for {} with instanceId = {}", runnableName, maxInstanceId);
+ Preconditions.checkState(containerId != null,
+ "No container found for {} with instanceId = {}", runnableName, instanceId);
- LOG.info("Stopping service: {} {}", runnableName, lastController.getRunId());
- lastController.stopAndWait();
- containers.remove(runnableName, lastContainerId);
- removeContainerInfo(lastContainerId);
- removeInstanceId(runnableName, maxInstanceId);
- resourceReport.removeRunnableResources(runnableName, lastContainerId);
- containerChange.signalAll();
- } finally {
- containerLock.unlock();
- }
+ LOG.info("Stopping service: {} {}", runnableName, controller.getRunId());
+ controller.stopAndWait();
+ containers.remove(runnableName, containerId);
+ removeContainerInfo(containerId);
+ removeInstanceId(runnableName, instanceId);
+ resourceReport.removeRunnableResources(runnableName, containerId);
+ containerChange.signalAll();
}
/**
@@ -446,6 +464,9 @@ final class RunningContainers {
return instanceId;
}
+ /**
+ * Remove instance id for a given runnable.
+ */
private void removeInstanceId(String runnableName, int instanceId) {
BitSet instances = runnableInstances.get(runnableName);
if (instances == null) {
@@ -469,7 +490,7 @@ final class RunningContainers {
}
/**
- * Returns nnumber of running instances for the given runnable.
+ * Returns number of running instances for the given runnable.
*/
private int getRunningInstances(String runableName) {
BitSet instances = runnableInstances.get(runableName);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d4a1508e/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
index d635af1..0a8414e 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -18,9 +18,14 @@
package org.apache.twill.yarn;
import com.google.common.base.Charsets;
+import com.google.common.base.Stopwatch;
+import com.google.common.collect.Maps;
import com.google.common.io.LineReader;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.ResourceSpecification;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
import org.apache.twill.api.TwillRunner;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.logging.PrinterLogHandler;
@@ -37,10 +42,13 @@ import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.net.Socket;
import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import javax.annotation.Nullable;
/**
* Using echo server to test various behavior of YarnTwillService.
@@ -107,6 +115,22 @@ public final class EchoServerTestRun extends BaseYarnTest {
controller.changeInstances("EchoServer", 2);
Assert.assertTrue(waitForSize(echoServices, 2, 120));
+ // Test restart on instances for runnable
+ Map<Integer, String> instanceIdToContainerId = Maps.newHashMap();
+ ResourceReport report = waitForAfterRestartResourceReport(controller, "EchoServer", 30L, TimeUnit.SECONDS, 2, null);
+ Assert.assertTrue(report != null);
+ Collection<TwillRunResources> runResources = report.getRunnableResources("EchoServer");
+ for (TwillRunResources twillRunResources : runResources) {
+ instanceIdToContainerId.put(twillRunResources.getInstanceId(), twillRunResources.getContainerId());
+ }
+
+ controller.restartAllInstances("EchoServer");
+ Assert.assertTrue(waitForSize(echoServices, 2, 120));
+
+ report = waitForAfterRestartResourceReport(controller, "EchoServer", 30L, TimeUnit.SECONDS, 2,
+ instanceIdToContainerId);
+ Assert.assertTrue(report != null);
+
// Make sure still only one app is running
Iterable<TwillRunner.LiveInfo> apps = runner.lookupLive();
Assert.assertTrue(waitForSize(apps, 1, 120));
@@ -132,4 +156,50 @@ public final class EchoServerTestRun extends BaseYarnTest {
// Sleep a bit before exiting.
TimeUnit.SECONDS.sleep(2);
}
+
+ /**
+ * Need helper method here to wait for getting resource report because {@link TwillController#getResourceReport()}
+ * could return null if the application has not fully started.
+ *
+ * This method helps validate restart scenario.
+ *
+ * To avoid long sleep if instanceIdToContainerId is passed, then compare the container ids to ones before.
+ * Otherwise just return the valid resource report.
+ */
+ @Nullable
+ private ResourceReport waitForAfterRestartResourceReport(TwillController controller, String runnable, long timeout,
+ TimeUnit timeoutUnit, int numOfResources,
+ @Nullable Map<Integer, String> instanceIdToContainerId) {
+ Stopwatch stopwatch = new Stopwatch();
+ stopwatch.start();
+ do {
+ ResourceReport report = controller.getResourceReport();
+ if (report == null || report.getRunnableResources(runnable) == null) {
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ } else if (report.getRunnableResources(runnable) == null ||
+ report.getRunnableResources(runnable).size() != numOfResources) {
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ } else {
+ if (instanceIdToContainerId == null) {
+ return report;
+ }
+ Collection<TwillRunResources> runResources = report.getRunnableResources(runnable);
+ boolean isSameContainer = false;
+ for (TwillRunResources twillRunResources : runResources) {
+ int instanceId = twillRunResources.getInstanceId();
+ if (twillRunResources.getContainerId().equals(instanceIdToContainerId.get(instanceId))) {
+ // found same container id lets wait again.
+ isSameContainer = true;
+ break;
+ }
+ }
+ if (!isSameContainer) {
+ LOG.error("Unable to get different container ids for restart.");
+ return report;
+ }
+ Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
+ }
+ } while (stopwatch.elapsedTime(timeoutUnit) < timeout);
+ return null;
+ }
}
[08/25] incubator-twill git commit: (TWILL-133) Make the ZKClient
stoppable even it is still trying to connect
Posted by ch...@apache.org.
(TWILL-133) Make the ZKClient stoppable even it is still trying to connect
- Also include refactoring of the DefaultZKClient class to simplify
shutdown logic
- Fix a thread leaking bug in LeaderElection
This closes #41 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/2054c5f9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/2054c5f9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/2054c5f9
Branch: refs/heads/site
Commit: 2054c5f9364effabfa3008a3a9043b08b1d1b64c
Parents: ca33245
Author: Terence Yim <ch...@apache.org>
Authored: Fri Jun 12 17:13:05 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 15 19:20:41 2015 -0700
----------------------------------------------------------------------
.../zookeeper/DefaultZKClientService.java | 164 +++++++++++++------
.../internal/zookeeper/LeaderElection.java | 12 +-
.../apache/twill/zookeeper/ZKClientTest.java | 85 +++++++---
3 files changed, 185 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2054c5f9/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
index 7b9b345..dc2bfa9 100644
--- a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
+++ b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
@@ -58,7 +58,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nullable;
@@ -91,11 +90,11 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
Watcher connectionWatcher, Multimap<String, byte[]> authInfos) {
this.zkStr = zkStr;
this.sessionTimeout = sessionTimeout;
- this.connectionWatchers = new CopyOnWriteArrayList<Watcher>();
+ this.connectionWatchers = new CopyOnWriteArrayList<>();
this.authInfos = copyAuthInfo(authInfos);
addConnectionWatcher(connectionWatcher);
- this.zooKeeper = new AtomicReference<ZooKeeper>();
+ this.zooKeeper = new AtomicReference<>();
serviceDelegate = new ServiceDelegate();
}
@@ -111,7 +110,7 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
}
@Override
- public Cancellable addConnectionWatcher(Watcher watcher) {
+ public Cancellable addConnectionWatcher(final Watcher watcher) {
if (watcher == null) {
return new Cancellable() {
@Override
@@ -121,12 +120,13 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
};
}
- final Watcher wrappedWatcher = wrapWatcher(watcher);
- connectionWatchers.add(wrappedWatcher);
+ // Invocation of connection watchers are already done inside the event thread,
+ // hence no need to wrap the watcher again.
+ connectionWatchers.add(watcher);
return new Cancellable() {
@Override
public void cancel() {
- connectionWatchers.remove(wrappedWatcher);
+ connectionWatchers.remove(watcher);
}
};
}
@@ -391,25 +391,57 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
private final class ServiceDelegate extends AbstractService implements Watcher {
- private final AtomicBoolean stopNotified = new AtomicBoolean(false);
- private volatile boolean executorStopped;
+ private final Runnable stopTask;
- @Override
- protected void doStart() {
- // A single thread executor
- eventExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
- Threads.createDaemonThreadFactory("zk-client-EventThread")) {
+ private ServiceDelegate() {
+ // Creates the stop task runnable in constructor so that if the stop() method is called from shutdown hook,
+ // it won't fail with class loading error due to failure to load inner class from the shutdown thread.
+ this.stopTask = createStopTask();
+
+ // Add a listener for state changes so that we can terminate the service even it is in STARTING state upoon
+ // stop is requested
+ addListener(new Listener() {
@Override
- protected void terminated() {
- super.terminated();
+ public void starting() {
+ // no-op
+ }
- // Only call notifyStopped if the executor.shutdown() returned, otherwise deadlock (TWILL-110) can occur.
- // Also, notifyStopped() should only be called once.
- if (executorStopped && stopNotified.compareAndSet(false, true)) {
- notifyStopped();
+ @Override
+ public void running() {
+ // no-op
+ }
+
+ @Override
+ public void stopping(State from) {
+ if (from == State.STARTING) {
+ // If it is still starting, just notify that it's started to transit out of the STARTING phase.
+ notifyStarted();
}
}
- };
+
+ @Override
+ public void terminated(State from) {
+ // no-op
+ }
+
+ @Override
+ public void failed(State from, Throwable failure) {
+ eventExecutor.shutdownNow();
+ // Close the ZK client if there is exception. It is needed because the stop task may not get executed
+ closeZooKeeper(zooKeeper.getAndSet(null));
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ protected void doStart() {
+ // A single thread executor for all events
+ ThreadPoolExecutor executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS,
+ new LinkedBlockingQueue<Runnable>(),
+ Threads.createDaemonThreadFactory("zk-client-EventThread"));
+ // Just discard the execution if the executor is closed
+ executor.setRejectedExecutionHandler(new ThreadPoolExecutor.DiscardPolicy());
+ eventExecutor = executor;
try {
zooKeeper.set(createZooKeeper());
@@ -420,29 +452,21 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
@Override
protected void doStop() {
- ZooKeeper zk = zooKeeper.getAndSet(null);
- if (zk != null) {
- try {
- zk.close();
- } catch (InterruptedException e) {
- notifyFailed(e);
- } finally {
- eventExecutor.shutdown();
- executorStopped = true;
-
- // If the executor state is terminated, meaning the terminate() method is triggered,
- // call notifyStopped() if it hasn't been called yet.
- if (eventExecutor.isTerminated() && stopNotified.compareAndSet(false, true)) {
- notifyStopped();
- }
- }
- }
+ // Submit a task to the executor to make sure all pending events in the executor are fired before
+ // transiting this Service into STOPPED state
+ eventExecutor.submit(stopTask);
+ eventExecutor.shutdown();
}
@Override
public void process(WatchedEvent event) {
+ State state = state();
+ if (state == State.TERMINATED || state == State.FAILED) {
+ return;
+ }
+
try {
- if (event.getState() == Event.KeeperState.SyncConnected && state() == State.STARTING) {
+ if (event.getState() == Event.KeeperState.SyncConnected && state == State.STARTING) {
LOG.debug("Connected to ZooKeeper: {}", zkStr);
notifyStarted();
return;
@@ -451,23 +475,27 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
LOG.info("ZooKeeper session expired: {}", zkStr);
// When connection expired, simply reconnect again
- Thread t = new Thread(new Runnable() {
+ if (state != State.RUNNING) {
+ return;
+ }
+ eventExecutor.submit(new Runnable() {
@Override
public void run() {
+ // Only reconnect if the current state is running
+ if (state() != State.RUNNING) {
+ return;
+ }
try {
LOG.info("Reconnect to ZooKeeper due to expiration: {}", zkStr);
- zooKeeper.set(createZooKeeper());
+ closeZooKeeper(zooKeeper.getAndSet(createZooKeeper()));
} catch (IOException e) {
- zooKeeper.set(null);
notifyFailed(e);
}
}
- }, "zk-reconnect");
- t.setDaemon(true);
- t.start();
+ });
}
} finally {
- if (event.getType() == Event.EventType.None && !connectionWatchers.isEmpty()) {
+ if (event.getType() == Event.EventType.None) {
for (Watcher connectionWatcher : connectionWatchers) {
connectionWatcher.process(event);
}
@@ -475,16 +503,58 @@ public final class DefaultZKClientService extends AbstractZKClient implements ZK
}
}
+
+ /**
+ * Creates a {@link Runnable} task that will get executed in the event executor for transiting this
+ * Service into STOPPED state.
+ */
+ private Runnable createStopTask() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // Close the ZK connection in this task will make sure if there is ZK connection created
+ // after doStop() was called but before this task has been executed is also closed.
+ // It is possible to happen when the following sequence happens:
+ //
+ // 1. session expired, hence the expired event is triggered
+ // 2. The reconnect task executed. With Service.state() == RUNNING, it creates a new ZK client
+ // 3. Service.stop() gets called, Service.state() changed to STOPPING
+ // 4. The new ZK client created from the reconnect thread update the zooKeeper with the new one
+ closeZooKeeper(zooKeeper.getAndSet(null));
+ notifyStopped();
+ } catch (Exception e) {
+ notifyFailed(e);
+ }
+ }
+ };
+ }
+
/**
* Creates a new ZooKeeper connection.
*/
private ZooKeeper createZooKeeper() throws IOException {
- ZooKeeper zk = new ZooKeeper(zkStr, sessionTimeout, this);
+ ZooKeeper zk = new ZooKeeper(zkStr, sessionTimeout, wrapWatcher(this));
for (Map.Entry<String, byte[]> authInfo : authInfos.entries()) {
zk.addAuthInfo(authInfo.getKey(), authInfo.getValue());
}
return zk;
}
+
+ /**
+ * Closes the given {@link ZooKeeper} if it is not null. If there is InterruptedException,
+ * it will get logged.
+ */
+ private void closeZooKeeper(@Nullable ZooKeeper zk) {
+ try {
+ if (zk != null) {
+ zk.close();
+ }
+ } catch (InterruptedException e) {
+ LOG.warn("Interrupted when closing ZooKeeper", e);
+ Thread.currentThread().interrupt();
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2054c5f9/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/LeaderElection.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/LeaderElection.java b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/LeaderElection.java
index 837a5ae..8433b18 100644
--- a/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/LeaderElection.java
+++ b/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/LeaderElection.java
@@ -101,12 +101,20 @@ public final class LeaderElection extends AbstractService {
Futures.addCallback(completion, new FutureCallback<String>() {
@Override
public void onSuccess(String result) {
- notifyStopped();
+ try {
+ notifyStopped();
+ } finally {
+ executor.shutdown();
+ }
}
@Override
public void onFailure(Throwable t) {
- notifyFailed(t);
+ try {
+ notifyFailed(t);
+ } finally {
+ executor.shutdown();
+ }
}
}, Threads.SAME_THREAD_EXECUTOR);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2054c5f9/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
index 97dec03..162d4db 100644
--- a/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
+++ b/twill-zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.net.ServerSocket;
import java.security.NoSuchAlgorithmException;
import java.util.Arrays;
import java.util.List;
@@ -235,38 +236,40 @@ public class ZKClientTest {
}
}
}).build(), RetryStrategies.fixDelay(0, TimeUnit.SECONDS)));
- client.startAndWait();
-
- zkServer.stopAndWait();
-
- Assert.assertTrue(disconnectLatch.await(1, TimeUnit.SECONDS));
final CountDownLatch createLatch = new CountDownLatch(1);
- Futures.addCallback(client.create("/testretry/test", null, CreateMode.PERSISTENT), new FutureCallback<String>() {
- @Override
- public void onSuccess(String result) {
- createLatch.countDown();
- }
+ client.startAndWait();
+ try {
+ zkServer.stopAndWait();
- @Override
- public void onFailure(Throwable t) {
- t.printStackTrace(System.out);
- }
- });
-
- TimeUnit.SECONDS.sleep(2);
- zkServer = InMemoryZKServer.builder()
- .setDataDir(dataDir)
- .setAutoCleanDataDir(true)
- .setPort(port)
- .setTickTime(1000)
- .build();
- zkServer.startAndWait();
+ Assert.assertTrue(disconnectLatch.await(1, TimeUnit.SECONDS));
+ Futures.addCallback(client.create("/testretry/test", null, CreateMode.PERSISTENT), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ createLatch.countDown();
+ }
- try {
- Assert.assertTrue(createLatch.await(10, TimeUnit.SECONDS));
+ @Override
+ public void onFailure(Throwable t) {
+ t.printStackTrace(System.out);
+ }
+ });
+
+ TimeUnit.SECONDS.sleep(2);
+ zkServer = InMemoryZKServer.builder()
+ .setDataDir(dataDir)
+ .setAutoCleanDataDir(true)
+ .setPort(port)
+ .setTickTime(1000)
+ .build();
+ zkServer.startAndWait();
+ try {
+ Assert.assertTrue(createLatch.await(10, TimeUnit.SECONDS));
+ } finally {
+ zkServer.stopAndWait();
+ }
} finally {
- zkServer.stopAndWait();
+ client.stopAndWait();
}
}
@@ -353,4 +356,32 @@ public class ZKClientTest {
zkServer.stopAndWait();
}
}
+
+ @Test
+ public void testStop() throws IOException, InterruptedException, ExecutionException {
+ try (final ServerSocket serverSocket = new ServerSocket(0)) {
+ // A latch to make sure at least one connection attempt from the zk client has been made
+ final CountDownLatch connectLatch = new CountDownLatch(1);
+ Thread serverThread = new Thread() {
+ public void run() {
+ try {
+ while (!interrupted()) {
+ serverSocket.accept().close();
+ connectLatch.countDown();
+ }
+ } catch (Exception e) {
+ // no-op
+ }
+ }
+ };
+ serverThread.start();
+
+ ZKClientService zkClient = ZKClientService.Builder.of("localhost:" + serverSocket.getLocalPort()).build();
+ zkClient.start();
+ Assert.assertTrue(connectLatch.await(10, TimeUnit.SECONDS));
+
+ zkClient.stopAndWait();
+ serverThread.interrupt();
+ }
+ }
}
[11/25] incubator-twill git commit: Add missing Javdoc for classes
related to YarnAppClient.
Posted by ch...@apache.org.
Add missing Javdoc for classes related to YarnAppClient.
Oh and remove extra line in TwillController.
This closes #45 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/dc9320b5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/dc9320b5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/dc9320b5
Branch: refs/heads/site
Commit: dc9320b54d83fb5b63d5cc19707e53e686505d28
Parents: 7350f28
Author: hsaputra <hs...@apache.org>
Authored: Fri Jun 19 14:27:48 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 22 16:02:28 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/twill/api/TwillController.java | 1 -
.../org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java | 5 +++++
.../org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java | 5 +++++
.../twill/internal/yarn/VersionDetectYarnAppClientFactory.java | 2 +-
4 files changed, 11 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/dc9320b5/twill-api/src/main/java/org/apache/twill/api/TwillController.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillController.java b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
index 65b9b2d..7d88bb7 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillController.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
@@ -41,7 +41,6 @@ public interface TwillController extends ServiceController {
*/
ServiceDiscovered discoverService(String serviceName);
-
/**
* Changes the number of running instances of a given runnable.
*
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/dc9320b5/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
index dfe4e67..3afa49a 100644
--- a/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
@@ -50,7 +50,12 @@ import java.util.List;
import javax.annotation.Nullable;
/**
+ * <p>
+ * The service implementation of {@link YarnAppClient} for Apache Hadoop 2.0.
*
+ * The {@link VersionDetectYarnAppClientFactory} class will decide to return instance of this class for
+ * Apache Hadoop 2.0.
+ * </p>
*/
public final class Hadoop20YarnAppClient extends AbstractIdleService implements YarnAppClient {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/dc9320b5/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
index ac126ce..046e3f1 100644
--- a/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
+++ b/twill-yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
@@ -46,7 +46,12 @@ import java.util.List;
import javax.annotation.Nullable;
/**
+ * <p>
+ * The service implementation of {@link YarnAppClient} for Apache Hadoop 2.1 and beyond.
*
+ * The {@link VersionDetectYarnAppClientFactory} class will decide to return instance of this class for
+ * Apache Hadoop 2.1 and beyond.
+ * </p>
*/
public final class Hadoop21YarnAppClient extends AbstractIdleService implements YarnAppClient {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/dc9320b5/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
index 04bd30d..9f6e8a1 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
@@ -21,7 +21,7 @@ import com.google.common.base.Throwables;
import org.apache.hadoop.conf.Configuration;
/**
- *
+ * Helper factory class to return the right instance of {@link YarnAppClient} based on Apache Hadoop version.
*/
public final class VersionDetectYarnAppClientFactory implements YarnAppClientFactory {
[22/25] incubator-twill git commit: Bump version to
0.7.0-incubating-SNAPSHOT
Posted by ch...@apache.org.
Bump version to 0.7.0-incubating-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/85a626a5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/85a626a5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/85a626a5
Branch: refs/heads/site
Commit: 85a626a566ac388400097fcbe5ed928f8bc3e790
Parents: c1d977b
Author: Terence Yim <ch...@apache.org>
Authored: Tue Jul 14 22:41:22 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 14 22:41:22 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
twill-api/pom.xml | 2 +-
twill-common/pom.xml | 2 +-
twill-core/pom.xml | 2 +-
twill-discovery-api/pom.xml | 2 +-
twill-discovery-core/pom.xml | 2 +-
twill-examples/echo/pom.xml | 2 +-
twill-examples/pom.xml | 2 +-
twill-examples/yarn/pom.xml | 2 +-
twill-ext/pom.xml | 2 +-
twill-java8-test/pom.xml | 2 +-
twill-yarn/pom.xml | 2 +-
twill-zookeeper/pom.xml | 2 +-
13 files changed, 13 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e026296..a2e1b9b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,7 +29,7 @@
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Apache Twill</name>
<url>http://twill.incubator.apache.org</url>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-api/pom.xml b/twill-api/pom.xml
index 102c1d0..c011dee 100644
--- a/twill-api/pom.xml
+++ b/twill-api/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>twill-api</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-common/pom.xml
----------------------------------------------------------------------
diff --git a/twill-common/pom.xml b/twill-common/pom.xml
index cc3030b..b371acb 100644
--- a/twill-common/pom.xml
+++ b/twill-common/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-core/pom.xml b/twill-core/pom.xml
index 8f1de96..26ae718 100644
--- a/twill-core/pom.xml
+++ b/twill-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-discovery-api/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-api/pom.xml b/twill-discovery-api/pom.xml
index 589022d..147c6bc 100644
--- a/twill-discovery-api/pom.xml
+++ b/twill-discovery-api/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-discovery-core/pom.xml
----------------------------------------------------------------------
diff --git a/twill-discovery-core/pom.xml b/twill-discovery-core/pom.xml
index 8159250..fb3dcbb 100644
--- a/twill-discovery-core/pom.xml
+++ b/twill-discovery-core/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-examples/echo/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/echo/pom.xml b/twill-examples/echo/pom.xml
index 634a1ac..8ba89ff 100644
--- a/twill-examples/echo/pom.xml
+++ b/twill-examples/echo/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<name>Apache Twill examples: Echo</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-examples/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/pom.xml b/twill-examples/pom.xml
index ce968bb..10163c2 100644
--- a/twill-examples/pom.xml
+++ b/twill-examples/pom.xml
@@ -25,7 +25,7 @@ limitations under the License.
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>twill-examples</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-examples/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/pom.xml b/twill-examples/yarn/pom.xml
index 17e7a33..71f86e8 100644
--- a/twill-examples/yarn/pom.xml
+++ b/twill-examples/yarn/pom.xml
@@ -24,7 +24,7 @@ limitations under the License.
<parent>
<artifactId>twill-examples</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<name>Apache Twill examples: YARN</name>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-ext/pom.xml
----------------------------------------------------------------------
diff --git a/twill-ext/pom.xml b/twill-ext/pom.xml
index 0f1d674..539fd20 100644
--- a/twill-ext/pom.xml
+++ b/twill-ext/pom.xml
@@ -22,7 +22,7 @@ limitations under the License.
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-java8-test/pom.xml
----------------------------------------------------------------------
diff --git a/twill-java8-test/pom.xml b/twill-java8-test/pom.xml
index 8f45842..892a025 100644
--- a/twill-java8-test/pom.xml
+++ b/twill-java8-test/pom.xml
@@ -24,7 +24,7 @@
<parent>
<groupId>org.apache.twill</groupId>
<artifactId>twill-parent</artifactId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<artifactId>twill-java8-test</artifactId>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/twill-yarn/pom.xml b/twill-yarn/pom.xml
index 117a505..9d61831 100644
--- a/twill-yarn/pom.xml
+++ b/twill-yarn/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/85a626a5/twill-zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/twill-zookeeper/pom.xml b/twill-zookeeper/pom.xml
index 5a98f95..6cf302d 100644
--- a/twill-zookeeper/pom.xml
+++ b/twill-zookeeper/pom.xml
@@ -22,7 +22,7 @@
<parent>
<artifactId>twill-parent</artifactId>
<groupId>org.apache.twill</groupId>
- <version>0.6.0-incubating</version>
+ <version>0.7.0-incubating-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
[15/25] incubator-twill git commit: (TWILL-139) Added workaround to
retry starting Kafka multiple times
Posted by ch...@apache.org.
(TWILL-139) Added workaround to retry starting Kafka multiple times
- In case race condition happened, the startup will fail with a ZkTimeoutException
- Retry upon catching the timeout exception
This closes #51 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ecaf51b1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ecaf51b1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ecaf51b1
Branch: refs/heads/site
Commit: ecaf51b1179ae2c22bef69c9ff6c88c5946008e6
Parents: 43b080d
Author: Terence Yim <ch...@apache.org>
Authored: Wed Jun 24 14:30:28 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Wed Jun 24 21:09:05 2015 -0700
----------------------------------------------------------------------
.../internal/kafka/EmbeddedKafkaServer.java | 74 ++++++++++++++++++--
.../appmaster/ApplicationMasterMain.java | 8 ++-
2 files changed, 75 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ecaf51b1/twill-core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java b/twill-core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
index 0beab44..cd86dcb 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
@@ -17,9 +17,14 @@
*/
package org.apache.twill.internal.kafka;
+import com.google.common.base.Throwables;
import com.google.common.util.concurrent.AbstractIdleService;
import kafka.server.KafkaConfig;
-import kafka.server.KafkaServerStartable;
+import kafka.server.KafkaServer;
+import kafka.utils.Time;
+import org.I0Itec.zkclient.exception.ZkTimeoutException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Properties;
@@ -29,20 +34,77 @@ import java.util.Properties;
*/
public final class EmbeddedKafkaServer extends AbstractIdleService {
- private final KafkaServerStartable server;
+ public static final String START_TIMEOUT_RETRIES = "twill.kafka.start.timeout.retries";
+
+ private static final Logger LOG = LoggerFactory.getLogger(EmbeddedKafkaServer.class);
+ private static final String DEFAULT_START_TIMEOUT_RETRIES = "5";
+
+ private final int startTimeoutRetries;
+ private final KafkaConfig kafkaConfig;
+ private KafkaServer server;
public EmbeddedKafkaServer(Properties properties) {
- server = new KafkaServerStartable(new KafkaConfig(properties));
+ this.startTimeoutRetries = Integer.parseInt(properties.getProperty(START_TIMEOUT_RETRIES,
+ DEFAULT_START_TIMEOUT_RETRIES));
+ this.kafkaConfig = new KafkaConfig(properties);
}
@Override
protected void startUp() throws Exception {
- server.startup();
+ int tries = 0;
+ do {
+ KafkaServer kafkaServer = createKafkaServer(kafkaConfig);
+ try {
+ kafkaServer.startup();
+ server = kafkaServer;
+ } catch (Exception e) {
+ kafkaServer.shutdown();
+ kafkaServer.awaitShutdown();
+
+ Throwable rootCause = Throwables.getRootCause(e);
+ if (rootCause instanceof ZkTimeoutException) {
+ // Potentially caused by race condition bug described in TWILL-139.
+ LOG.warn("Timeout when connecting to ZooKeeper from KafkaServer. Attempt number {}.", tries, rootCause);
+ } else {
+ throw e;
+ }
+ }
+ } while (server == null && ++tries < startTimeoutRetries);
+
+ if (server == null) {
+ throw new IllegalStateException("Failed to start Kafka server after " + tries + " attempts.");
+ }
}
@Override
protected void shutDown() throws Exception {
- server.shutdown();
- server.awaitShutdown();
+ if (server != null) {
+ server.shutdown();
+ server.awaitShutdown();
+ }
+ }
+
+ private KafkaServer createKafkaServer(KafkaConfig kafkaConfig) {
+ return new KafkaServer(kafkaConfig, new Time() {
+
+ @Override
+ public long milliseconds() {
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ public long nanoseconds() {
+ return System.nanoTime();
+ }
+
+ @Override
+ public void sleep(long ms) {
+ try {
+ Thread.sleep(ms);
+ } catch (InterruptedException e) {
+ Thread.interrupted();
+ }
+ }
+ });
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ecaf51b1/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
index 561a68b..3e8cb93 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
@@ -179,7 +179,13 @@ public final class ApplicationMasterMain extends ServiceMain {
prop.setProperty("log.flush.interval.ms", "1000");
prop.setProperty("log.segment.bytes", "536870912");
prop.setProperty("zookeeper.connect", kafkaZKConnect);
- prop.setProperty("zookeeper.connection.timeout.ms", "1000000");
+ // Set the connection timeout to relatively short time (3 seconds).
+ // It is only used by the org.I0Itec.zkclient.ZKClient inside KafkaServer
+ // to block and wait for ZK connection goes into SyncConnected state.
+ // However, due to race condition described in TWILL-139 in the ZK client library used by Kafka,
+ // when ZK authentication is enabled, the ZK client may hang until connection timeout.
+ // Setting it to lower value allow the AM to retry multiple times if race happens.
+ prop.setProperty("zookeeper.connection.timeout.ms", "3000");
prop.setProperty("default.replication.factor", "1");
return prop;
}
[05/25] incubator-twill git commit: TWILL-135 Update the pom.xml to
get Java7 API link and update copyright year
Posted by ch...@apache.org.
TWILL-135 Update the pom.xml to get Java7 API link and update copyright year
Since latest version of Twill requires Java7, need to update pom.xml to have
the right JDK api link and also update the year in the copyright notice for Twill JavaDoc
This closes #40 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/0d3c5eea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/0d3c5eea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/0d3c5eea
Branch: refs/heads/site
Commit: 0d3c5eea4476bbf1bd0fb76d5edebd1abb3ffbd5
Parents: 4e63577
Author: hsaputra <hs...@apache.org>
Authored: Fri Jun 12 11:31:57 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jun 12 14:10:16 2015 -0700
----------------------------------------------------------------------
pom.xml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/0d3c5eea/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ec43e8b..1556534 100644
--- a/pom.xml
+++ b/pom.xml
@@ -221,10 +221,10 @@
<configuration>
<excludePackageNames>*.internal.*:echo:*.example.*</excludePackageNames>
<links>
- <link>http://download.oracle.com/javase/6/docs/api/</link>
+ <link>http://docs.oracle.com/javase/7/docs/api/</link>
</links>
<bottom>
- <![CDATA[Copyright © 2013 <a href="http://www.apache.org">The Apache Software Foundation</a>. All rights reserved.]]>
+ <![CDATA[Copyright © 2013-2015 <a href="http://www.apache.org">The Apache Software Foundation</a>. All rights reserved.]]>
</bottom>
</configuration>
<executions>
[16/25] incubator-twill git commit: (TWILL-117) support adding set of
classes to the Application Master and runnables jar using the TwillPreparer
API. the classpath is archived under "application-classpath" in the launcher
jar.
Posted by ch...@apache.org.
(TWILL-117) support adding set of classes to the Application Master and runnables jar using the TwillPreparer API. the classpath is archived under "application-classpath" in the launcher jar.
This closes #49 on GitHub.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d181b7ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d181b7ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d181b7ce
Branch: refs/heads/site
Commit: d181b7cef40735520b1e198876fa3d60bb47fa40
Parents: ecaf51b
Author: shankar <sh...@cask.co>
Authored: Wed Jun 17 11:36:25 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Wed Jul 1 11:34:25 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/ClassAcceptor.java | 39 ++++++++++
.../org/apache/twill/api/TwillPreparer.java | 20 +++++
.../twill/internal/ApplicationBundler.java | 80 +++++++++++---------
.../org/apache/twill/internal/Constants.java | 2 +
.../twill/internal/utils/Dependencies.java | 18 +----
.../apache/twill/launcher/TwillLauncher.java | 34 +++++----
.../apache/twill/example/yarn/HelloWorld.java | 29 ++++++-
.../apache/twill/yarn/YarnTwillPreparer.java | 40 ++++++++--
8 files changed, 185 insertions(+), 77 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-api/src/main/java/org/apache/twill/api/ClassAcceptor.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/ClassAcceptor.java b/twill-api/src/main/java/org/apache/twill/api/ClassAcceptor.java
new file mode 100644
index 0000000..db8216f
--- /dev/null
+++ b/twill-api/src/main/java/org/apache/twill/api/ClassAcceptor.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import java.net.URL;
+
+/**
+ * Class that can be used to determine if class can be accepted.
+ */
+public class ClassAcceptor {
+ /**
+ * Invoked to determine if class can be accepted. default behavior returns true.
+ *
+ * @param className Name of the class.
+ * @param classUrl URL for the class resource.
+ * @param classPathUrl URL for the class path resource that contains the class resource.
+ * If the URL protocol is {@code file}, it would be the path to root package.
+ * If the URL protocol is {@code jar}, it would be the jar file.
+ * @return true to accept the given class, false otherwise.
+ */
+ public boolean accept(String className, URL classUrl, URL classPathUrl) {
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
index c2c62a1..f60080a 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
@@ -179,6 +179,26 @@ public interface TwillPreparer {
TwillPreparer withClassPaths(Iterable<String> classPaths);
/**
+ * Adds the set of paths to the classpath on the target machine for ApplicationMaster and all runnables.
+ * @return This {@link TwillPreparer}
+ */
+ TwillPreparer withApplicationClassPaths(String... classPaths);
+
+ /**
+ * Adds the set of paths to the classpath on the target machine for ApplicationMaster and all runnables.
+ * @return This {@link TwillPreparer}
+ */
+ TwillPreparer withApplicationClassPaths(Iterable<String> classPaths);
+
+ /**
+ * Uses {@link ClassAcceptor} to determine the classes to include in the bundle jar for
+ * ApplicationMaster and all runnables.
+ * @param classAcceptor to specify which classes to include in the bundle jar
+ * @return This {@link TwillPreparer}
+ */
+ TwillPreparer withBundlerClassAcceptor(ClassAcceptor classAcceptor);
+
+ /**
* Adds security credentials for the runtime environment to gives application access to resources.
*
* @param secureStore Contains security token available for the runtime environment.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
index c09b881..73efc81 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
@@ -28,6 +28,7 @@ import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
import com.google.common.io.Closeables;
import com.google.common.io.Files;
+import org.apache.twill.api.ClassAcceptor;
import org.apache.twill.filesystem.Location;
import org.apache.twill.internal.utils.Dependencies;
import org.slf4j.Logger;
@@ -43,7 +44,6 @@ import java.io.OutputStream;
import java.net.URI;
import java.net.URL;
import java.util.Collections;
-import java.util.List;
import java.util.Queue;
import java.util.Set;
import java.util.jar.JarEntry;
@@ -57,36 +57,22 @@ import java.util.zip.CheckedOutputStream;
public final class ApplicationBundler {
private static final Logger LOG = LoggerFactory.getLogger(ApplicationBundler.class);
-
+
public static final String SUBDIR_CLASSES = "classes/";
public static final String SUBDIR_LIB = "lib/";
public static final String SUBDIR_RESOURCES = "resources/";
- private final List<String> excludePackages;
- private final List<String> includePackages;
+ private final ClassAcceptor classAcceptor;
private final Set<String> bootstrapClassPaths;
private final CRC32 crc32;
/**
* Constructs a ApplicationBundler.
*
- * @param excludePackages Class packages to exclude
- */
- public ApplicationBundler(Iterable<String> excludePackages) {
- this(excludePackages, ImmutableList.<String>of());
- }
-
- /**
- * Constructs a ApplicationBundler.
- *
- * @param excludePackages Class packages to exclude
- * @param includePackages Class packages that should be included. Anything in this list will override the
- * one provided in excludePackages.
+ * @param classAcceptor ClassAcceptor for class packages to include
*/
- public ApplicationBundler(Iterable<String> excludePackages, Iterable<String> includePackages) {
- this.excludePackages = ImmutableList.copyOf(excludePackages);
- this.includePackages = ImmutableList.copyOf(includePackages);
-
+ public ApplicationBundler(ClassAcceptor classAcceptor) {
+ this.classAcceptor = classAcceptor;
ImmutableSet.Builder<String> builder = ImmutableSet.builder();
for (String classpath : Splitter.on(File.pathSeparatorChar).split(System.getProperty("sun.boot.class.path"))) {
File file = new File(classpath);
@@ -99,7 +85,41 @@ public final class ApplicationBundler {
}
this.bootstrapClassPaths = builder.build();
this.crc32 = new CRC32();
+ }
+ /**
+ * Constructs a ApplicationBundler.
+ *
+ * @param excludePackages Class packages to exclude
+ */
+ public ApplicationBundler(Iterable<String> excludePackages) {
+ this(excludePackages, ImmutableList.<String>of());
+ }
+
+ /**
+ * Constructs a ApplicationBundler.
+ *
+ * @param excludePackages Class packages to exclude
+ * @param includePackages Class packages that should be included. Anything in this list will override the
+ * one provided in excludePackages.
+ */
+ public ApplicationBundler(final Iterable<String> excludePackages, final Iterable<String> includePackages) {
+ this(new ClassAcceptor() {
+ @Override
+ public boolean accept(String className, URL classUrl, URL classPathUrl) {
+ for (String includePackage : includePackages) {
+ if (className.startsWith(includePackage)) {
+ return true;
+ }
+ }
+ for (String excludePackage : excludePackages) {
+ if (className.startsWith(excludePackage)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ });
}
public void createBundle(Location target, Iterable<Class<?>> classes) throws IOException {
@@ -175,29 +195,15 @@ public final class ApplicationBundler {
// Record the set of classpath URL that are already added to the jar
final Set<URL> seenClassPaths = Sets.newHashSet();
- Dependencies.findClassDependencies(classLoader, new Dependencies.ClassAcceptor() {
+ Dependencies.findClassDependencies(classLoader, new ClassAcceptor() {
@Override
public boolean accept(String className, URL classUrl, URL classPathUrl) {
if (bootstrapClassPaths.contains(classPathUrl.getFile())) {
return false;
}
-
- boolean shouldInclude = false;
- for (String include : includePackages) {
- if (className.startsWith(include)) {
- shouldInclude = true;
- break;
- }
- }
-
- if (!shouldInclude) {
- for (String exclude : excludePackages) {
- if (className.startsWith(exclude)) {
- return false;
- }
- }
+ if (!classAcceptor.accept(className, classUrl, classPathUrl)) {
+ return false;
}
-
if (seenClassPaths.add(classPathUrl)) {
putEntry(className, classUrl, classPathUrl, entries, jarOut);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-core/src/main/java/org/apache/twill/internal/Constants.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/Constants.java b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
index fbc6e70..64b029d 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/Constants.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
@@ -45,6 +45,8 @@ public final class Constants {
public static final String STDOUT = "stdout";
public static final String STDERR = "stderr";
+ public static final String CLASSPATH = "classpath";
+ public static final String APPLICATION_CLASSPATH = "application-classpath";
/**
* Constants for names of internal files that are shared between client, AM and containers.
*/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
index f6f8dde..eb55557 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
@@ -22,6 +22,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
import com.google.common.io.ByteStreams;
+import org.apache.twill.api.ClassAcceptor;
import org.objectweb.asm.AnnotationVisitor;
import org.objectweb.asm.ClassReader;
import org.objectweb.asm.ClassVisitor;
@@ -47,23 +48,6 @@ import java.util.Set;
*/
public final class Dependencies {
- /**
- * Represents a callback for accepting a class during dependency traversal.
- */
- public interface ClassAcceptor {
- /**
- * Invoked when a class is being found as a dependency.
- *
- * @param className Name of the class.
- * @param classUrl URL for the class resource.
- * @param classPathUrl URL for the class path resource that contains the class resource.
- * If the URL protocol is {@code file}, it would be the path to root package.
- * If the URL protocol is {@code jar}, it would be the jar file.
- * @return true keep finding dependencies on the given class.
- */
- boolean accept(String className, URL classUrl, URL classPathUrl);
- }
-
public static void findClassDependencies(ClassLoader classLoader,
ClassAcceptor acceptor,
String...classesToResolve) throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
index 3484349..3405709 100644
--- a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
+++ b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
@@ -17,6 +17,8 @@
*/
package org.apache.twill.launcher;
+import org.apache.twill.internal.Constants;
+
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
@@ -156,22 +158,11 @@ public final class TwillLauncher {
}
if (useClassPath) {
- InputStream is = ClassLoader.getSystemResourceAsStream("classpath");
- if (is != null) {
- try {
- BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
- String line = reader.readLine();
- if (line != null) {
- for (String path : line.split(":")) {
- urls.addAll(getClassPaths(path));
- }
- }
- } finally {
- is.close();
- }
- }
+ addClassPathsToList(urls, Constants.CLASSPATH);
}
+ addClassPathsToList(urls, Constants.APPLICATION_CLASSPATH);
+
return new URLClassLoader(urls.toArray(new URL[0]));
} catch (Exception e) {
@@ -179,6 +170,21 @@ public final class TwillLauncher {
}
}
+ private static void addClassPathsToList(List<URL> urls, String resource) throws IOException {
+ try (InputStream is = ClassLoader.getSystemResourceAsStream(resource)) {
+ if (is != null) {
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")))) {
+ String line = reader.readLine();
+ if (line != null) {
+ for (String path : line.split(":")) {
+ urls.addAll(getClassPaths(path.trim()));
+ }
+ }
+ }
+ }
+ }
+ }
+
private static Collection<URL> getClassPaths(String path) throws MalformedURLException {
String classpath = expand(path);
if (classpath.endsWith("/*")) {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
----------------------------------------------------------------------
diff --git a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
index 76e4948..a9cbba8 100644
--- a/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
+++ b/twill-examples/yarn/src/main/java/org/apache/twill/example/yarn/HelloWorld.java
@@ -17,9 +17,14 @@
*/
package org.apache.twill.example.yarn;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
import com.google.common.util.concurrent.Futures;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.ClassAcceptor;
import org.apache.twill.api.TwillController;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.logging.PrinterLogHandler;
@@ -28,6 +33,8 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.PrintWriter;
+import java.net.URL;
+import java.util.List;
import java.util.concurrent.ExecutionException;
/**
@@ -57,15 +64,22 @@ public class HelloWorld {
}
String zkStr = args[0];
-
+ YarnConfiguration yarnConfiguration = new YarnConfiguration();
final TwillRunnerService twillRunner =
new YarnTwillRunnerService(
- new YarnConfiguration(), zkStr);
+ yarnConfiguration, zkStr);
twillRunner.start();
+ String yarnClasspath =
+ yarnConfiguration.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH,
+ Joiner.on(",").join(YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
+ List<String> applicationClassPaths = Lists.newArrayList();
+ Iterables.addAll(applicationClassPaths, Splitter.on(",").split(yarnClasspath));
final TwillController controller =
twillRunner.prepare(new HelloWorldRunnable())
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .withApplicationClassPaths(applicationClassPaths)
+ .withBundlerClassAcceptor(new HadoopClassExcluder())
.start();
Runtime.getRuntime().addShutdownHook(new Thread() {
@@ -85,4 +99,15 @@ public class HelloWorld {
e.printStackTrace();
}
}
+
+ static class HadoopClassExcluder extends ClassAcceptor {
+ @Override
+ public boolean accept(String className, URL classUrl, URL classPathUrl) {
+ // exclude hadoop but not hbase package
+ if (className.startsWith("org.apache.hadoop") && !className.startsWith("org.apache.hadoop.hbase")) {
+ return false;
+ }
+ return true;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d181b7ce/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index aca4728..4e9f76d 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.ClassAcceptor;
import org.apache.twill.api.EventHandlerSpecification;
import org.apache.twill.api.LocalFile;
import org.apache.twill.api.RunId;
@@ -122,12 +123,14 @@ final class YarnTwillPreparer implements TwillPreparer {
private final List<URI> resources = Lists.newArrayList();
private final List<String> classPaths = Lists.newArrayList();
private final ListMultimap<String, String> runnableArgs = ArrayListMultimap.create();
+ private final List<String> applicationClassPaths = Lists.newArrayList();
private final Credentials credentials;
private final int reservedMemory;
private String user;
private String schedulerQueue;
private String extraOptions;
private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG;
+ private ClassAcceptor classAcceptor;
private LogEntry.Level logLevel;
YarnTwillPreparer(YarnConfiguration yarnConfig, TwillSpecification twillSpec,
@@ -147,6 +150,7 @@ final class YarnTwillPreparer implements TwillPreparer {
this.user = System.getProperty("user.name");
this.extraOptions = extraOptions;
this.logLevel = logLevel;
+ this.classAcceptor = new ClassAcceptor();
}
@Override
@@ -246,6 +250,23 @@ final class YarnTwillPreparer implements TwillPreparer {
}
@Override
+ public TwillPreparer withApplicationClassPaths(String... classPaths) {
+ return withApplicationClassPaths(ImmutableList.copyOf(classPaths));
+ }
+
+ @Override
+ public TwillPreparer withApplicationClassPaths(Iterable<String> classPaths) {
+ Iterables.addAll(this.applicationClassPaths, classPaths);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withBundlerClassAcceptor(ClassAcceptor classAcceptor) {
+ this.classAcceptor = classAcceptor;
+ return this;
+ }
+
+ @Override
public TwillPreparer addSecureStore(SecureStore secureStore) {
Object store = secureStore.getStore();
Preconditions.checkArgument(store instanceof Credentials, "Only Hadoop Credentials is supported.");
@@ -264,7 +285,6 @@ final class YarnTwillPreparer implements TwillPreparer {
try {
final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue);
final ApplicationId appId = launcher.getContainerInfo();
-
Callable<ProcessController<YarnApplicationReport>> submitTask =
new Callable<ProcessController<YarnApplicationReport>>() {
@Override
@@ -304,6 +324,7 @@ final class YarnTwillPreparer implements TwillPreparer {
.put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
.put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
.put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
+
if (logLevel != null) {
LOG.debug("Log level is set to {} for the Twill application.", logLevel);
builder.put(EnvKeys.TWILL_APP_LOG_LEVEL, logLevel.toString());
@@ -351,7 +372,7 @@ final class YarnTwillPreparer implements TwillPreparer {
}
private ApplicationBundler createBundler() {
- return new ApplicationBundler(ImmutableList.<String>of());
+ return new ApplicationBundler(classAcceptor);
}
private LocalFile createLocalFile(String name, Location location) throws IOException {
@@ -502,7 +523,7 @@ final class YarnTwillPreparer implements TwillPreparer {
if (classLoader == null) {
classLoader = getClass().getClassLoader();
}
- Dependencies.findClassDependencies(classLoader, new Dependencies.ClassAcceptor() {
+ Dependencies.findClassDependencies(classLoader, new ClassAcceptor() {
@Override
public boolean accept(String className, URL classUrl, URL classPathUrl) {
Preconditions.checkArgument(className.startsWith(launcherName) || className.equals(portFinderName),
@@ -520,10 +541,8 @@ final class YarnTwillPreparer implements TwillPreparer {
}, launcherName, portFinderName);
try {
- if (!classPaths.isEmpty()) {
- jarOut.putNextEntry(new JarEntry("classpath"));
- jarOut.write(Joiner.on(':').join(classPaths).getBytes(Charsets.UTF_8));
- }
+ addClassPaths(Constants.CLASSPATH, classPaths, jarOut);
+ addClassPaths(Constants.APPLICATION_CLASSPATH, applicationClassPaths, jarOut);
} finally {
jarOut.close();
}
@@ -532,6 +551,13 @@ final class YarnTwillPreparer implements TwillPreparer {
localFiles.put(Constants.Files.LAUNCHER_JAR, createLocalFile(Constants.Files.LAUNCHER_JAR, location));
}
+ private void addClassPaths(String classpathId, List<String> classPaths, JarOutputStream jarOut) throws IOException {
+ if (!classPaths.isEmpty()) {
+ jarOut.putNextEntry(new JarEntry(classpathId));
+ jarOut.write(Joiner.on(':').join(classPaths).getBytes(Charsets.UTF_8));
+ }
+ }
+
private void saveJvmOptions(Map<String, LocalFile> localFiles) throws IOException {
if ((extraOptions == null || extraOptions.isEmpty()) &&
JvmOptions.DebugOptions.NO_DEBUG.equals(debugOptions)) {
[04/25] incubator-twill git commit: Change call for deprecated method
YarnAppClient#createLauncher in YarnTwillPreparer
Posted by ch...@apache.org.
Change call for deprecated method YarnAppClient#createLauncher in YarnTwillPreparer
This closes #38 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/4e635771
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/4e635771
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/4e635771
Branch: refs/heads/site
Commit: 4e635771bb7f0278d9f4a39d6e51fa6f9af9e789
Parents: 70b6858
Author: hsaputra <hs...@apache.org>
Authored: Tue Jun 2 13:04:16 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jun 12 14:08:48 2015 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/4e635771/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 3ae27bf..11ceba2 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -253,7 +253,7 @@ final class YarnTwillPreparer implements TwillPreparer {
@Override
public TwillController start() {
try {
- final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(user, twillSpec, schedulerQueue);
+ final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue);
final ApplicationId appId = launcher.getContainerInfo();
Callable<ProcessController<YarnApplicationReport>> submitTask =
[19/25] incubator-twill git commit: Follow up fix for the TWILL-116
PR to update the comments for constants used in restart commands.
Posted by ch...@apache.org.
Follow up fix for the TWILL-116 PR to update the comments for constants used in restart commands.
This closes #54 on GitHub.
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/9495e300
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/9495e300
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/9495e300
Branch: refs/heads/site
Commit: 9495e300f2d4ddc01d6960ff6392eac950f18a78
Parents: 3a0f6e1
Author: hsaputra <hs...@apache.org>
Authored: Tue Jul 14 10:52:18 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 14 18:04:06 2015 -0700
----------------------------------------------------------------------
twill-core/src/main/java/org/apache/twill/internal/Constants.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/9495e300/twill-core/src/main/java/org/apache/twill/internal/Constants.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/Constants.java b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
index e3a2194..defd013 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/Constants.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/Constants.java
@@ -48,7 +48,7 @@ public final class Constants {
public static final String CLASSPATH = "classpath";
public static final String APPLICATION_CLASSPATH = "application-classpath";
- /** For runnables instance lifecycle ZK path */
+ /** Command names for the restart runnable instances. */
public static final String RESTART_ALL_RUNNABLE_INSTANCES = "restartAllRunnableInstances";
public static final String RESTART_RUNNABLES_INSTANCES = "restartRunnablesInstances";
[02/25] incubator-twill git commit: (TWILL-130) Upgrade to Java 7
Posted by ch...@apache.org.
(TWILL-130) Upgrade to Java 7
- Also upgrade try-finally block to ARM style.
This closes #35 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/2175c664
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/2175c664
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/2175c664
Branch: refs/heads/site
Commit: 2175c66466a10dbae1345584217338c2b46fbee9
Parents: f24896b
Author: Terence Yim <ch...@apache.org>
Authored: Thu May 21 14:47:35 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue May 26 17:31:33 2015 -0700
----------------------------------------------------------------------
pom.xml | 4 +--
.../twill/internal/ApplicationBundler.java | 15 ++--------
.../twill/internal/json/ArgumentsCodec.java | 10 ++-----
.../twill/internal/json/JvmOptionsCodec.java | 10 ++-----
.../json/TwillSpecificationAdapter.java | 11 ++-----
.../twill/internal/utils/Dependencies.java | 5 +---
.../apache/twill/internal/utils/Networks.java | 9 ++----
.../org/apache/twill/launcher/FindFreePort.java | 5 +---
.../apache/twill/launcher/TwillLauncher.java | 10 ++-----
.../internal/utils/ApplicationBundlerTest.java | 5 +---
.../org/apache/twill/ext/BundledJarRunner.java | 15 ++++------
.../apache/twill/ext/BundledJarRunnerTest.java | 12 ++------
.../appmaster/ApplicationMasterService.java | 12 +++-----
.../internal/container/TwillContainerMain.java | 10 ++-----
.../internal/yarn/AbstractYarnTwillService.java | 5 +---
.../apache/twill/yarn/YarnTwillPreparer.java | 30 ++++++--------------
.../twill/yarn/YarnTwillRunnerService.java | 10 ++-----
.../org/apache/twill/yarn/DistributedShell.java | 7 ++---
.../apache/twill/yarn/EchoServerTestRun.java | 9 +++---
.../twill/yarn/FailureRestartTestRun.java | 5 +---
.../org/apache/twill/yarn/LocalFileTestRun.java | 10 ++-----
.../twill/yarn/ResourceReportTestRun.java | 18 ++++++------
.../org/apache/twill/yarn/SocketServer.java | 13 +++------
23 files changed, 66 insertions(+), 174 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f41e075..ec43e8b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -298,8 +298,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>3.1</version>
<configuration>
- <source>1.6</source>
- <target>1.6</target>
+ <source>1.7</source>
+ <target>1.7</target>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
index 1f8326d..c09b881 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
@@ -130,8 +130,7 @@ public final class ApplicationBundler {
File tmpJar = File.createTempFile(target.getName(), ".tmp");
try {
Set<String> entries = Sets.newHashSet();
- JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(tmpJar));
- try {
+ try (JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(tmpJar))) {
// Find class dependencies
findDependencies(classes, entries, jarOut);
@@ -139,8 +138,6 @@ public final class ApplicationBundler {
for (URI resource : resources) {
copyResource(resource, entries, jarOut);
}
- } finally {
- jarOut.close();
}
LOG.debug("copying temporary bundle to destination {} ({} bytes)", target, tmpJar.length());
// Copy the tmp jar into destination.
@@ -268,9 +265,8 @@ public final class ApplicationBundler {
LOG.trace("adding bundle entry " + entry);
try {
JarEntry jarEntry = new JarEntry(entry);
- InputStream is = url.openStream();
- try {
+ try (InputStream is = url.openStream()) {
if (compress) {
jarOut.putNextEntry(jarEntry);
ByteStreams.copy(is, jarOut);
@@ -288,8 +284,6 @@ public final class ApplicationBundler {
jarOut.putNextEntry(jarEntry);
os.transfer(jarOut);
}
- } finally {
- is.close();
}
jarOut.closeEntry();
} catch (Exception e) {
@@ -350,11 +344,8 @@ public final class ApplicationBundler {
if (entries.add(path)) {
saveDirEntry(prefix, entries, jarOut);
jarOut.putNextEntry(new JarEntry(path));
- InputStream is = url.openStream();
- try {
+ try (InputStream is = url.openStream()) {
ByteStreams.copy(is, jarOut);
- } finally {
- is.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java b/twill-core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
index 484231f..341d3ee 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
@@ -49,21 +49,15 @@ public final class ArgumentsCodec implements JsonSerializer<Arguments>, JsonDese
.create();
public static void encode(Arguments arguments, OutputSupplier<? extends Writer> writerSupplier) throws IOException {
- Writer writer = writerSupplier.getOutput();
- try {
+ try (Writer writer = writerSupplier.getOutput()) {
GSON.toJson(arguments, writer);
- } finally {
- writer.close();
}
}
public static Arguments decode(InputSupplier<? extends Reader> readerSupplier) throws IOException {
- Reader reader = readerSupplier.getInput();
- try {
+ try (Reader reader = readerSupplier.getInput()) {
return GSON.fromJson(reader, Arguments.class);
- } finally {
- reader.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/json/JvmOptionsCodec.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/json/JvmOptionsCodec.java b/twill-core/src/main/java/org/apache/twill/internal/json/JvmOptionsCodec.java
index 09fc91b..807840f 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/json/JvmOptionsCodec.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/json/JvmOptionsCodec.java
@@ -49,20 +49,14 @@ public class JvmOptionsCodec implements JsonSerializer<JvmOptions>, JsonDeserial
.create();
public static void encode(JvmOptions jvmOptions, OutputSupplier<? extends Writer> writerSupplier) throws IOException {
- Writer writer = writerSupplier.getOutput();
- try {
+ try (Writer writer = writerSupplier.getOutput()) {
GSON.toJson(jvmOptions, writer);
- } finally {
- writer.close();
}
}
public static JvmOptions decode(InputSupplier<? extends Reader> readerSupplier) throws IOException {
- Reader reader = readerSupplier.getInput();
- try {
+ try (Reader reader = readerSupplier.getInput()) {
return GSON.fromJson(reader, JvmOptions.class);
- } finally {
- reader.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java b/twill-core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
index d882096..eda0c71 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
@@ -82,11 +82,8 @@ public final class TwillSpecificationAdapter {
}
public void toJson(TwillSpecification spec, File file) throws IOException {
- Writer writer = Files.newWriter(file, Charsets.UTF_8);
- try {
+ try (Writer writer = Files.newWriter(file, Charsets.UTF_8)) {
toJson(spec, writer);
- } finally {
- writer.close();
}
}
@@ -99,17 +96,15 @@ public final class TwillSpecificationAdapter {
}
public TwillSpecification fromJson(File file) throws IOException {
- Reader reader = Files.newReader(file, Charsets.UTF_8);
- try {
+ try (Reader reader = Files.newReader(file, Charsets.UTF_8)) {
return fromJson(reader);
- } finally {
- reader.close();
}
}
// This is to get around gson ignoring of inner class
private static final class TwillSpecificationTypeAdapterFactory implements TypeAdapterFactory {
+ @SuppressWarnings("unchecked")
@Override
public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
Class<?> rawType = type.getRawType();
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
index 90f9d18..f6f8dde 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
@@ -97,8 +97,7 @@ public final class Dependencies {
continue;
}
- InputStream is = classUrl.openStream();
- try {
+ try (InputStream is = classUrl.openStream()) {
// Visit the bytecode to lookup classes that the visiting class is depended on.
new ClassReader(ByteStreams.toByteArray(is)).accept(new DependencyClassVisitor(new DependencyAcceptor() {
@Override
@@ -109,8 +108,6 @@ public final class Dependencies {
}
}
}), ClassReader.SKIP_DEBUG + ClassReader.SKIP_FRAMES);
- } finally {
- is.close();
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/internal/utils/Networks.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/utils/Networks.java b/twill-core/src/main/java/org/apache/twill/internal/utils/Networks.java
index 8e7d736..f60c7b6 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/utils/Networks.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/utils/Networks.java
@@ -30,13 +30,8 @@ public final class Networks {
* @return A port number or -1 for failure.
*/
public static int getRandomPort() {
- try {
- ServerSocket socket = new ServerSocket(0);
- try {
- return socket.getLocalPort();
- } finally {
- socket.close();
- }
+ try (ServerSocket socket = new ServerSocket(0)) {
+ return socket.getLocalPort();
} catch (IOException e) {
return -1;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/launcher/FindFreePort.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/launcher/FindFreePort.java b/twill-core/src/main/java/org/apache/twill/launcher/FindFreePort.java
index 89e6b57..702a68d 100644
--- a/twill-core/src/main/java/org/apache/twill/launcher/FindFreePort.java
+++ b/twill-core/src/main/java/org/apache/twill/launcher/FindFreePort.java
@@ -25,11 +25,8 @@ import java.net.ServerSocket;
public class FindFreePort {
public static void main(String[] args) throws Exception {
- ServerSocket socket = new ServerSocket(0);
- try {
+ try (ServerSocket socket = new ServerSocket(0)) {
System.out.println(socket.getLocalPort());
- } finally {
- socket.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
index 028d638..3484349 100644
--- a/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
+++ b/twill-core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
@@ -112,8 +112,7 @@ public final class TwillLauncher {
}
private static void unJar(File jarFile, File targetDir) throws IOException {
- JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile));
- try {
+ try (JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile))) {
JarEntry jarEntry = jarInput.getNextJarEntry();
while (jarEntry != null) {
File target = new File(targetDir, jarEntry.getName());
@@ -125,22 +124,17 @@ public final class TwillLauncher {
}
jarEntry = jarInput.getNextJarEntry();
}
- } finally {
- jarInput.close();
}
}
private static void copy(InputStream is, File file) throws IOException {
byte[] buf = new byte[8192];
- OutputStream os = new BufferedOutputStream(new FileOutputStream(file));
- try {
+ try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) {
int len = is.read(buf);
while (len != -1) {
os.write(buf, 0, len);
len = is.read(buf);
}
- } finally {
- os.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
----------------------------------------------------------------------
diff --git a/twill-core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java b/twill-core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
index 38d0a05..16c6c78 100644
--- a/twill-core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
+++ b/twill-core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
@@ -69,8 +69,7 @@ public class ApplicationBundlerTest {
}
private void unjar(File jarFile, File targetDir) throws IOException {
- JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile));
- try {
+ try (JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile))) {
JarEntry jarEntry = jarInput.getNextJarEntry();
while (jarEntry != null) {
File target = new File(targetDir, jarEntry.getName());
@@ -83,8 +82,6 @@ public class ApplicationBundlerTest {
jarEntry = jarInput.getNextJarEntry();
}
- } finally {
- jarInput.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-ext/src/main/java/org/apache/twill/ext/BundledJarRunner.java
----------------------------------------------------------------------
diff --git a/twill-ext/src/main/java/org/apache/twill/ext/BundledJarRunner.java b/twill-ext/src/main/java/org/apache/twill/ext/BundledJarRunner.java
index 0398b8a..4b8641e 100644
--- a/twill-ext/src/main/java/org/apache/twill/ext/BundledJarRunner.java
+++ b/twill-ext/src/main/java/org/apache/twill/ext/BundledJarRunner.java
@@ -134,16 +134,11 @@ public class BundledJarRunner {
} else {
output.getParentFile().mkdirs();
- OutputStream os = new FileOutputStream(output);
- try {
- InputStream is = jarFile.getInputStream(entry);
- try {
- ByteStreams.copy(is, os);
- } finally {
- is.close();
- }
- } finally {
- os.close();
+ try (
+ OutputStream os = new FileOutputStream(output);
+ InputStream is = jarFile.getInputStream(entry)
+ ) {
+ ByteStreams.copy(is, os);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-ext/src/test/java/org/apache/twill/ext/BundledJarRunnerTest.java
----------------------------------------------------------------------
diff --git a/twill-ext/src/test/java/org/apache/twill/ext/BundledJarRunnerTest.java b/twill-ext/src/test/java/org/apache/twill/ext/BundledJarRunnerTest.java
index f534c90..8cfa150 100644
--- a/twill-ext/src/test/java/org/apache/twill/ext/BundledJarRunnerTest.java
+++ b/twill-ext/src/test/java/org/apache/twill/ext/BundledJarRunnerTest.java
@@ -18,7 +18,6 @@
package org.apache.twill.ext;
import com.google.common.io.ByteStreams;
-
import org.junit.Assert;
import org.junit.ClassRule;
import org.junit.Test;
@@ -87,18 +86,13 @@ public class BundledJarRunnerTest {
private void createJarFileFromClass(String className, File jarfile) throws IOException {
String classAsPath = className.replace(".", "/") + ".class";
String packagePath = classAsPath.substring(0, classAsPath.lastIndexOf("/") + 1);
- FileOutputStream fout = null;
- JarOutputStream jarout = null;
- try {
- fout = new FileOutputStream(jarfile.getAbsolutePath());
- jarout = new JarOutputStream(fout);
+ try (
+ JarOutputStream jarout = new JarOutputStream(new FileOutputStream(jarfile.getAbsolutePath()))
+ ) {
jarout.putNextEntry(new ZipEntry(packagePath));
jarout.putNextEntry(new ZipEntry(classAsPath));
jarout.write(getClassBytes(classAsPath));
jarout.closeEntry();
- } finally {
- jarout.close();
- fout.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index 8f8952b..1ebeabe 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -640,14 +640,10 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
}
private List<LocalFile> getLocalizeFiles() {
- try {
- Reader reader = Files.newReader(new File(Constants.Files.LOCALIZE_FILES), Charsets.UTF_8);
- try {
- return new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec())
- .create().fromJson(reader, new TypeToken<List<LocalFile>>() { }.getType());
- } finally {
- reader.close();
- }
+ try (Reader reader = Files.newReader(new File(Constants.Files.LOCALIZE_FILES), Charsets.UTF_8)) {
+ return new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec())
+ .create().fromJson(reader, new TypeToken<List<LocalFile>>() {
+ }.getType());
} catch (IOException e) {
throw Throwables.propagate(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
index c493b6c..0545b4b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -118,11 +118,8 @@ public final class TwillContainerMain extends ServiceMain {
File file = new File(Constants.Files.CREDENTIALS);
if (file.exists()) {
Credentials credentials = new Credentials();
- DataInputStream input = new DataInputStream(new FileInputStream(file));
- try {
+ try (DataInputStream input = new DataInputStream(new FileInputStream(file))) {
credentials.readTokenStorageStream(input);
- } finally {
- input.close();
}
UserGroupInformation.getCurrentUser().addCredentials(credentials);
@@ -165,11 +162,8 @@ public final class TwillContainerMain extends ServiceMain {
}
private static TwillSpecification loadTwillSpec(File specFile) throws IOException {
- Reader reader = Files.newReader(specFile, Charsets.UTF_8);
- try {
+ try (Reader reader = Files.newReader(specFile, Charsets.UTF_8)) {
return TwillSpecificationAdapter.create().fromJson(reader);
- } finally {
- reader.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnTwillService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnTwillService.java b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnTwillService.java
index 961bc30..64f81b4 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnTwillService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnTwillService.java
@@ -83,11 +83,8 @@ public abstract class AbstractYarnTwillService extends AbstractTwillService {
try {
Credentials credentials = new Credentials();
Location location = getSecureStoreLocation();
- DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()));
- try {
+ try (DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()))) {
credentials.readTokenStorageStream(input);
- } finally {
- input.close();
}
UserGroupInformation.getCurrentUser().addCredentials(credentials);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index ee889a8..3ae27bf 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -449,8 +449,7 @@ final class YarnTwillPreparer implements TwillPreparer {
// Serialize into a local temp file.
LOG.debug("Create and copy {}", Constants.Files.TWILL_SPEC);
Location location = createTempLocation(Constants.Files.TWILL_SPEC);
- Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
- try {
+ try (Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8)) {
EventHandlerSpecification eventHandler = spec.getEventHandler();
if (eventHandler == null) {
eventHandler = new LogOnlyEventHandler().configure();
@@ -459,8 +458,6 @@ final class YarnTwillPreparer implements TwillPreparer {
TwillSpecificationAdapter.create().toJson(
new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), spec.getPlacementPolicies(),
eventHandler), writer);
- } finally {
- writer.close();
}
LOG.debug("Done {}", Constants.Files.TWILL_SPEC);
@@ -501,11 +498,8 @@ final class YarnTwillPreparer implements TwillPreparer {
"Launcher jar should not have dependencies: %s", className);
try {
jarOut.putNextEntry(new JarEntry(className.replace('.', '/') + ".class"));
- InputStream is = classUrl.openStream();
- try {
+ try (InputStream is = classUrl.openStream()) {
ByteStreams.copy(is, jarOut);
- } finally {
- is.close();
}
} catch (IOException e) {
throw Throwables.propagate(e);
@@ -567,29 +561,21 @@ final class YarnTwillPreparer implements TwillPreparer {
Map<String, LocalFile> localize = ImmutableMap.copyOf(Maps.filterKeys(localFiles, Predicates.in(includes)));
LOG.debug("Create and copy {}", Constants.Files.LOCALIZE_FILES);
Location location = createTempLocation(Constants.Files.LOCALIZE_FILES);
- Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
- try {
+ try (Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8)) {
new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec())
.create().toJson(localize.values(), new TypeToken<List<LocalFile>>() {
}.getType(), writer);
- } finally {
- writer.close();
}
LOG.debug("Done {}", Constants.Files.LOCALIZE_FILES);
localFiles.put(Constants.Files.LOCALIZE_FILES, createLocalFile(Constants.Files.LOCALIZE_FILES, location));
}
private Location copyFromURL(URL url, Location target) throws IOException {
- InputStream is = url.openStream();
- try {
- OutputStream os = new BufferedOutputStream(target.getOutputStream());
- try {
- ByteStreams.copy(is, os);
- } finally {
- os.close();
- }
- } finally {
- is.close();
+ try (
+ InputStream is = url.openStream();
+ OutputStream os = new BufferedOutputStream(target.getOutputStream())
+ ) {
+ ByteStreams.copy(is, os);
}
return target;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 690d41e..73f467f 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -587,11 +587,8 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
// Try to read the old credentials.
Credentials credentials = new Credentials();
if (credentialsLocation.exists()) {
- DataInputStream is = new DataInputStream(new BufferedInputStream(credentialsLocation.getInputStream()));
- try {
+ try (DataInputStream is = new DataInputStream(new BufferedInputStream(credentialsLocation.getInputStream()))) {
credentials.readTokenStorageStream(is);
- } finally {
- is.close();
}
}
@@ -602,11 +599,8 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
Location tmpLocation = credentialsLocation.getTempFile(Constants.Files.CREDENTIALS);
// Save the credentials store with user-only permission.
- DataOutputStream os = new DataOutputStream(new BufferedOutputStream(tmpLocation.getOutputStream("600")));
- try {
+ try (DataOutputStream os = new DataOutputStream(new BufferedOutputStream(tmpLocation.getOutputStream("600")))) {
credentials.writeTokenStorageToStream(os);
- } finally {
- os.close();
}
// Rename the tmp file into the credentials location
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
index fb37820..18691a0 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
@@ -47,15 +47,14 @@ public final class DistributedShell extends AbstractTwillRunnable {
try {
Process process = new ProcessBuilder(ImmutableList.copyOf(Splitter.on(' ').split(cmd)))
.redirectErrorStream(true).start();
- BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charsets.US_ASCII));
- try {
+ try (
+ BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charsets.US_ASCII))
+ ) {
String line = reader.readLine();
while (line != null) {
LOG.info(line);
line = reader.readLine();
}
- } finally {
- reader.close();
}
} catch (IOException e) {
LOG.error("Fail to execute command " + cmd, e);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
index bb525a8..d635af1 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -81,16 +81,15 @@ public final class EchoServerTestRun extends BaseYarnTest {
for (Discoverable discoverable : echoServices) {
String msg = "Hello: " + discoverable.getSocketAddress();
- Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
- discoverable.getSocketAddress().getPort());
- try {
+ try (
+ Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
+ discoverable.getSocketAddress().getPort())
+ ) {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
writer.println(msg);
Assert.assertEquals(msg, reader.readLine());
- } finally {
- socket.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
index a0c195d..8525b3b 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
@@ -84,8 +84,7 @@ public final class FailureRestartTestRun extends BaseYarnTest {
Set<Integer> instances = Sets.newHashSet();
for (Discoverable discoverable : discoverables) {
InetSocketAddress socketAddress = discoverable.getSocketAddress();
- Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort());
- try {
+ try (Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort())) {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
@@ -95,8 +94,6 @@ public final class FailureRestartTestRun extends BaseYarnTest {
String line = reader.readLine();
Assert.assertTrue(line.endsWith(msg));
instances.add(Integer.parseInt(line.substring(0, line.length() - msg.length())));
- } finally {
- socket.close();
}
}
return instances;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
index fd146f2..e20dd64 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
@@ -69,8 +69,7 @@ public final class LocalFileTestRun extends BaseYarnTest {
Assert.assertTrue(waitForSize(discoverables, 1, 60));
InetSocketAddress socketAddress = discoverables.iterator().next().getSocketAddress();
- Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort());
- try {
+ try (Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort())) {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
@@ -78,8 +77,6 @@ public final class LocalFileTestRun extends BaseYarnTest {
writer.println(msg);
Assert.assertEquals(header, reader.readLine());
Assert.assertEquals(msg, reader.readLine());
- } finally {
- socket.close();
}
controller.terminate().get(120, TimeUnit.SECONDS);
@@ -99,12 +96,9 @@ public final class LocalFileTestRun extends BaseYarnTest {
public LocalFileApplication() throws Exception {
// Create a jar file that contains the header.txt file inside.
headerFile = tmpFolder.newFile("header.jar");
- JarOutputStream os = new JarOutputStream(new FileOutputStream(headerFile));
- try {
+ try (JarOutputStream os = new JarOutputStream(new FileOutputStream(headerFile))) {
os.putNextEntry(new JarEntry("header.txt"));
ByteStreams.copy(getClass().getClassLoader().getResourceAsStream("header.txt"), os);
- } finally {
- os.close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
index 50f1e08..86dfeaf 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
@@ -114,15 +114,14 @@ public final class ResourceReportTestRun extends BaseYarnTest {
// check environment of the runnable.
Discoverable discoverable = envEchoServices.iterator().next();
for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
- Socket socket = new Socket(discoverable.getSocketAddress().getHostName(),
- discoverable.getSocketAddress().getPort());
- try {
+ try (
+ Socket socket = new Socket(discoverable.getSocketAddress().getHostName(),
+ discoverable.getSocketAddress().getPort())
+ ) {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
writer.println(expected.getKey());
Assert.assertEquals(expected.getValue(), reader.readLine());
- } finally {
- socket.close();
}
}
@@ -165,13 +164,12 @@ public final class ResourceReportTestRun extends BaseYarnTest {
// cause a divide by 0 in one server
Discoverable discoverable = echoServices.iterator().next();
- Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
- discoverable.getSocketAddress().getPort());
- try {
+ try (
+ Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
+ discoverable.getSocketAddress().getPort())
+ ) {
PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
writer.println("0");
- } finally {
- socket.close();
}
// takes some time for app master to find out the container completed...
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/2175c664/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java b/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
index 8c37b78..fd38576 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
@@ -78,15 +78,10 @@ public abstract class SocketServer extends AbstractTwillRunnable {
public void run() {
try {
while (running) {
- try {
- Socket socket = serverSocket.accept();
- try {
- BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
- PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream()), true);
- handleRequest(reader, writer);
- } finally {
- socket.close();
- }
+ try (Socket socket = serverSocket.accept()) {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream()), true);
+ handleRequest(reader, writer);
} catch (SocketException e) {
LOG.info("Socket exception: " + e);
}
[12/25] incubator-twill git commit: TWILL-137 Be able to get log
level for each runnable via TwillController.getResourceReport
Posted by ch...@apache.org.
TWILL-137 Be able to get log level for each runnable via TwillController.getResourceReport
With TWILL-24 resolved, the client should be able to get the log level for an application
for each runnable.
This involves adding log level to TwillRunResources interface and its implementations.
Changes include adding log level to the TwillRunResources and modify unit tests to test
the new property.
This closes #46 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d864ed18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d864ed18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d864ed18
Branch: refs/heads/site
Commit: d864ed18089455bc39d0436980b3036e11f647bb
Parents: dc9320b
Author: hsaputra <hs...@apache.org>
Authored: Sun Jun 21 20:31:23 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 22 17:36:00 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/TwillController.java | 5 +-
.../org/apache/twill/api/TwillRunResources.java | 8 +++
.../internal/DefaultTwillRunResources.java | 12 +++-
.../internal/json/TwillRunResourcesCodec.java | 27 ++++++--
.../appmaster/ApplicationMasterService.java | 2 +-
.../internal/appmaster/RunningContainers.java | 10 ++-
.../org/apache/twill/yarn/LogLevelTestRun.java | 67 +++++++++++++++++++-
7 files changed, 116 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/api/TwillController.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillController.java b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
index 7d88bb7..b84f817 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillController.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
@@ -22,6 +22,7 @@ import org.apache.twill.discovery.Discoverable;
import org.apache.twill.discovery.ServiceDiscovered;
import java.util.concurrent.Future;
+import javax.annotation.Nullable;
/**
* For controlling a running application.
@@ -55,7 +56,9 @@ public interface TwillController extends ServiceController {
/**
* Get a snapshot of the resources used by the application, broken down by each runnable.
*
- * @return A {@link ResourceReport} containing information about resources used by the application.
+ * @return A {@link ResourceReport} containing information about resources used by the application or
+ * null in case the user calls this before the application completely starts.
*/
+ @Nullable
ResourceReport getResourceReport();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java b/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
index eea9619..1a665ea 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
@@ -17,6 +17,8 @@
*/
package org.apache.twill.api;
+import org.apache.twill.api.logging.LogEntry;
+
/**
* Information about the container the {@link TwillRunnable}
* is running in.
@@ -53,4 +55,10 @@ public interface TwillRunResources {
* @return the debug port of the container's JVM, or null if not debug-enabled.
*/
Integer getDebugPort();
+
+ /**
+ * @return the enabled log level for the container where the runnable is running in.
+ */
+ LogEntry.Level getLogLevel();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java b/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
index ae99daf..6671e63 100644
--- a/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
+++ b/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
@@ -18,6 +18,7 @@
package org.apache.twill.internal;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry.Level;
/**
* Straightforward implementation of {@link org.apache.twill.api.TwillRunResources}.
@@ -29,15 +30,17 @@ public class DefaultTwillRunResources implements TwillRunResources {
private final int memoryMB;
private final String host;
private final Integer debugPort;
+ private final Level logLevel;
- public DefaultTwillRunResources(int instanceId, String containerId,
- int cores, int memoryMB, String host, Integer debugPort) {
+ public DefaultTwillRunResources(int instanceId, String containerId, int cores, int memoryMB,
+ String host, Integer debugPort, Level logLevel) {
this.instanceId = instanceId;
this.containerId = containerId;
this.virtualCores = cores;
this.memoryMB = memoryMB;
this.host = host;
this.debugPort = debugPort;
+ this.logLevel = logLevel;
}
/**
@@ -87,6 +90,11 @@ public class DefaultTwillRunResources implements TwillRunResources {
}
@Override
+ public Level getLogLevel() {
+ return logLevel;
+ }
+
+ @Override
public boolean equals(Object o) {
if (!(o instanceof TwillRunResources)) {
return false;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java b/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
index 552469b..9a6f555 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
@@ -25,6 +25,7 @@ import com.google.gson.JsonParseException;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.internal.DefaultTwillRunResources;
import java.lang.reflect.Type;
@@ -34,18 +35,28 @@ import java.lang.reflect.Type;
*/
public final class TwillRunResourcesCodec implements JsonSerializer<TwillRunResources>,
JsonDeserializer<TwillRunResources> {
+ private final String CONTAINER_ID = "containerId";
+ private final String INSTANCE_ID = "instanceId";
+ private final String HOST = "host";
+ private final String MEMORY_MB = "memoryMB";
+ private final String VIRTUAL_CORES = "virtualCores";
+ private final String DEBUG_PORT = "debugPort";
+ private final String LOG_LEVEL = "logLevel";
@Override
public JsonElement serialize(TwillRunResources src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject json = new JsonObject();
- json.addProperty("containerId", src.getContainerId());
- json.addProperty("instanceId", src.getInstanceId());
- json.addProperty("host", src.getHost());
- json.addProperty("memoryMB", src.getMemoryMB());
- json.addProperty("virtualCores", src.getVirtualCores());
+ json.addProperty(CONTAINER_ID, src.getContainerId());
+ json.addProperty(INSTANCE_ID, src.getInstanceId());
+ json.addProperty(HOST, src.getHost());
+ json.addProperty(MEMORY_MB, src.getMemoryMB());
+ json.addProperty(VIRTUAL_CORES, src.getVirtualCores());
if (src.getDebugPort() != null) {
- json.addProperty("debugPort", src.getDebugPort());
+ json.addProperty(DEBUG_PORT, src.getDebugPort());
+ }
+ if (src.getLogLevel() != null) {
+ json.addProperty(LOG_LEVEL, src.getLogLevel().toString());
}
return json;
@@ -60,6 +71,8 @@ public final class TwillRunResourcesCodec implements JsonSerializer<TwillRunReso
jsonObj.get("virtualCores").getAsInt(),
jsonObj.get("memoryMB").getAsInt(),
jsonObj.get("host").getAsString(),
- jsonObj.has("debugPort") ? jsonObj.get("debugPort").getAsInt() : null);
+ jsonObj.has("debugPort") ? jsonObj.get("debugPort").getAsInt() : null,
+ jsonObj.has("logLevel") ? LogEntry.Level.valueOf(
+ jsonObj.get("logLevel").getAsString()) : LogEntry.Level.INFO);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index b85ce97..5c89a79 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -191,7 +191,7 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
appMasterContainerId.toString(),
Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)),
Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)),
- appMasterHost, null);
+ appMasterHost, null, null);
String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString();
return new RunningContainers(appId, appMasterResources, zkClient);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
index 3c82467..4a56229 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.RunId;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.internal.ContainerExitCodes;
import org.apache.twill.internal.ContainerInfo;
import org.apache.twill.internal.ContainerLiveNodeData;
import org.apache.twill.internal.DefaultResourceReport;
import org.apache.twill.internal.DefaultTwillRunResources;
+import org.apache.twill.internal.EnvKeys;
import org.apache.twill.internal.RunIds;
import org.apache.twill.internal.TwillContainerController;
import org.apache.twill.internal.TwillContainerLauncher;
@@ -132,7 +134,8 @@ final class RunningContainers {
containerInfo.getVirtualCores(),
containerInfo.getMemoryMB(),
containerInfo.getHost().getHostName(),
- controller);
+ controller,
+ System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL));
resourceReport.addRunResources(runnableName, resources);
containerStats.put(runnableName, containerInfo);
@@ -517,8 +520,9 @@ final class RunningContainers {
private DynamicTwillRunResources(int instanceId, String containerId,
int cores, int memoryMB, String host,
- TwillContainerController controller) {
- super(instanceId, containerId, cores, memoryMB, host, null);
+ TwillContainerController controller, String logLevel) {
+ super(instanceId, containerId, cores, memoryMB, host, null,
+ (logLevel != null) ? LogEntry.Level.valueOf(logLevel) : null);
this.controller = controller;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
index 7f33371..7990a5b 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
@@ -17,17 +17,24 @@
*/
package org.apache.twill.yarn;
+import com.google.common.base.Stopwatch;
import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.TwillSpecification;
import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.Threads;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.PrintWriter;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
@@ -68,17 +75,75 @@ public class LogLevelTestRun extends BaseYarnTest {
}
}
+ /**
+ * A test TwillApplication to test setting log level to DEBUG.
+ */
+ public static final class LogLevelTestApplication implements TwillApplication {
+
+ @Override
+ public TwillSpecification configure() {
+ return TwillSpecification.Builder.with()
+ .setName(LogLevelTestApplication.class.getSimpleName())
+ .withRunnable()
+ .add(LogLevelTestRunnable.class.getSimpleName(), new LogLevelTestRunnable()).noLocalFiles()
+ .anyOrder()
+ .build();
+ }
+ }
+
@Test
public void testDebugLogLevel()throws Exception {
YarnTwillRunnerService runner = getTwillRunner();
runner.start();
- TwillController controller = runner.prepare(new LogLevelTestRunnable())
+ // Set log level to DEBUG
+ TwillController controller = runner.prepare(new LogLevelTestApplication())
.setLogLevel(LogEntry.Level.DEBUG)
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
.start();
+ // Lets wait until the service is running
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.onRunning(new Runnable() {
+ @Override
+ public void run() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ Assert.assertTrue(running.await(200, TimeUnit.SECONDS));
+
+ LogEntry.Level logLevel = waitForLogLevel(controller, LogLevelTestRunnable.class.getSimpleName(), 5L,
+ TimeUnit.SECONDS);
+
+ // Verify we got DEBUG log level.
+ Assert.assertEquals(LogEntry.Level.DEBUG, logLevel);
+
controller.terminate().get(120, TimeUnit.SECONDS);
+
+ // Sleep a bit for full cleanup
+ TimeUnit.SECONDS.sleep(2);
}
+ // Need helper method here to wait for getting resource report because {@link TwillController#getResourceReport()}
+ // could return null if the application has not fully started.
+ private LogEntry.Level waitForLogLevel(TwillController controller, String runnable, long timeout,
+ TimeUnit timeoutUnit) throws InterruptedException {
+
+ Stopwatch stopwatch = new Stopwatch();
+ stopwatch.start();
+ do {
+ ResourceReport report = controller.getResourceReport();
+ if (report == null || report.getRunnableResources(runnable) == null) {
+ continue;
+ }
+ for (TwillRunResources resources : report.getRunnableResources(runnable)) {
+ if (resources.getLogLevel() != null) {
+ return resources.getLogLevel();
+ }
+ }
+ TimeUnit.MILLISECONDS.sleep(100);
+ } while (stopwatch.elapsedTime(timeoutUnit) < timeout);
+
+ return null;
+ }
}
[24/25] incubator-twill git commit: Fix a typo for other archive URL
from comits to commits in the pom.xml
Posted by ch...@apache.org.
Fix a typo for other archive URL from comits to commits in the pom.xml
This closes #55 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d835cafc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d835cafc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d835cafc
Branch: refs/heads/site
Commit: d835cafc90ffd9910ebdc63018d36ebce6ee50d0
Parents: ecec4b3
Author: hsaputra <hs...@apache.org>
Authored: Wed Jul 15 11:04:32 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Tue Jul 21 10:57:15 2015 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d835cafc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a2e1b9b..7685b58 100644
--- a/pom.xml
+++ b/pom.xml
@@ -163,7 +163,7 @@
<unsubscribe>commits-unsubscribe@twill.incubator.apache.org</unsubscribe>
<archive>http://mail-archives.apache.org/mod_mbox/twill-commits/</archive>
<otherArchives>
- <otherArchive>http://twill-comits.markmail.org</otherArchive>
+ <otherArchive>http://twill-commits.markmail.org</otherArchive>
</otherArchives>
</mailingList>
</mailingLists>
[13/25] incubator-twill git commit: Merge branch 'master' into site
Posted by ch...@apache.org.
Merge branch 'master' into site
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/356f9eae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/356f9eae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/356f9eae
Branch: refs/heads/site
Commit: 356f9eae23ad183f814ccbba6bb76e2daf46ba87
Parents: 5c6f578 d864ed1
Author: Terence Yim <ch...@apache.org>
Authored: Mon Jun 22 17:44:00 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 22 17:44:00 2015 -0700
----------------------------------------------------------------------
pom.xml | 8 +-
.../org/apache/twill/api/TwillController.java | 6 +-
.../org/apache/twill/api/TwillPreparer.java | 10 ++
.../org/apache/twill/api/TwillRunResources.java | 8 +
.../internal/DefaultTwillRunResources.java | 12 +-
.../twill/internal/ApplicationBundler.java | 15 +-
.../twill/internal/BasicTwillContext.java | 3 +-
.../java/org/apache/twill/internal/EnvKeys.java | 3 +
.../org/apache/twill/internal/JvmOptions.java | 32 ++++
.../apache/twill/internal/ProcessLauncher.java | 2 +-
.../twill/internal/TwillContainerLauncher.java | 10 ++
.../twill/internal/json/ArgumentsCodec.java | 10 +-
.../twill/internal/json/JvmOptionsCodec.java | 10 +-
.../internal/json/TwillRunResourcesCodec.java | 27 ++-
.../json/TwillSpecificationAdapter.java | 11 +-
.../twill/internal/logging/KafkaAppender.java | 2 +-
.../apache/twill/internal/state/Message.java | 2 +-
.../twill/internal/state/MessageCallback.java | 2 +-
.../twill/internal/state/SimpleMessage.java | 2 +-
.../twill/internal/utils/Dependencies.java | 5 +-
.../apache/twill/internal/utils/Networks.java | 9 +-
.../org/apache/twill/launcher/FindFreePort.java | 5 +-
.../apache/twill/launcher/TwillLauncher.java | 10 +-
.../apache/twill/internal/DebugOptionsTest.java | 99 +++++++++++
.../internal/utils/ApplicationBundlerTest.java | 5 +-
.../org/apache/twill/ext/BundledJarRunner.java | 15 +-
.../apache/twill/ext/BundledJarRunnerTest.java | 12 +-
.../internal/yarn/Hadoop20YarnAppClient.java | 5 +
.../internal/yarn/Hadoop21YarnAMClient.java | 2 +-
.../internal/yarn/Hadoop21YarnAppClient.java | 5 +
.../internal/yarn/Hadoop21YarnNMClient.java | 2 +-
.../org/apache/twill/internal/ServiceMain.java | 8 +-
.../appmaster/ApplicationMasterService.java | 19 +--
.../internal/appmaster/RunningContainers.java | 10 +-
.../internal/container/TwillContainerMain.java | 20 ++-
.../internal/yarn/AbstractYarnTwillService.java | 5 +-
.../yarn/VersionDetectYarnAppClientFactory.java | 2 +-
.../apache/twill/yarn/YarnTwillPreparer.java | 69 ++++----
.../twill/yarn/YarnTwillRunnerService.java | 13 +-
.../org/apache/twill/yarn/DistributedShell.java | 7 +-
.../apache/twill/yarn/EchoServerTestRun.java | 9 +-
.../twill/yarn/FailureRestartTestRun.java | 5 +-
.../org/apache/twill/yarn/LocalFileTestRun.java | 10 +-
.../org/apache/twill/yarn/LogLevelTestRun.java | 149 +++++++++++++++++
.../twill/yarn/ResourceReportTestRun.java | 18 +-
.../org/apache/twill/yarn/SocketServer.java | 13 +-
.../zookeeper/DefaultZKClientService.java | 164 +++++++++++++------
.../internal/zookeeper/LeaderElection.java | 12 +-
.../org/apache/twill/zookeeper/ZKClients.java | 10 +-
.../apache/twill/zookeeper/ZKClientTest.java | 85 +++++++---
50 files changed, 681 insertions(+), 296 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/356f9eae/pom.xml
----------------------------------------------------------------------
[23/25] incubator-twill git commit: (TWILL-142) Add longer time for
timeout in EchoServerTestRun to check whether the restart successful.
Posted by ch...@apache.org.
(TWILL-142) Add longer time for timeout in EchoServerTestRun to check whether the restart successful.
Bump up the timout for stopwatch to wait until all containers have been restarted.
With new feature in TWILL-116 being committed, the EchoServerTestRun test become bit flaky due to
sometimes the mini cluster could not allocate new containers in current proposed timeout which is 30s.
This closes #56 on Github
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/ecec4b3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/ecec4b3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/ecec4b3c
Branch: refs/heads/site
Commit: ecec4b3cdd757cbd45eeb30360421f06eba0fecf
Parents: 85a626a
Author: hsaputra <hs...@apache.org>
Authored: Wed Jul 15 15:59:05 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jul 16 15:52:30 2015 -0700
----------------------------------------------------------------------
.../internal/appmaster/ApplicationMasterService.java | 2 +-
.../java/org/apache/twill/yarn/EchoServerTestRun.java | 12 +++++++++---
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ecec4b3c/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index f76cd0b..cbf013b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -832,7 +832,7 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
for (Map.Entry<String, String> option : requestCommand.getOptions().entrySet()) {
String runnableName = option.getKey();
Set<Integer> restartedInstanceIds = GSON.fromJson(option.getValue(),
- new TypeToken<Set<Integer>>() {}.getType());
+ new TypeToken<Set<Integer>>() {}.getType());
LOG.debug("Start restarting runnable {} instances {}", runnableName, restartedInstanceIds);
restartRunnableInstances(runnableName, restartedInstanceIds);
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/ecec4b3c/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
index 0a8414e..3f0f20c 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -117,7 +117,8 @@ public final class EchoServerTestRun extends BaseYarnTest {
// Test restart on instances for runnable
Map<Integer, String> instanceIdToContainerId = Maps.newHashMap();
- ResourceReport report = waitForAfterRestartResourceReport(controller, "EchoServer", 30L, TimeUnit.SECONDS, 2, null);
+ ResourceReport report = waitForAfterRestartResourceReport(controller, "EchoServer", 15L,
+ TimeUnit.MINUTES, 2, null);
Assert.assertTrue(report != null);
Collection<TwillRunResources> runResources = report.getRunnableResources("EchoServer");
for (TwillRunResources twillRunResources : runResources) {
@@ -127,7 +128,7 @@ public final class EchoServerTestRun extends BaseYarnTest {
controller.restartAllInstances("EchoServer");
Assert.assertTrue(waitForSize(echoServices, 2, 120));
- report = waitForAfterRestartResourceReport(controller, "EchoServer", 30L, TimeUnit.SECONDS, 2,
+ report = waitForAfterRestartResourceReport(controller, "EchoServer", 15L, TimeUnit.MINUTES, 2,
instanceIdToContainerId);
Assert.assertTrue(report != null);
@@ -181,6 +182,7 @@ public final class EchoServerTestRun extends BaseYarnTest {
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
} else {
if (instanceIdToContainerId == null) {
+ LOG.info("Return resource report without comparing container ids.");
return report;
}
Collection<TwillRunResources> runResources = report.getRunnableResources(runnable);
@@ -189,17 +191,21 @@ public final class EchoServerTestRun extends BaseYarnTest {
int instanceId = twillRunResources.getInstanceId();
if (twillRunResources.getContainerId().equals(instanceIdToContainerId.get(instanceId))) {
// found same container id lets wait again.
+ LOG.warn("Found an instance id {} with same container id {} for restart all, let's wait for a while.",
+ instanceId, twillRunResources.getContainerId());
isSameContainer = true;
break;
}
}
if (!isSameContainer) {
- LOG.error("Unable to get different container ids for restart.");
+ LOG.info("Get set of different container ids for restart.");
return report;
}
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
}
} while (stopwatch.elapsedTime(timeoutUnit) < timeout);
+
+ LOG.error("Unable to get different container ids for restart.");
return null;
}
}
[09/25] incubator-twill git commit: TWILL-136 Override equals and
hashCode for JvmOptions.DebugOptions to test equality
Posted by ch...@apache.org.
TWILL-136 Override equals and hashCode for JvmOptions.DebugOptions to test equality
The current code for JvmOptions.DebugOptions does not have equals and hashCode overriden for equality test.
This would cause fail comparison for DebugOptions.NO_DEBUG when being used in YarnTwillPreparer:
final class YarnTwillPreparer implements TwillPreparer {
...
@Override
public TwillPreparer enableDebugging(boolean doSuspend, String... runnables) {
this.debugOptions = new JvmOptions.DebugOptions(true, doSuspend, ImmutableSet.copyOf(runnables));
return this;
}
....
private void saveJvmOptions(Map<String, LocalFile> localFiles) throws IOException {
if ((extraOptions == null || extraOptions.isEmpty()) &&
JvmOptions.DebugOptions.NO_DEBUG.equals(this.debugOptions)) {
// If no vm options, no need to localize the file.
return;
}
...
}
...
}
This closes #43 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/feee57ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/feee57ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/feee57ae
Branch: refs/heads/site
Commit: feee57aeb403d2d3fa7f0c5ee327c8a6c63284d6
Parents: 2054c5f
Author: hsaputra <hs...@apache.org>
Authored: Wed Jun 17 15:08:41 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Thu Jun 18 10:09:23 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/internal/JvmOptions.java | 32 +++++++
.../apache/twill/internal/DebugOptionsTest.java | 99 ++++++++++++++++++++
2 files changed, 131 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/feee57ae/twill-core/src/main/java/org/apache/twill/internal/JvmOptions.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/JvmOptions.java b/twill-core/src/main/java/org/apache/twill/internal/JvmOptions.java
index f827433..945561b 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/JvmOptions.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/JvmOptions.java
@@ -19,7 +19,9 @@ package org.apache.twill.internal;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
+import com.google.common.primitives.Booleans;
+import java.util.Objects;
import java.util.Set;
/**
@@ -83,5 +85,35 @@ public final class JvmOptions {
public boolean doDebug(String runnable) {
return doDebug && (runnables == null || runnables.contains(runnable));
}
+
+ @Override
+ public boolean equals(Object object) {
+ if (this == object) {
+ return true;
+ }
+ if (!(object instanceof DebugOptions)) {
+ return false;
+ }
+
+ DebugOptions that = (DebugOptions) object;
+ return (this.doDebug == that.doDebug()) && (this.doSuspend == this.doSuspend()) &&
+ Objects.equals(this.runnables, that.getRunnables());
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 17;
+ hash = 31 * hash + Booleans.hashCode(doDebug);
+ hash = 31 * hash + Booleans.hashCode(doSuspend);
+ hash = 31 * hash + Objects.hashCode(runnables);
+ return hash;
+ }
+
+ @Override
+ public String toString() {
+ return "{\"doDebug\":" + doDebug + ",\"doSuspend\":" + doSuspend + ",\"runnables\":" + (runnables != null ?
+ runnables.toString() : "none") + "}";
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/feee57ae/twill-core/src/test/java/org/apache/twill/internal/DebugOptionsTest.java
----------------------------------------------------------------------
diff --git a/twill-core/src/test/java/org/apache/twill/internal/DebugOptionsTest.java b/twill-core/src/test/java/org/apache/twill/internal/DebugOptionsTest.java
new file mode 100644
index 0000000..2a07987
--- /dev/null
+++ b/twill-core/src/test/java/org/apache/twill/internal/DebugOptionsTest.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import com.google.common.collect.Sets;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Set;
+
+/**
+ * Unit test for {@link org.apache.twill.internal.JvmOptions.DebugOptions} class
+ */
+public class DebugOptionsTest {
+
+ @Test
+ public void testNoDebug() {
+ JvmOptions.DebugOptions noDebug = new JvmOptions.DebugOptions(false, false, null);
+
+ Assert.assertEquals(JvmOptions.DebugOptions.NO_DEBUG, noDebug);
+ }
+
+ @Test
+ public void testWithNull() {
+ JvmOptions.DebugOptions noDebug = new JvmOptions.DebugOptions(false, false, null);
+
+ Assert.assertNotEquals(noDebug, null);
+ }
+
+ @Test
+ public void testDoDebug() {
+ JvmOptions.DebugOptions option1 = new JvmOptions.DebugOptions(true, false, null);
+
+ Assert.assertNotEquals(JvmOptions.DebugOptions.NO_DEBUG, option1);
+
+ JvmOptions.DebugOptions option2 = new JvmOptions.DebugOptions(true, false, null);
+
+ Assert.assertEquals(option1, option2);
+ }
+
+ @Test
+ public void testDoSuspend() {
+ JvmOptions.DebugOptions option1 = new JvmOptions.DebugOptions(true, true, null);
+
+ Assert.assertNotEquals(JvmOptions.DebugOptions.NO_DEBUG, option1);
+
+ JvmOptions.DebugOptions option2 = new JvmOptions.DebugOptions(true, true, null);
+
+ Assert.assertEquals(option1, option2);
+
+ }
+
+ @Test
+ public void testSameRunnables() {
+ Set<String> runnables = Sets.newHashSet();
+ runnables.add("runnable1");
+ runnables.add("runnable2");
+
+ JvmOptions.DebugOptions option1 = new JvmOptions.DebugOptions(false, false, runnables);
+
+ JvmOptions.DebugOptions option2 = new JvmOptions.DebugOptions(false, false, runnables);
+
+ Assert.assertEquals(option1, option2);
+ }
+
+ @Test
+ public void testDifferentRunnables() {
+ Set<String> runnables1 = Sets.newHashSet();
+ runnables1.add("runnable1");
+
+ JvmOptions.DebugOptions option1 = new JvmOptions.DebugOptions(true, false, runnables1);
+
+ Assert.assertNotEquals(option1, JvmOptions.DebugOptions.NO_DEBUG);
+
+ Set<String> runnables2 = Sets.newHashSet();
+ runnables2.add("runnable2-1");
+ runnables2.add("runnable2-2");
+
+ JvmOptions.DebugOptions option2 = new JvmOptions.DebugOptions(true, false, runnables2);
+
+ Assert.assertNotEquals(option1, option2);
+
+ }
+}
[06/25] incubator-twill git commit: TWILL-24 Add log level settable
for Twill application.
Posted by ch...@apache.org.
TWILL-24 Add log level settable for Twill application.
This PR allow client to set log level for the Twill applications in the container level.
The TwillPreparer now has new TwillPreparer#enableLogLevel method to pass log level that
will be set on env variable in app master and pass to app containers.
Add unit test to test the new feature
This closes #39 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d8462a44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d8462a44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d8462a44
Branch: refs/heads/site
Commit: d8462a44e004c729cf3bc5906993c5a71a7f0625
Parents: 0d3c5ee
Author: hsaputra <hs...@apache.org>
Authored: Thu Jun 11 16:07:46 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jun 12 18:02:56 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/TwillPreparer.java | 10 +++
.../java/org/apache/twill/internal/EnvKeys.java | 3 +
.../org/apache/twill/internal/ServiceMain.java | 8 +-
.../appmaster/ApplicationMasterService.java | 3 +
.../internal/container/TwillContainerMain.java | 8 ++
.../apache/twill/yarn/YarnTwillPreparer.java | 37 ++++++---
.../twill/yarn/YarnTwillRunnerService.java | 3 +-
.../org/apache/twill/yarn/LogLevelTestRun.java | 84 ++++++++++++++++++++
8 files changed, 141 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
index c7ae5e5..c2c62a1 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
@@ -17,6 +17,7 @@
*/
package org.apache.twill.api;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import java.net.URI;
@@ -186,6 +187,15 @@ public interface TwillPreparer {
TwillPreparer addSecureStore(SecureStore secureStore);
/**
+ * Set the log level for Twill applications running in a container.
+ *
+ * @param logLevel the {@link LogEntry.Level} that should be set.
+ * The level match the {@code Logback} levels.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer setLogLevel(LogEntry.Level logLevel);
+
+ /**
* Starts the application.
* @return A {@link TwillController} for controlling the running application.
*/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
index 5fa42f8..6ee6ac8 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
@@ -37,6 +37,9 @@ public final class EnvKeys {
public static final String TWILL_APP_DIR = "TWILL_APP_DIR";
public static final String TWILL_APP_NAME = "TWILL_APP_NAME";
+
+ public static final String TWILL_APP_LOG_LEVEL = "TWILL_APP_LOG_LEVEL";
+
public static final String TWILL_RUNNABLE_NAME = "TWILL_RUNNABLE_NAME";
public static final String TWILL_LOG_KAFKA_ZK = "TWILL_LOG_KAFKA_ZK";
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
index e986a99..d7ef72b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -227,7 +227,13 @@ public abstract class ServiceMain {
}
}
- private String getLoggerLevel(Logger logger) {
+ /**
+ * Override to return the right log level for the service
+ *
+ * @param logger the {@link Logger} instance of the service context.
+ * @return String of log level based on {@code slf4j} log levels.
+ */
+ protected String getLoggerLevel(Logger logger) {
if (logger instanceof ch.qos.logback.classic.Logger) {
return ((ch.qos.logback.classic.Logger) logger).getLevel().toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index 1ebeabe..f248750 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -603,6 +603,8 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
String runnableName = provisionRequest.getRuntimeSpec().getName();
LOG.info("Starting runnable {} with {}", runnableName, processLauncher);
+ LOG.debug("Log level for Twill runnable {} is {}", runnableName, System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL));
+
int containerCount = expectedContainers.getExpected(runnableName);
ProcessLauncher.PrepareLaunchContext launchContext = processLauncher.prepareLaunch(
@@ -611,6 +613,7 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
.put(EnvKeys.TWILL_FS_USER, System.getenv(EnvKeys.TWILL_FS_USER))
.put(EnvKeys.TWILL_APP_RUN_ID, runId.getId())
.put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.TWILL_APP_LOG_LEVEL, System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL))
.put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
.put(EnvKeys.TWILL_LOG_KAFKA_ZK, getKafkaZKConnect())
.build()
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
index 0545b4b..a29b27c 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -19,6 +19,7 @@ package org.apache.twill.internal.container;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.common.util.concurrent.AbstractService;
import com.google.common.util.concurrent.Service;
@@ -110,6 +111,13 @@ public final class TwillContainerMain extends ServiceMain {
new TwillZKPathService(containerZKClient, runId));
}
+ @Override
+ protected String getLoggerLevel(Logger logger) {
+ String appLogLevel = System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL);
+
+ return Strings.isNullOrEmpty(appLogLevel) ? super.getLoggerLevel(logger) : appLogLevel;
+ }
+
private static void loadSecureStore() throws IOException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 11ceba2..aca4728 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -50,6 +50,7 @@ import org.apache.twill.api.SecureStore;
import org.apache.twill.api.TwillController;
import org.apache.twill.api.TwillPreparer;
import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
@@ -127,10 +128,11 @@ final class YarnTwillPreparer implements TwillPreparer {
private String schedulerQueue;
private String extraOptions;
private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG;
+ private LogEntry.Level logLevel;
YarnTwillPreparer(YarnConfiguration yarnConfig, TwillSpecification twillSpec,
YarnAppClient yarnAppClient, ZKClient zkClient,
- LocationFactory locationFactory, String extraOptions,
+ LocationFactory locationFactory, String extraOptions, LogEntry.Level logLevel,
YarnTwillControllerFactory controllerFactory) {
this.yarnConfig = yarnConfig;
this.twillSpec = twillSpec;
@@ -144,6 +146,7 @@ final class YarnTwillPreparer implements TwillPreparer {
Configs.Defaults.JAVA_RESERVED_MEMORY_MB);
this.user = System.getProperty("user.name");
this.extraOptions = extraOptions;
+ this.logLevel = logLevel;
}
@Override
@@ -251,6 +254,12 @@ final class YarnTwillPreparer implements TwillPreparer {
}
@Override
+ public TwillPreparer setLogLevel(LogEntry.Level logLevel) {
+ this.logLevel = logLevel;
+ return this;
+ }
+
+ @Override
public TwillController start() {
try {
final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue);
@@ -287,18 +296,20 @@ final class YarnTwillPreparer implements TwillPreparer {
// appMaster.jar
// org.apache.twill.internal.appmaster.ApplicationMasterMain
// false
- return launcher.prepareLaunch(
- ImmutableMap.<String, String>builder()
- .put(EnvKeys.TWILL_FS_USER, fsUser)
- .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString())
- .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
- .put(EnvKeys.TWILL_RUN_ID, runId.getId())
- .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
- .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
- .put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS))
- .build(),
- localFiles.values(), credentials
- ).addCommand(
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder()
+ .put(EnvKeys.TWILL_FS_USER, fsUser)
+ .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString())
+ .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
+ .put(EnvKeys.TWILL_RUN_ID, runId.getId())
+ .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
+ .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
+ if (logLevel != null) {
+ LOG.debug("Log level is set to {} for the Twill application.", logLevel);
+ builder.put(EnvKeys.TWILL_APP_LOG_LEVEL, logLevel.toString());
+ }
+ return launcher.prepareLaunch(builder.build(), localFiles.values(), credentials)
+ .addCommand(
"$JAVA_HOME/bin/java",
"-Djava.io.tmpdir=tmp",
"-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR,
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 73f467f..8a13017 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -58,6 +58,7 @@ import org.apache.twill.api.TwillPreparer;
import org.apache.twill.api.TwillRunnable;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.common.Cancellable;
import org.apache.twill.common.Threads;
@@ -277,7 +278,7 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
final String appName = twillSpec.getName();
return new YarnTwillPreparer(yarnConfig, twillSpec, yarnAppClient, zkClientService, locationFactory, jvmOptions,
- new YarnTwillControllerFactory() {
+ LogEntry.Level.INFO, new YarnTwillControllerFactory() {
@Override
public YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
Callable<ProcessController<YarnApplicationReport>> startUp) {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
new file mode 100644
index 0000000..7f33371
--- /dev/null
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.logging.LogEntry;
+import org.apache.twill.api.logging.PrinterLogHandler;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintWriter;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test class whether enable certain log level for application container works.
+ */
+public class LogLevelTestRun extends BaseYarnTest {
+
+ /**
+ * A test of TwillRunnable to see if the DEBUG log level is enabled.
+ * */
+ public static final class LogLevelTestRunnable extends AbstractTwillRunnable {
+ public static final Logger LOG = LoggerFactory.getLogger(LogLevelTestRunnable.class);
+
+ private volatile Thread runThread;
+
+ @Override
+ public void run() {
+ this.runThread = Thread.currentThread();
+ while (!Thread.interrupted()) {
+ // check if DEBUG log level is enabled
+ boolean isDebug = LOG.isDebugEnabled();
+
+ Assert.assertTrue(isDebug);
+
+ try {
+ TimeUnit.MILLISECONDS.sleep(200);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (runThread != null) {
+ runThread.interrupt();
+ }
+ }
+ }
+
+ @Test
+ public void testDebugLogLevel()throws Exception {
+ YarnTwillRunnerService runner = getTwillRunner();
+ runner.start();
+
+ TwillController controller = runner.prepare(new LogLevelTestRunnable())
+ .setLogLevel(LogEntry.Level.DEBUG)
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
+ .start();
+
+ controller.terminate().get(120, TimeUnit.SECONDS);
+ }
+
+}
[10/25] incubator-twill git commit: Simple change to add missing
Javadoc to class header and public method.
Posted by ch...@apache.org.
Simple change to add missing Javadoc to class header and public method.
This closes #44 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/7350f285
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/7350f285
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/7350f285
Branch: refs/heads/site
Commit: 7350f2856dd2a45119bc6449d3ae0630acf72bdf
Parents: feee57a
Author: hsaputra <hs...@apache.org>
Authored: Thu Jun 18 11:45:05 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 22 16:02:25 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/twill/zookeeper/ZKClients.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/7350f285/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
----------------------------------------------------------------------
diff --git a/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java b/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
index f67c1bd..13b5827 100644
--- a/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
+++ b/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
@@ -22,7 +22,7 @@ import org.apache.twill.internal.zookeeper.NamespaceZKClient;
import org.apache.twill.internal.zookeeper.RewatchOnExpireZKClient;
/**
- *
+ * Utility class to create {@link ZKClient} instances.
*/
public final class ZKClients {
@@ -51,7 +51,13 @@ public final class ZKClients {
return new FailureRetryZKClient(client, retryStrategy);
}
-
+ /**
+ * Creates a {@link ZKClient} that will add prefix namespace for every paths.
+ *
+ * @param zkClient The {@link ZKClient} for operations delegation.
+ * @param namespace The prefix namespace to be prepended to paths.
+ * @return A {@link ZKClient} that will add namespace to every path.
+ */
public static ZKClient namespace(ZKClient zkClient, String namespace) {
return new NamespaceZKClient(zkClient, namespace);
}
[25/25] incubator-twill git commit: Merge branch 'master' into site
Posted by ch...@apache.org.
Merge branch 'master' into site
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/1fb6c78d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/1fb6c78d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/1fb6c78d
Branch: refs/heads/site
Commit: 1fb6c78da4d33085c2e347989e34c2259cf2bb97
Parents: 356f9ea d835caf
Author: Terence Yim <ch...@apache.org>
Authored: Fri Jul 24 11:29:19 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jul 24 11:29:19 2015 -0700
----------------------------------------------------------------------
pom.xml | 4 +-
twill-api/pom.xml | 2 +-
.../org/apache/twill/api/ClassAcceptor.java | 39 ++++++++++
.../org/apache/twill/api/ResourceReport.java | 10 +--
.../apache/twill/api/ResourceSpecification.java | 6 +-
.../org/apache/twill/api/TwillController.java | 28 +++++++
.../org/apache/twill/api/TwillPreparer.java | 20 +++++
twill-common/pom.xml | 2 +-
.../org/apache/twill/filesystem/Location.java | 2 +-
twill-core/pom.xml | 2 +-
.../twill/internal/AbstractTwillController.java | 81 +++++++++++++++++++-
.../internal/AbstractZKServiceController.java | 1 +
.../twill/internal/ApplicationBundler.java | 80 ++++++++++---------
.../org/apache/twill/internal/Constants.java | 7 ++
.../org/apache/twill/internal/ZKMessages.java | 2 +-
.../internal/kafka/EmbeddedKafkaServer.java | 74 ++++++++++++++++--
.../apache/twill/internal/state/Message.java | 3 +-
.../twill/internal/state/SystemMessages.java | 27 ++++++-
.../twill/internal/utils/Dependencies.java | 18 +----
.../apache/twill/kafka/client/Compression.java | 2 +-
.../twill/kafka/client/KafkaPublisher.java | 2 +-
.../apache/twill/launcher/TwillLauncher.java | 34 ++++----
twill-discovery-api/pom.xml | 2 +-
twill-discovery-core/pom.xml | 2 +-
twill-examples/echo/pom.xml | 2 +-
twill-examples/pom.xml | 2 +-
twill-examples/yarn/pom.xml | 2 +-
.../apache/twill/example/yarn/HelloWorld.java | 26 ++++++-
twill-ext/pom.xml | 2 +-
twill-java8-test/pom.xml | 2 +-
twill-yarn/pom.xml | 2 +-
.../yarn/Hadoop21YarnContainerInfo.java | 2 +-
.../org/apache/twill/internal/ServiceMain.java | 2 +-
.../appmaster/ApplicationMasterMain.java | 8 +-
.../appmaster/ApplicationMasterService.java | 79 +++++++++++++++++++
.../internal/appmaster/RunningContainers.java | 65 ++++++++++------
.../internal/appmaster/TrackerService.java | 1 -
.../apache/twill/yarn/YarnTwillPreparer.java | 40 ++++++++--
.../apache/twill/yarn/EchoServerTestRun.java | 76 ++++++++++++++++++
twill-zookeeper/pom.xml | 2 +-
40 files changed, 625 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1fb6c78d/pom.xml
----------------------------------------------------------------------