You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2015/06/13 03:03:04 UTC
incubator-twill git commit: TWILL-24 Add log level settable for Twill
application.
Repository: incubator-twill
Updated Branches:
refs/heads/master 0d3c5eea4 -> d8462a44e
TWILL-24 Add log level settable for Twill application.
This PR allow client to set log level for the Twill applications in the container level.
The TwillPreparer now has new TwillPreparer#enableLogLevel method to pass log level that
will be set on env variable in app master and pass to app containers.
Add unit test to test the new feature
This closes #39 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d8462a44
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d8462a44
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d8462a44
Branch: refs/heads/master
Commit: d8462a44e004c729cf3bc5906993c5a71a7f0625
Parents: 0d3c5ee
Author: hsaputra <hs...@apache.org>
Authored: Thu Jun 11 16:07:46 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Fri Jun 12 18:02:56 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/TwillPreparer.java | 10 +++
.../java/org/apache/twill/internal/EnvKeys.java | 3 +
.../org/apache/twill/internal/ServiceMain.java | 8 +-
.../appmaster/ApplicationMasterService.java | 3 +
.../internal/container/TwillContainerMain.java | 8 ++
.../apache/twill/yarn/YarnTwillPreparer.java | 37 ++++++---
.../twill/yarn/YarnTwillRunnerService.java | 3 +-
.../org/apache/twill/yarn/LogLevelTestRun.java | 84 ++++++++++++++++++++
8 files changed, 141 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
index c7ae5e5..c2c62a1 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillPreparer.java
@@ -17,6 +17,7 @@
*/
package org.apache.twill.api;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import java.net.URI;
@@ -186,6 +187,15 @@ public interface TwillPreparer {
TwillPreparer addSecureStore(SecureStore secureStore);
/**
+ * Set the log level for Twill applications running in a container.
+ *
+ * @param logLevel the {@link LogEntry.Level} that should be set.
+ * The level match the {@code Logback} levels.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer setLogLevel(LogEntry.Level logLevel);
+
+ /**
* Starts the application.
* @return A {@link TwillController} for controlling the running application.
*/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
index 5fa42f8..6ee6ac8 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/EnvKeys.java
@@ -37,6 +37,9 @@ public final class EnvKeys {
public static final String TWILL_APP_DIR = "TWILL_APP_DIR";
public static final String TWILL_APP_NAME = "TWILL_APP_NAME";
+
+ public static final String TWILL_APP_LOG_LEVEL = "TWILL_APP_LOG_LEVEL";
+
public static final String TWILL_RUNNABLE_NAME = "TWILL_RUNNABLE_NAME";
public static final String TWILL_LOG_KAFKA_ZK = "TWILL_LOG_KAFKA_ZK";
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
index e986a99..d7ef72b 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -227,7 +227,13 @@ public abstract class ServiceMain {
}
}
- private String getLoggerLevel(Logger logger) {
+ /**
+ * Override to return the right log level for the service
+ *
+ * @param logger the {@link Logger} instance of the service context.
+ * @return String of log level based on {@code slf4j} log levels.
+ */
+ protected String getLoggerLevel(Logger logger) {
if (logger instanceof ch.qos.logback.classic.Logger) {
return ((ch.qos.logback.classic.Logger) logger).getLevel().toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index 1ebeabe..f248750 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -603,6 +603,8 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
String runnableName = provisionRequest.getRuntimeSpec().getName();
LOG.info("Starting runnable {} with {}", runnableName, processLauncher);
+ LOG.debug("Log level for Twill runnable {} is {}", runnableName, System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL));
+
int containerCount = expectedContainers.getExpected(runnableName);
ProcessLauncher.PrepareLaunchContext launchContext = processLauncher.prepareLaunch(
@@ -611,6 +613,7 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
.put(EnvKeys.TWILL_FS_USER, System.getenv(EnvKeys.TWILL_FS_USER))
.put(EnvKeys.TWILL_APP_RUN_ID, runId.getId())
.put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.TWILL_APP_LOG_LEVEL, System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL))
.put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
.put(EnvKeys.TWILL_LOG_KAFKA_ZK, getKafkaZKConnect())
.build()
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
index 0545b4b..a29b27c 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -19,6 +19,7 @@ package org.apache.twill.internal.container;
import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
import com.google.common.io.Files;
import com.google.common.util.concurrent.AbstractService;
import com.google.common.util.concurrent.Service;
@@ -110,6 +111,13 @@ public final class TwillContainerMain extends ServiceMain {
new TwillZKPathService(containerZKClient, runId));
}
+ @Override
+ protected String getLoggerLevel(Logger logger) {
+ String appLogLevel = System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL);
+
+ return Strings.isNullOrEmpty(appLogLevel) ? super.getLoggerLevel(logger) : appLogLevel;
+ }
+
private static void loadSecureStore() throws IOException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
index 11ceba2..aca4728 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -50,6 +50,7 @@ import org.apache.twill.api.SecureStore;
import org.apache.twill.api.TwillController;
import org.apache.twill.api.TwillPreparer;
import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.filesystem.Location;
import org.apache.twill.filesystem.LocationFactory;
@@ -127,10 +128,11 @@ final class YarnTwillPreparer implements TwillPreparer {
private String schedulerQueue;
private String extraOptions;
private JvmOptions.DebugOptions debugOptions = JvmOptions.DebugOptions.NO_DEBUG;
+ private LogEntry.Level logLevel;
YarnTwillPreparer(YarnConfiguration yarnConfig, TwillSpecification twillSpec,
YarnAppClient yarnAppClient, ZKClient zkClient,
- LocationFactory locationFactory, String extraOptions,
+ LocationFactory locationFactory, String extraOptions, LogEntry.Level logLevel,
YarnTwillControllerFactory controllerFactory) {
this.yarnConfig = yarnConfig;
this.twillSpec = twillSpec;
@@ -144,6 +146,7 @@ final class YarnTwillPreparer implements TwillPreparer {
Configs.Defaults.JAVA_RESERVED_MEMORY_MB);
this.user = System.getProperty("user.name");
this.extraOptions = extraOptions;
+ this.logLevel = logLevel;
}
@Override
@@ -251,6 +254,12 @@ final class YarnTwillPreparer implements TwillPreparer {
}
@Override
+ public TwillPreparer setLogLevel(LogEntry.Level logLevel) {
+ this.logLevel = logLevel;
+ return this;
+ }
+
+ @Override
public TwillController start() {
try {
final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(twillSpec, schedulerQueue);
@@ -287,18 +296,20 @@ final class YarnTwillPreparer implements TwillPreparer {
// appMaster.jar
// org.apache.twill.internal.appmaster.ApplicationMasterMain
// false
- return launcher.prepareLaunch(
- ImmutableMap.<String, String>builder()
- .put(EnvKeys.TWILL_FS_USER, fsUser)
- .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString())
- .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
- .put(EnvKeys.TWILL_RUN_ID, runId.getId())
- .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
- .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
- .put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS))
- .build(),
- localFiles.values(), credentials
- ).addCommand(
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.<String, String>builder()
+ .put(EnvKeys.TWILL_FS_USER, fsUser)
+ .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString())
+ .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
+ .put(EnvKeys.TWILL_RUN_ID, runId.getId())
+ .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
+ .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.YARN_RM_SCHEDULER_ADDRESS, yarnConfig.get(YarnConfiguration.RM_SCHEDULER_ADDRESS));
+ if (logLevel != null) {
+ LOG.debug("Log level is set to {} for the Twill application.", logLevel);
+ builder.put(EnvKeys.TWILL_APP_LOG_LEVEL, logLevel.toString());
+ }
+ return launcher.prepareLaunch(builder.build(), localFiles.values(), credentials)
+ .addCommand(
"$JAVA_HOME/bin/java",
"-Djava.io.tmpdir=tmp",
"-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR,
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
index 73f467f..8a13017 100644
--- a/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -58,6 +58,7 @@ import org.apache.twill.api.TwillPreparer;
import org.apache.twill.api.TwillRunnable;
import org.apache.twill.api.TwillRunnerService;
import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.LogHandler;
import org.apache.twill.common.Cancellable;
import org.apache.twill.common.Threads;
@@ -277,7 +278,7 @@ public final class YarnTwillRunnerService implements TwillRunnerService {
final String appName = twillSpec.getName();
return new YarnTwillPreparer(yarnConfig, twillSpec, yarnAppClient, zkClientService, locationFactory, jvmOptions,
- new YarnTwillControllerFactory() {
+ LogEntry.Level.INFO, new YarnTwillControllerFactory() {
@Override
public YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
Callable<ProcessController<YarnApplicationReport>> startUp) {
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d8462a44/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
new file mode 100644
index 0000000..7f33371
--- /dev/null
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.logging.LogEntry;
+import org.apache.twill.api.logging.PrinterLogHandler;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.PrintWriter;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test class whether enable certain log level for application container works.
+ */
+public class LogLevelTestRun extends BaseYarnTest {
+
+ /**
+ * A test of TwillRunnable to see if the DEBUG log level is enabled.
+ * */
+ public static final class LogLevelTestRunnable extends AbstractTwillRunnable {
+ public static final Logger LOG = LoggerFactory.getLogger(LogLevelTestRunnable.class);
+
+ private volatile Thread runThread;
+
+ @Override
+ public void run() {
+ this.runThread = Thread.currentThread();
+ while (!Thread.interrupted()) {
+ // check if DEBUG log level is enabled
+ boolean isDebug = LOG.isDebugEnabled();
+
+ Assert.assertTrue(isDebug);
+
+ try {
+ TimeUnit.MILLISECONDS.sleep(200);
+ } catch (InterruptedException e) {
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ if (runThread != null) {
+ runThread.interrupt();
+ }
+ }
+ }
+
+ @Test
+ public void testDebugLogLevel()throws Exception {
+ YarnTwillRunnerService runner = getTwillRunner();
+ runner.start();
+
+ TwillController controller = runner.prepare(new LogLevelTestRunnable())
+ .setLogLevel(LogEntry.Level.DEBUG)
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
+ .start();
+
+ controller.terminate().get(120, TimeUnit.SECONDS);
+ }
+
+}