You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2015/07/24 20:29:46 UTC
[12/25] incubator-twill git commit: TWILL-137 Be able to get log
level for each runnable via TwillController.getResourceReport
TWILL-137 Be able to get log level for each runnable via TwillController.getResourceReport
With TWILL-24 resolved, the client should be able to get the log level for an application
for each runnable.
This involves adding log level to TwillRunResources interface and its implementations.
Changes include adding log level to the TwillRunResources and modify unit tests to test
the new property.
This closes #46 on GitHub
Signed-off-by: Terence Yim <ch...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/d864ed18
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/d864ed18
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/d864ed18
Branch: refs/heads/site
Commit: d864ed18089455bc39d0436980b3036e11f647bb
Parents: dc9320b
Author: hsaputra <hs...@apache.org>
Authored: Sun Jun 21 20:31:23 2015 -0700
Committer: Terence Yim <ch...@apache.org>
Committed: Mon Jun 22 17:36:00 2015 -0700
----------------------------------------------------------------------
.../org/apache/twill/api/TwillController.java | 5 +-
.../org/apache/twill/api/TwillRunResources.java | 8 +++
.../internal/DefaultTwillRunResources.java | 12 +++-
.../internal/json/TwillRunResourcesCodec.java | 27 ++++++--
.../appmaster/ApplicationMasterService.java | 2 +-
.../internal/appmaster/RunningContainers.java | 10 ++-
.../org/apache/twill/yarn/LogLevelTestRun.java | 67 +++++++++++++++++++-
7 files changed, 116 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/api/TwillController.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillController.java b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
index 7d88bb7..b84f817 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillController.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillController.java
@@ -22,6 +22,7 @@ import org.apache.twill.discovery.Discoverable;
import org.apache.twill.discovery.ServiceDiscovered;
import java.util.concurrent.Future;
+import javax.annotation.Nullable;
/**
* For controlling a running application.
@@ -55,7 +56,9 @@ public interface TwillController extends ServiceController {
/**
* Get a snapshot of the resources used by the application, broken down by each runnable.
*
- * @return A {@link ResourceReport} containing information about resources used by the application.
+ * @return A {@link ResourceReport} containing information about resources used by the application or
+ * null in case the user calls this before the application completely starts.
*/
+ @Nullable
ResourceReport getResourceReport();
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java b/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
index eea9619..1a665ea 100644
--- a/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
+++ b/twill-api/src/main/java/org/apache/twill/api/TwillRunResources.java
@@ -17,6 +17,8 @@
*/
package org.apache.twill.api;
+import org.apache.twill.api.logging.LogEntry;
+
/**
* Information about the container the {@link TwillRunnable}
* is running in.
@@ -53,4 +55,10 @@ public interface TwillRunResources {
* @return the debug port of the container's JVM, or null if not debug-enabled.
*/
Integer getDebugPort();
+
+ /**
+ * @return the enabled log level for the container where the runnable is running in.
+ */
+ LogEntry.Level getLogLevel();
+
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
----------------------------------------------------------------------
diff --git a/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java b/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
index ae99daf..6671e63 100644
--- a/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
+++ b/twill-api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
@@ -18,6 +18,7 @@
package org.apache.twill.internal;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry.Level;
/**
* Straightforward implementation of {@link org.apache.twill.api.TwillRunResources}.
@@ -29,15 +30,17 @@ public class DefaultTwillRunResources implements TwillRunResources {
private final int memoryMB;
private final String host;
private final Integer debugPort;
+ private final Level logLevel;
- public DefaultTwillRunResources(int instanceId, String containerId,
- int cores, int memoryMB, String host, Integer debugPort) {
+ public DefaultTwillRunResources(int instanceId, String containerId, int cores, int memoryMB,
+ String host, Integer debugPort, Level logLevel) {
this.instanceId = instanceId;
this.containerId = containerId;
this.virtualCores = cores;
this.memoryMB = memoryMB;
this.host = host;
this.debugPort = debugPort;
+ this.logLevel = logLevel;
}
/**
@@ -87,6 +90,11 @@ public class DefaultTwillRunResources implements TwillRunResources {
}
@Override
+ public Level getLogLevel() {
+ return logLevel;
+ }
+
+ @Override
public boolean equals(Object o) {
if (!(o instanceof TwillRunResources)) {
return false;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
----------------------------------------------------------------------
diff --git a/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java b/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
index 552469b..9a6f555 100644
--- a/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
+++ b/twill-core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
@@ -25,6 +25,7 @@ import com.google.gson.JsonParseException;
import com.google.gson.JsonSerializationContext;
import com.google.gson.JsonSerializer;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.internal.DefaultTwillRunResources;
import java.lang.reflect.Type;
@@ -34,18 +35,28 @@ import java.lang.reflect.Type;
*/
public final class TwillRunResourcesCodec implements JsonSerializer<TwillRunResources>,
JsonDeserializer<TwillRunResources> {
+ private final String CONTAINER_ID = "containerId";
+ private final String INSTANCE_ID = "instanceId";
+ private final String HOST = "host";
+ private final String MEMORY_MB = "memoryMB";
+ private final String VIRTUAL_CORES = "virtualCores";
+ private final String DEBUG_PORT = "debugPort";
+ private final String LOG_LEVEL = "logLevel";
@Override
public JsonElement serialize(TwillRunResources src, Type typeOfSrc, JsonSerializationContext context) {
JsonObject json = new JsonObject();
- json.addProperty("containerId", src.getContainerId());
- json.addProperty("instanceId", src.getInstanceId());
- json.addProperty("host", src.getHost());
- json.addProperty("memoryMB", src.getMemoryMB());
- json.addProperty("virtualCores", src.getVirtualCores());
+ json.addProperty(CONTAINER_ID, src.getContainerId());
+ json.addProperty(INSTANCE_ID, src.getInstanceId());
+ json.addProperty(HOST, src.getHost());
+ json.addProperty(MEMORY_MB, src.getMemoryMB());
+ json.addProperty(VIRTUAL_CORES, src.getVirtualCores());
if (src.getDebugPort() != null) {
- json.addProperty("debugPort", src.getDebugPort());
+ json.addProperty(DEBUG_PORT, src.getDebugPort());
+ }
+ if (src.getLogLevel() != null) {
+ json.addProperty(LOG_LEVEL, src.getLogLevel().toString());
}
return json;
@@ -60,6 +71,8 @@ public final class TwillRunResourcesCodec implements JsonSerializer<TwillRunReso
jsonObj.get("virtualCores").getAsInt(),
jsonObj.get("memoryMB").getAsInt(),
jsonObj.get("host").getAsString(),
- jsonObj.has("debugPort") ? jsonObj.get("debugPort").getAsInt() : null);
+ jsonObj.has("debugPort") ? jsonObj.get("debugPort").getAsInt() : null,
+ jsonObj.has("logLevel") ? LogEntry.Level.valueOf(
+ jsonObj.get("logLevel").getAsString()) : LogEntry.Level.INFO);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
index b85ce97..5c89a79 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -191,7 +191,7 @@ public final class ApplicationMasterService extends AbstractYarnTwillService imp
appMasterContainerId.toString(),
Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)),
Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)),
- appMasterHost, null);
+ appMasterHost, null, null);
String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString();
return new RunningContainers(appId, appMasterResources, zkClient);
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
index 3c82467..4a56229 100644
--- a/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
+++ b/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
@@ -36,11 +36,13 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.twill.api.ResourceReport;
import org.apache.twill.api.RunId;
import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.internal.ContainerExitCodes;
import org.apache.twill.internal.ContainerInfo;
import org.apache.twill.internal.ContainerLiveNodeData;
import org.apache.twill.internal.DefaultResourceReport;
import org.apache.twill.internal.DefaultTwillRunResources;
+import org.apache.twill.internal.EnvKeys;
import org.apache.twill.internal.RunIds;
import org.apache.twill.internal.TwillContainerController;
import org.apache.twill.internal.TwillContainerLauncher;
@@ -132,7 +134,8 @@ final class RunningContainers {
containerInfo.getVirtualCores(),
containerInfo.getMemoryMB(),
containerInfo.getHost().getHostName(),
- controller);
+ controller,
+ System.getenv(EnvKeys.TWILL_APP_LOG_LEVEL));
resourceReport.addRunResources(runnableName, resources);
containerStats.put(runnableName, containerInfo);
@@ -517,8 +520,9 @@ final class RunningContainers {
private DynamicTwillRunResources(int instanceId, String containerId,
int cores, int memoryMB, String host,
- TwillContainerController controller) {
- super(instanceId, containerId, cores, memoryMB, host, null);
+ TwillContainerController controller, String logLevel) {
+ super(instanceId, containerId, cores, memoryMB, host, null,
+ (logLevel != null) ? LogEntry.Level.valueOf(logLevel) : null);
this.controller = controller;
}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/d864ed18/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
----------------------------------------------------------------------
diff --git a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
index 7f33371..7990a5b 100644
--- a/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
+++ b/twill-yarn/src/test/java/org/apache/twill/yarn/LogLevelTestRun.java
@@ -17,17 +17,24 @@
*/
package org.apache.twill.yarn;
+import com.google.common.base.Stopwatch;
import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.TwillApplication;
import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.TwillSpecification;
import org.apache.twill.api.logging.LogEntry;
import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.Threads;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.PrintWriter;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
/**
@@ -68,17 +75,75 @@ public class LogLevelTestRun extends BaseYarnTest {
}
}
+ /**
+ * A test TwillApplication to test setting log level to DEBUG.
+ */
+ public static final class LogLevelTestApplication implements TwillApplication {
+
+ @Override
+ public TwillSpecification configure() {
+ return TwillSpecification.Builder.with()
+ .setName(LogLevelTestApplication.class.getSimpleName())
+ .withRunnable()
+ .add(LogLevelTestRunnable.class.getSimpleName(), new LogLevelTestRunnable()).noLocalFiles()
+ .anyOrder()
+ .build();
+ }
+ }
+
@Test
public void testDebugLogLevel()throws Exception {
YarnTwillRunnerService runner = getTwillRunner();
runner.start();
- TwillController controller = runner.prepare(new LogLevelTestRunnable())
+ // Set log level to DEBUG
+ TwillController controller = runner.prepare(new LogLevelTestApplication())
.setLogLevel(LogEntry.Level.DEBUG)
.addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
.start();
+ // Lets wait until the service is running
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.onRunning(new Runnable() {
+ @Override
+ public void run() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ Assert.assertTrue(running.await(200, TimeUnit.SECONDS));
+
+ LogEntry.Level logLevel = waitForLogLevel(controller, LogLevelTestRunnable.class.getSimpleName(), 5L,
+ TimeUnit.SECONDS);
+
+ // Verify we got DEBUG log level.
+ Assert.assertEquals(LogEntry.Level.DEBUG, logLevel);
+
controller.terminate().get(120, TimeUnit.SECONDS);
+
+ // Sleep a bit for full cleanup
+ TimeUnit.SECONDS.sleep(2);
}
+ // Need helper method here to wait for getting resource report because {@link TwillController#getResourceReport()}
+ // could return null if the application has not fully started.
+ private LogEntry.Level waitForLogLevel(TwillController controller, String runnable, long timeout,
+ TimeUnit timeoutUnit) throws InterruptedException {
+
+ Stopwatch stopwatch = new Stopwatch();
+ stopwatch.start();
+ do {
+ ResourceReport report = controller.getResourceReport();
+ if (report == null || report.getRunnableResources(runnable) == null) {
+ continue;
+ }
+ for (TwillRunResources resources : report.getRunnableResources(runnable)) {
+ if (resources.getLogLevel() != null) {
+ return resources.getLogLevel();
+ }
+ }
+ TimeUnit.MILLISECONDS.sleep(100);
+ } while (stopwatch.elapsedTime(timeoutUnit) < timeout);
+
+ return null;
+ }
}