You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/09/03 20:30:22 UTC
svn commit: r1519787 [4/6] - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project: ./ hadoop-yarn/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apach...
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestLogDumper.java Tue Sep 3 18:30:05 2013
@@ -19,14 +19,30 @@
package org.apache.hadoop.yarn.logaggregation;
import static org.junit.Assert.assertTrue;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+
+import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Before;
import org.junit.Test;
public class TestLogDumper {
+ ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+
+ @Before
+ public void setUp() {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = new PrintStream(sysOutStream);
+ System.setOut(sysOut);
+ }
+
@Test
public void testFailResultCodes() throws Exception {
Configuration conf = new YarnConfiguration();
@@ -44,4 +60,30 @@ public class TestLogDumper {
"nonexistentnode:1234", "nobody");
assertTrue("Should return an error code", exitCode != 0);
}
+
+ @Test
+ public void testHelpMessage() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ LogDumper dumper = new LogDumper();
+ dumper.setConf(conf);
+
+ int exitCode = dumper.run(new String[]{});
+ assertTrue(exitCode == -1);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Retrieve logs for completed YARN applications.");
+ pw.println("usage: yarn logs -applicationId <application ID> [OPTIONS]");
+ pw.println();
+ pw.println("general options are:");
+ pw.println(" -appOwner <Application Owner> AppOwner (assumed to be current user if");
+ pw.println(" not specified)");
+ pw.println(" -containerId <Container ID> ContainerId (must be specified if node");
+ pw.println(" address is specified)");
+ pw.println(" -nodeAddress <Node Address> NodeAddress in the format nodename:port");
+ pw.println(" (must be specified if container id is");
+ pw.println(" specified)");
+ pw.close();
+ String appReportStr = baos.toString("UTF-8");
+ Assert.assertEquals(appReportStr, sysOutStream.toString());
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestCompositeService.java Tue Sep 3 18:30:05 2013
@@ -19,11 +19,14 @@
package org.apache.hadoop.yarn.util;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.BreakableService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@@ -314,6 +317,26 @@ public class TestCompositeService {
composite.init(new Configuration());
assertInState(STATE.INITED, child);
}
+
+ @Test (timeout = 1000)
+ public void testAddIfService() {
+ CompositeService testService = new CompositeService("TestService") {
+ Service service;
+ @Override
+ public void serviceInit(Configuration conf) {
+ Integer notAService = new Integer(0);
+ assertFalse("Added an integer as a service",
+ addIfService(notAService));
+
+ service = new AbstractService("Service") {};
+ assertTrue("Unable to add a service", addIfService(service));
+ }
+ };
+
+ testService.init(new Configuration());
+ assertEquals("Incorrect number of services",
+ 1, testService.getServices().size());
+ }
public static class CompositeServiceAddingAChild extends CompositeService{
Service child;
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java Tue Sep 3 18:30:05 2013
@@ -18,30 +18,47 @@
package org.apache.hadoop.yarn.webapp;
-import org.apache.commons.lang.ArrayUtils;
-import org.apache.hadoop.yarn.MockApps;
-import org.apache.hadoop.yarn.webapp.Controller;
-import org.apache.hadoop.yarn.webapp.WebApp;
-import org.apache.hadoop.yarn.webapp.WebApps;
-import org.apache.hadoop.yarn.webapp.view.HtmlPage;
-import org.apache.hadoop.yarn.webapp.view.JQueryUI;
-import org.apache.hadoop.yarn.webapp.view.TextPage;
-
-import com.google.inject.Inject;
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_TABLE;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.ext.ContextResolver;
+import javax.ws.rs.ext.Provider;
+import javax.xml.bind.JAXBContext;
-import static org.apache.hadoop.yarn.util.StringHelper.*;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI.*;
-
+import org.apache.commons.lang.ArrayUtils;
+import org.apache.hadoop.yarn.MockApps;
+import org.apache.hadoop.yarn.webapp.view.HtmlPage;
+import org.apache.hadoop.yarn.webapp.view.JQueryUI;
+import org.apache.hadoop.yarn.webapp.view.TextPage;
import org.junit.Test;
-import static org.junit.Assert.*;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+import com.sun.jersey.api.json.JSONConfiguration;
+import com.sun.jersey.api.json.JSONJAXBContext;
+
public class TestWebApp {
static final Logger LOG = LoggerFactory.getLogger(TestWebApp.class);
@@ -227,14 +244,19 @@ public class TestWebApp {
}
@Test public void testCustomRoutes() throws Exception {
- WebApp app = WebApps.$for("test", this).start(new WebApp() {
- @Override public void setup() {
- route("/:foo", FooController.class);
- route("/bar/foo", FooController.class, "bar");
- route("/foo/:foo", DefaultController.class);
- route("/foo/bar/:foo", DefaultController.class, "index");
- }
- });
+ WebApp app =
+ WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+ @Override
+ public void setup() {
+ bind(MyTestJAXBContextResolver.class);
+ bind(MyTestWebService.class);
+
+ route("/:foo", FooController.class);
+ route("/bar/foo", FooController.class, "bar");
+ route("/foo/:foo", DefaultController.class);
+ route("/foo/bar/:foo", DefaultController.class, "index");
+ }
+ });
String baseUrl = baseUrl(app);
try {
assertEquals("foo", getContent(baseUrl).trim());
@@ -245,6 +267,31 @@ public class TestWebApp {
assertEquals("default1", getContent(baseUrl +"test/foo/1").trim());
assertEquals("default2", getContent(baseUrl +"test/foo/bar/2").trim());
assertEquals(404, getResponseCode(baseUrl +"test/goo"));
+ assertEquals(200, getResponseCode(baseUrl +"ws/v1/test"));
+ assertTrue(getContent(baseUrl +"ws/v1/test").contains("myInfo"));
+ } finally {
+ app.stop();
+ }
+ }
+
+ // This is to test the GuiceFilter should only be applied to webAppContext,
+ // not to staticContext and logContext;
+ @Test public void testYARNWebAppContext() throws Exception {
+ // setting up the log context
+ System.setProperty("hadoop.log.dir", "/Not/Existing/dir");
+ WebApp app = WebApps.$for("test", this).start(new WebApp() {
+ @Override public void setup() {
+ route("/", FooController.class);
+ }
+ });
+ String baseUrl = baseUrl(app);
+ try {
+ // should not redirect to foo
+ assertFalse("foo".equals(getContent(baseUrl +"static").trim()));
+ // Not able to access a non-existing dir, should not redirect to foo.
+ assertEquals(404, getResponseCode(baseUrl +"logs"));
+ // should be able to redirect to foo.
+ assertEquals("foo", getContent(baseUrl).trim());
} finally {
app.stop();
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Tue Sep 3 18:30:05 2013
@@ -73,6 +73,8 @@
<goal>protoc</goal>
</goals>
<configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java Tue Sep 3 18:30:05 2013
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import com.google.protobuf.TextFormat;
+
public class NodeHealthStatusPBImpl extends NodeHealthStatus {
private NodeHealthStatusProto.Builder builder;
@@ -62,7 +64,7 @@ public class NodeHealthStatusPBImpl exte
@Override
public String toString() {
- return getProto().toString().replaceAll("\\n", ", ").replaceAll("\\s+", " ");
+ return TextFormat.shortDebugString(getProto());
}
private void mergeLocalToProto() {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Tue Sep 3 18:30:05 2013
@@ -165,6 +165,8 @@
<goal>protoc</goal>
</goals>
<configuration>
+ <protocVersion>${protobuf.version}</protocVersion>
+ <protocCommand>${protoc.path}</protocCommand>
<imports>
<param>${basedir}/../../../../hadoop-common-project/hadoop-common/src/main/proto</param>
<param>${basedir}/../../hadoop-yarn-api/src/main/proto</param>
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Tue Sep 3 18:30:05 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
/**
* Context interface for sharing information across components in the
@@ -61,4 +62,8 @@ public interface Context {
NodeHealthStatus getNodeHealthStatus();
ContainerManagementProtocol getContainerManager();
+
+ LocalDirsHandlerService getLocalDirsHandler();
+
+ ApplicationACLsManager getApplicationACLsManager();
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Tue Sep 3 18:30:05 2013
@@ -123,7 +123,8 @@ public class NodeManager extends Composi
protected NMContext createNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager) {
- return new NMContext(containerTokenSecretManager, nmTokenSecretManager);
+ return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
+ dirsHandler, aclsManager);
}
protected void doSecureLogin() throws IOException {
@@ -142,9 +143,6 @@ public class NodeManager extends Composi
NMTokenSecretManagerInNM nmTokenSecretManager =
new NMTokenSecretManagerInNM();
- this.context =
- createNMContext(containerTokenSecretManager, nmTokenSecretManager);
-
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -165,7 +163,9 @@ public class NodeManager extends Composi
addService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.getDiskHandler();
-
+ this.context = createNMContext(containerTokenSecretManager,
+ nmTokenSecretManager);
+
nodeStatusUpdater =
createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
@@ -319,14 +319,19 @@ public class NodeManager extends Composi
private final NMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInNM nmTokenSecretManager;
private ContainerManagementProtocol containerManager;
+ private final LocalDirsHandlerService dirsHandler;
+ private final ApplicationACLsManager aclsManager;
private WebServer webServer;
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
-
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
- NMTokenSecretManagerInNM nmTokenSecretManager) {
+ NMTokenSecretManagerInNM nmTokenSecretManager,
+ LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager) {
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
+ this.dirsHandler = dirsHandler;
+ this.aclsManager = aclsManager;
this.nodeHealthStatus.setIsNodeHealthy(true);
this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -386,6 +391,16 @@ public class NodeManager extends Composi
public void setNodeId(NodeId nodeId) {
this.nodeId = nodeId;
}
+
+ @Override
+ public LocalDirsHandlerService getLocalDirsHandler() {
+ return dirsHandler;
+ }
+
+ @Override
+ public ApplicationACLsManager getApplicationACLsManager() {
+ return aclsManager;
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Tue Sep 3 18:30:05 2013
@@ -85,7 +85,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
-import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -193,12 +192,6 @@ public class ContainerManagerImpl extend
super.serviceInit(conf);
}
- private void addIfService(Object object) {
- if (object instanceof Service) {
- addService((Service) object);
- }
- }
-
protected LogHandler createLogHandler(Configuration conf, Context context,
DeletionService deletionService) {
if (conf.getBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED,
@@ -475,8 +468,7 @@ public class ContainerManagerImpl extend
// Create the application
Application application =
- new ApplicationImpl(dispatcher, this.aclsManager, user, applicationID,
- credentials, context);
+ new ApplicationImpl(dispatcher, user, applicationID, credentials, context);
if (null == context.getApplications().putIfAbsent(applicationID,
application)) {
LOG.info("Creating a new application reference for app " + applicationID);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Tue Sep 3 18:30:05 2013
@@ -73,14 +73,13 @@ public class ApplicationImpl implements
Map<ContainerId, Container> containers =
new HashMap<ContainerId, Container>();
- public ApplicationImpl(Dispatcher dispatcher,
- ApplicationACLsManager aclsManager, String user, ApplicationId appId,
+ public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId,
Credentials credentials, Context context) {
this.dispatcher = dispatcher;
this.user = user;
this.appId = appId;
this.credentials = credentials;
- this.aclsManager = aclsManager;
+ this.aclsManager = context.getApplicationACLsManager();
this.context = context;
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Tue Sep 3 18:30:05 2013
@@ -525,7 +525,8 @@ public class ContainerLaunch implements
@Override
public void env(String key, String value) {
- line("@set ", key, "=", value);
+ line("@set ", key, "=", value,
+ "\nif %errorlevel% neq 0 exit /b %errorlevel%");
}
@Override
@@ -588,20 +589,18 @@ public class ContainerLaunch implements
environment.put(Environment.LOG_DIRS.name(),
StringUtils.join(",", containerLogDirs));
- putEnvIfNotNull(environment, Environment.USER.name(), container.getUser());
+ environment.put(Environment.USER.name(), container.getUser());
- putEnvIfNotNull(environment,
- Environment.LOGNAME.name(),container.getUser());
-
- putEnvIfNotNull(environment,
- Environment.HOME.name(),
+ environment.put(Environment.LOGNAME.name(), container.getUser());
+
+ environment.put(Environment.HOME.name(),
conf.get(
YarnConfiguration.NM_USER_HOME_DIR,
YarnConfiguration.DEFAULT_NM_USER_HOME_DIR
)
);
- putEnvIfNotNull(environment, Environment.PWD.name(), pwd.toString());
+ environment.put(Environment.PWD.name(), pwd.toString());
putEnvIfNotNull(environment,
Environment.HADOOP_CONF_DIR.name(),
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java Tue Sep 3 18:30:05 2013
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -149,7 +150,8 @@ public class ContainersLauncher extends
dispatcher.getEventHandler().handle(
new ContainerExitEvent(containerId,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
- ExitCode.TERMINATED.getExitCode(),
+ Shell.WINDOWS ? ExitCode.FORCE_KILLED.getExitCode() :
+ ExitCode.TERMINATED.getExitCode(),
"Container terminated before launch."));
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Tue Sep 3 18:30:05 2013
@@ -28,36 +28,21 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
-import java.util.EnumSet;
import java.util.List;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.SecureIOUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.PRE;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
-import org.mortbay.log.Log;
import com.google.inject.Inject;
@@ -90,19 +75,11 @@ public class ContainerLogsPage extends N
public static class ContainersLogsBlock extends HtmlBlock implements
YarnWebParams {
- private final Configuration conf;
private final Context nmContext;
- private final ApplicationACLsManager aclsManager;
- private final LocalDirsHandlerService dirsHandler;
@Inject
- public ContainersLogsBlock(Configuration conf, Context context,
- ApplicationACLsManager aclsManager,
- LocalDirsHandlerService dirsHandler) {
- this.conf = conf;
+ public ContainersLogsBlock(Context context) {
this.nmContext = context;
- this.aclsManager = aclsManager;
- this.dirsHandler = dirsHandler;
}
@Override
@@ -114,229 +91,123 @@ public class ContainerLogsPage extends N
" server. Log Server url may not be configured");
//Intentional fallthrough.
}
-
+
ContainerId containerId;
try {
containerId = ConverterUtils.toContainerId($(CONTAINER_ID));
- } catch (IllegalArgumentException e) {
- html.h1("Invalid containerId " + $(CONTAINER_ID));
+ } catch (IllegalArgumentException ex) {
+ html.h1("Invalid container ID: " + $(CONTAINER_ID));
return;
}
- ApplicationId applicationId = containerId.getApplicationAttemptId()
- .getApplicationId();
- Application application = this.nmContext.getApplications().get(
- applicationId);
- Container container = this.nmContext.getContainers().get(containerId);
-
- if (application == null) {
- html.h1(
- "Unknown container. Container either has not started or "
- + "has already completed or "
- + "doesn't belong to this node at all.");
- return;
- }
- if (container == null) {
- // Container may have alerady completed, but logs not aggregated yet.
- printLogs(html, containerId, applicationId, application);
- return;
- }
-
- if (EnumSet.of(ContainerState.NEW, ContainerState.LOCALIZING,
- ContainerState.LOCALIZED).contains(container.getContainerState())) {
- html.h1("Container is not yet running. Current state is "
- + container.getContainerState());
- return;
- }
-
- if (container.getContainerState() == ContainerState.LOCALIZATION_FAILED) {
- html.h1("Container wasn't started. Localization failed.");
- return;
- }
-
- if (EnumSet.of(ContainerState.RUNNING,
- ContainerState.EXITED_WITH_FAILURE,
- ContainerState.EXITED_WITH_SUCCESS).contains(
- container.getContainerState())) {
- printLogs(html, containerId, applicationId, application);
- return;
- }
- if (EnumSet.of(ContainerState.KILLING,
- ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
- ContainerState.CONTAINER_RESOURCES_CLEANINGUP).contains(
- container.getContainerState())) {
- //Container may have generated some logs before being killed.
- printLogs(html, containerId, applicationId, application);
- return;
- }
- if (container.getContainerState().equals(ContainerState.DONE)) {
- // Prev state unknown. Logs may be available.
- printLogs(html, containerId, applicationId, application);
- return;
- } else {
- html.h1("Container is no longer running...");
- return;
+ try {
+ if ($(CONTAINER_LOG_TYPE).isEmpty()) {
+ List<File> logFiles = ContainerLogsUtils.getContainerLogDirs(containerId,
+ request().getRemoteUser(), nmContext);
+ printLogFileDirectory(html, logFiles);
+ } else {
+ File logFile = ContainerLogsUtils.getContainerLogFile(containerId,
+ $(CONTAINER_LOG_TYPE), request().getRemoteUser(), nmContext);
+ printLogFile(html, logFile);
+ }
+ } catch (YarnException ex) {
+ html.h1(ex.getMessage());
+ } catch (NotFoundException ex) {
+ html.h1(ex.getMessage());
}
}
-
- private void printLogs(Block html, ContainerId containerId,
- ApplicationId applicationId, Application application) {
- // Check for the authorization.
- String remoteUser = request().getRemoteUser();
- UserGroupInformation callerUGI = null;
-
- if (remoteUser != null) {
- callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
- }
- if (callerUGI != null
- && !this.aclsManager.checkAccess(callerUGI,
- ApplicationAccessType.VIEW_APP, application.getUser(),
- applicationId)) {
- html.h1(
- "User [" + remoteUser
- + "] is not authorized to view the logs for application "
- + applicationId);
+
+ private void printLogFile(Block html, File logFile) {
+ long start =
+ $("start").isEmpty() ? -4 * 1024 : Long.parseLong($("start"));
+ start = start < 0 ? logFile.length() + start : start;
+ start = start < 0 ? 0 : start;
+ long end =
+ $("end").isEmpty() ? logFile.length() : Long.parseLong($("end"));
+ end = end < 0 ? logFile.length() + end : end;
+ end = end < 0 ? logFile.length() : end;
+ if (start > end) {
+ html.h1("Invalid start and end values. Start: [" + start + "]"
+ + ", end[" + end + "]");
return;
- }
+ } else {
+ FileInputStream logByteStream = null;
- if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
- File logFile = null;
try {
- URI logPathURI = new URI(this.dirsHandler.getLogPathToRead(
- ContainerLaunch.getRelativeContainerLogDir(
- applicationId.toString(), containerId.toString())
- + Path.SEPARATOR + $(CONTAINER_LOG_TYPE)).toString());
- logFile = new File(logPathURI.getPath());
- } catch (URISyntaxException e) {
- html.h1("Cannot find this log on the local disk.");
- return;
- } catch (Exception e) {
- html.h1("Cannot find this log on the local disk.");
+ logByteStream = ContainerLogsUtils.openLogFileForRead($(CONTAINER_ID),
+ logFile, nmContext);
+ } catch (IOException ex) {
+ html.h1(ex.getMessage());
return;
}
- long start =
- $("start").isEmpty() ? -4 * 1024 : Long.parseLong($("start"));
- start = start < 0 ? logFile.length() + start : start;
- start = start < 0 ? 0 : start;
- long end =
- $("end").isEmpty() ? logFile.length() : Long.parseLong($("end"));
- end = end < 0 ? logFile.length() + end : end;
- end = end < 0 ? logFile.length() : end;
- if (start > end) {
- html.h1("Invalid start and end values. Start: [" + start + "]"
- + ", end[" + end + "]");
- return;
- } else {
- FileInputStream logByteStream = null;
-
- try {
- logByteStream =
- SecureIOUtils.openForRead(logFile, application.getUser(), null);
- } catch (IOException e) {
- LOG.error(
- "Exception reading log file " + logFile.getAbsolutePath(), e);
- if (e.getMessage().contains(
- "did not match expected owner '" + application.getUser()
- + "'")) {
- html.h1("Exception reading log file. Application submitted by '"
- + application.getUser()
- + "' doesn't own requested log file : "
- + logFile.getName());
- } else {
- html.h1("Exception reading log file. It might be because log "
- + "file was aggregated : " + logFile.getName());
- }
- return;
+
+ try {
+ long toRead = end - start;
+ if (toRead < logFile.length()) {
+ html.p()._("Showing " + toRead + " bytes. Click ")
+ .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
+ logFile.getName(), "?start=0"), "here").
+ _(" for full log")._();
}
- try {
- long toRead = end - start;
- if (toRead < logFile.length()) {
- html.p()._("Showing " + toRead + " bytes. Click ")
- .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
- logFile.getName(), "?start=0"), "here").
- _(" for full log")._();
- }
- // TODO Fix findBugs close warning along with IOUtils change
- IOUtils.skipFully(logByteStream, start);
- InputStreamReader reader = new InputStreamReader(logByteStream);
- int bufferSize = 65536;
- char[] cbuf = new char[bufferSize];
-
- int len = 0;
- int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
- PRE<Hamlet> pre = html.pre();
-
- while ((len = reader.read(cbuf, 0, currentToRead)) > 0
- && toRead > 0) {
- pre._(new String(cbuf, 0, len));
- toRead = toRead - len;
- currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
- }
+ IOUtils.skipFully(logByteStream, start);
+ InputStreamReader reader = new InputStreamReader(logByteStream);
+ int bufferSize = 65536;
+ char[] cbuf = new char[bufferSize];
+
+ int len = 0;
+ int currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
+ PRE<Hamlet> pre = html.pre();
+
+ while ((len = reader.read(cbuf, 0, currentToRead)) > 0
+ && toRead > 0) {
+ pre._(new String(cbuf, 0, len));
+ toRead = toRead - len;
+ currentToRead = toRead > bufferSize ? bufferSize : (int) toRead;
+ }
- pre._();
- reader.close();
+ pre._();
+ reader.close();
- } catch (IOException e) {
- LOG.error(
+ } catch (IOException e) {
+ LOG.error(
"Exception reading log file " + logFile.getAbsolutePath(), e);
- html.h1("Exception reading log file. It might be because log "
+ html.h1("Exception reading log file. It might be because log "
+ "file was aggregated : " + logFile.getName());
- } finally {
- if (logByteStream != null) {
- try {
- logByteStream.close();
- } catch (IOException e) {
- // Ignore
- }
+ } finally {
+ if (logByteStream != null) {
+ try {
+ logByteStream.close();
+ } catch (IOException e) {
+ // Ignore
}
}
}
- } else {
- // Print out log types in lexical order
- List<File> containerLogsDirs = getContainerLogDirs(containerId,
- dirsHandler);
- Collections.sort(containerLogsDirs);
- boolean foundLogFile = false;
- for (File containerLogsDir : containerLogsDirs) {
- File[] logFiles = containerLogsDir.listFiles();
- if (logFiles != null) {
- Arrays.sort(logFiles);
- for (File logFile : logFiles) {
- foundLogFile = true;
- html.p()
- .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
- logFile.getName(), "?start=-4096"),
- logFile.getName() + " : Total file length is "
- + logFile.length() + " bytes.")._();
- }
- }
- }
- if (!foundLogFile) {
- html.h1("No logs available for container " + containerId.toString());
- return;
- }
}
- return;
}
-
- static List<File> getContainerLogDirs(ContainerId containerId,
- LocalDirsHandlerService dirsHandler) {
- List<String> logDirs = dirsHandler.getLogDirs();
- List<File> containerLogDirs = new ArrayList<File>(logDirs.size());
- for (String logDir : logDirs) {
- try {
- logDir = new URI(logDir).getPath();
- } catch (URISyntaxException e) {
- Log.warn(e.getMessage());
+
+ private void printLogFileDirectory(Block html, List<File> containerLogsDirs) {
+ // Print out log types in lexical order
+ Collections.sort(containerLogsDirs);
+ boolean foundLogFile = false;
+ for (File containerLogsDir : containerLogsDirs) {
+ File[] logFiles = containerLogsDir.listFiles();
+ if (logFiles != null) {
+ Arrays.sort(logFiles);
+ for (File logFile : logFiles) {
+ foundLogFile = true;
+ html.p()
+ .a(url("containerlogs", $(CONTAINER_ID), $(APP_OWNER),
+ logFile.getName(), "?start=-4096"),
+ logFile.getName() + " : Total file length is "
+ + logFile.length() + " bytes.")._();
+ }
}
- String appIdStr = ConverterUtils.toString(containerId
- .getApplicationAttemptId().getApplicationId());
- File appLogDir = new File(logDir, appIdStr);
- String containerIdStr = ConverterUtils.toString(containerId);
- containerLogDirs.add(new File(appLogDir, containerIdStr));
}
- return containerLogDirs;
+ if (!foundLogFile) {
+ html.h1("No logs available for container " + $(CONTAINER_ID));
+ return;
+ }
}
}
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java Tue Sep 3 18:30:05 2013
@@ -17,19 +17,31 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.OutputStream;
import java.util.Map.Entry;
+import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -60,6 +72,9 @@ public class NMWebServices {
.getRecordFactory(null);
private @javax.ws.rs.core.Context
+ HttpServletRequest request;
+
+ private @javax.ws.rs.core.Context
HttpServletResponse response;
@javax.ws.rs.core.Context
@@ -179,5 +194,66 @@ public class NMWebServices {
.toString(), webapp.name());
}
-
+
+ /**
+ * Returns the contents of a container's log file in plain text.
+ *
+ * Only works for containers that are still in the NodeManager's memory, so
+ * logs are no longer available after the corresponding application is no
+ * longer running.
+ *
+ * @param containerIdStr
+ * The container ID
+ * @param filename
+ * The name of the log file
+ * @return
+ * The contents of the container's log file
+ */
+ @GET
+ @Path("/containerlogs/{containerid}/{filename}")
+ @Produces({ MediaType.TEXT_PLAIN })
+ @Public
+ @Unstable
+ public Response getLogs(@PathParam("containerid") String containerIdStr,
+ @PathParam("filename") String filename) {
+ ContainerId containerId;
+ try {
+ containerId = ConverterUtils.toContainerId(containerIdStr);
+ } catch (IllegalArgumentException ex) {
+ return Response.status(Status.BAD_REQUEST).build();
+ }
+
+ File logFile = null;
+ try {
+ logFile = ContainerLogsUtils.getContainerLogFile(
+ containerId, filename, request.getRemoteUser(), nmContext);
+ } catch (NotFoundException ex) {
+ return Response.status(Status.NOT_FOUND).entity(ex.getMessage()).build();
+ } catch (YarnException ex) {
+ return Response.serverError().entity(ex.getMessage()).build();
+ }
+
+ try {
+ final FileInputStream fis = ContainerLogsUtils.openLogFileForRead(
+ containerIdStr, logFile, nmContext);
+
+ StreamingOutput stream = new StreamingOutput() {
+ @Override
+ public void write(OutputStream os) throws IOException,
+ WebApplicationException {
+ int bufferSize = 65536;
+ byte[] buf = new byte[bufferSize];
+ int len;
+ while ((len = fis.read(buf, 0, bufferSize)) > 0) {
+ os.write(buf, 0, len);
+ }
+ os.flush();
+ }
+ };
+
+ return Response.ok(stream).build();
+ } catch (IOException ex) {
+ return Response.serverError().entity(ex.getMessage()).build();
+ }
+ }
}
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Sep 3 18:30:05 2013
@@ -59,8 +59,15 @@ public class WebServer extends AbstractS
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
this.webApp =
- WebApps.$for("node", Context.class, this.nmContext, "ws")
- .at(bindAddress).with(getConfig()).start(this.nmWebApp);
+ WebApps
+ .$for("node", Context.class, this.nmContext, "ws")
+ .at(bindAddress)
+ .with(getConfig())
+ .withHttpSpnegoPrincipalKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
+ .withHttpSpnegoKeytabKey(
+ YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
+ .start(this.nmWebApp);
this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java Tue Sep 3 18:30:05 2013
@@ -79,7 +79,7 @@ public class TestEventFlow {
YarnConfiguration conf = new YarnConfiguration();
Context context = new NMContext(new NMContainerTokenSecretManager(conf),
- new NMTokenSecretManagerInNM()) {
+ new NMTokenSecretManagerInNM(), null, null) {
@Override
public int getHttpPort() {
return 1234;
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Tue Sep 3 18:30:05 2013
@@ -957,15 +957,14 @@ public class TestNodeStatusUpdater {
@Test (timeout = 150000)
public void testNMConnectionToRM() throws Exception {
final long delta = 50000;
- final long connectionWaitSecs = 5;
- final long connectionRetryIntervalSecs = 1;
+ final long connectionWaitMs = 5000;
+ final long connectionRetryIntervalMs = 1000;
//Waiting for rmStartIntervalMS, RM will be started
final long rmStartIntervalMS = 2*1000;
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS,
- connectionWaitSecs);
- conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
- connectionRetryIntervalSecs);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
+ connectionWaitMs);
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ connectionRetryIntervalMs);
//Test NM try to connect to RM Several times, but finally fail
NodeManagerWithCustomNodeStatusUpdater nmWithUpdater;
@@ -987,15 +986,15 @@ public class TestNodeStatusUpdater {
} catch(Exception e) {
long t = System.currentTimeMillis();
long duration = t - waitStartTime;
- boolean waitTimeValid = (duration >= connectionWaitSecs * 1000)
- && (duration < (connectionWaitSecs * 1000 + delta));
+ boolean waitTimeValid = (duration >= connectionWaitMs)
+ && (duration < (connectionWaitMs + delta));
if(!waitTimeValid) {
//either the exception was too early, or it had a different cause.
//reject with the inner stack trace
throw new Exception("NM should have tried re-connecting to RM during " +
- "period of at least " + connectionWaitSecs + " seconds, but " +
- "stopped retrying within " + (connectionWaitSecs + delta/1000) +
- " seconds: " + e, e);
+ "period of at least " + connectionWaitMs + " ms, but " +
+ "stopped retrying within " + (connectionWaitMs + delta) +
+ " ms: " + e, e);
}
}
@@ -1149,14 +1148,14 @@ public class TestNodeStatusUpdater {
@Test(timeout = 200000)
public void testNodeStatusUpdaterRetryAndNMShutdown()
throws Exception {
- final long connectionWaitSecs = 1;
- final long connectionRetryIntervalSecs = 1;
+ final long connectionWaitSecs = 1000;
+ final long connectionRetryIntervalMs = 1000;
YarnConfiguration conf = createNMConfig();
- conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_SECS,
+ conf.setLong(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS,
connectionWaitSecs);
conf.setLong(YarnConfiguration
- .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
- connectionRetryIntervalSecs);
+ .RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS,
+ connectionRetryIntervalMs);
conf.setLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS, 5000);
CyclicBarrier syncBarrier = new CyclicBarrier(2);
nm = new MyNodeManager2(syncBarrier, conf);
@@ -1186,7 +1185,7 @@ public class TestNodeStatusUpdater {
public MyNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager) {
- super(containerTokenSecretManager, nmTokenSecretManager);
+ super(containerTokenSecretManager, nmTokenSecretManager, null, null);
}
@Override
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java Tue Sep 3 18:30:05 2013
@@ -100,7 +100,7 @@ public abstract class BaseContainerManag
protected static final int HTTP_PORT = 5412;
protected Configuration conf = new YarnConfiguration();
protected Context context = new NMContext(new NMContainerTokenSecretManager(
- conf), new NMTokenSecretManagerInNM()) {
+ conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf)) {
public int getHttpPort() {
return HTTP_PORT;
};
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java Tue Sep 3 18:30:05 2013
@@ -490,6 +490,8 @@ public class TestApplication {
when(context.getContainerTokenSecretManager()).thenReturn(
new NMContainerTokenSecretManager(conf));
+ when(context.getApplicationACLsManager()).thenReturn(
+ new ApplicationACLsManager(conf));
// Setting master key
MasterKey masterKey = new MasterKeyPBImpl();
@@ -501,8 +503,7 @@ public class TestApplication {
this.user = user;
this.appId = BuilderUtils.newApplicationId(timestamp, id);
- app = new ApplicationImpl(dispatcher, new ApplicationACLsManager(
- new Configuration()), this.user, appId, null, context);
+ app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
containers = new ArrayList<Container>();
for (int i = 0; i < numContainers; i++) {
Container container = createMockedContainer(this.appId, i);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java Tue Sep 3 18:30:05 2013
@@ -240,15 +240,10 @@ public class TestContainerLaunch extends
File shellFile = null;
try {
shellFile = Shell.appendScriptExtension(tmpDir, "hello");
- String timeoutCommand = Shell.WINDOWS ? "@echo \"hello\"" :
- "echo \"hello\"";
- PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
- FileUtil.setExecutable(shellFile, true);
- writer.println(timeoutCommand);
- writer.close();
Map<Path, List<String>> resources =
new HashMap<Path, List<String>>();
FileOutputStream fos = new FileOutputStream(shellFile);
+ FileUtil.setExecutable(shellFile, true);
Map<String, String> env = new HashMap<String, String>();
// invalid env
@@ -270,7 +265,9 @@ public class TestContainerLaunch extends
} catch(ExitCodeException e){
diagnostics = e.getMessage();
}
- Assert.assertTrue(diagnostics.contains("command not found"));
+ Assert.assertTrue(diagnostics.contains(Shell.WINDOWS ?
+ "is not recognized as an internal or external command" :
+ "command not found"));
Assert.assertTrue(shexc.getExitCode() != 0);
}
finally {
@@ -289,15 +286,16 @@ public class TestContainerLaunch extends
try {
shellFile = Shell.appendScriptExtension(tmpDir, "hello");
// echo "hello" to stdout and "error" to stderr and exit code with 2;
- String command = Shell.WINDOWS ? "@echo \"hello\"; @echo \"error\" 1>&2; exit 2;" :
- "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
+ String command = Shell.WINDOWS ?
+ "@echo \"hello\" & @echo \"error\" 1>&2 & exit /b 2" :
+ "echo \"hello\"; echo \"error\" 1>&2; exit 2;";
PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
FileUtil.setExecutable(shellFile, true);
writer.println(command);
writer.close();
Map<Path, List<String>> resources =
new HashMap<Path, List<String>>();
- FileOutputStream fos = new FileOutputStream(shellFile);
+ FileOutputStream fos = new FileOutputStream(shellFile, true);
Map<String, String> env = new HashMap<String, String>();
List<String> commands = new ArrayList<String>();
@@ -346,7 +344,6 @@ public class TestContainerLaunch extends
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
- int port = 12345;
ContainerId cId = ContainerId.newInstance(appAttemptId, 0);
Map<String, String> userSetEnv = new HashMap<String, String>();
userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
@@ -354,6 +351,11 @@ public class TestContainerLaunch extends
userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
+ userSetEnv.put(Environment.USER.key(), "user_set_" +
+ Environment.USER.key());
+ userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
+ userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
+ userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
containerLaunchContext.setEnvironment(userSetEnv);
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
@@ -371,6 +373,14 @@ public class TestContainerLaunch extends
+ processStartFile);
fileWriter.println("@echo " + Environment.LOCAL_DIRS.$() + ">> "
+ processStartFile);
+ fileWriter.println("@echo " + Environment.USER.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.LOGNAME.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.PWD.$() + ">> "
+ + processStartFile);
+ fileWriter.println("@echo " + Environment.HOME.$() + ">> "
+ + processStartFile);
fileWriter.println("@echo " + cId + ">> " + processStartFile);
fileWriter.println("@ping -n 100 127.0.0.1 >nul");
} else {
@@ -385,6 +395,14 @@ public class TestContainerLaunch extends
+ processStartFile);
fileWriter.write("\necho $" + Environment.LOCAL_DIRS.name() + " >> "
+ processStartFile);
+ fileWriter.write("\necho $" + Environment.USER.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.LOGNAME.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.PWD.name() + " >> "
+ + processStartFile);
+ fileWriter.write("\necho $" + Environment.HOME.name() + " >> "
+ + processStartFile);
fileWriter.write("\necho $$ >> " + processStartFile);
fileWriter.write("\nexec sleep 100");
}
@@ -452,6 +470,22 @@ public class TestContainerLaunch extends
reader.readLine());
Assert.assertEquals(String.valueOf(HTTP_PORT), reader.readLine());
Assert.assertEquals(StringUtils.join(",", appDirs), reader.readLine());
+ Assert.assertEquals(user, reader.readLine());
+ Assert.assertEquals(user, reader.readLine());
+ String obtainedPWD = reader.readLine();
+ boolean found = false;
+ for (Path localDir : appDirs) {
+ if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found);
+ Assert.assertEquals(
+ conf.get(
+ YarnConfiguration.NM_USER_HOME_DIR,
+ YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
+ reader.readLine());
Assert.assertEquals(cId.toString(), containerLaunchContext
.getEnvironment().get(Environment.CONTAINER_ID.name()));
@@ -465,6 +499,26 @@ public class TestContainerLaunch extends
.getEnvironment().get(Environment.LOCAL_DIRS.name()));
Assert.assertEquals(StringUtils.join(",", containerLogDirs),
containerLaunchContext.getEnvironment().get(Environment.LOG_DIRS.name()));
+ Assert.assertEquals(user, containerLaunchContext.getEnvironment()
+ .get(Environment.USER.name()));
+ Assert.assertEquals(user, containerLaunchContext.getEnvironment()
+ .get(Environment.LOGNAME.name()));
+ found = false;
+ obtainedPWD =
+ containerLaunchContext.getEnvironment().get(Environment.PWD.name());
+ for (Path localDir : appDirs) {
+ if (new Path(localDir, cId.toString()).toString().equals(obtainedPWD)) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue("Wrong local-dir found : " + obtainedPWD, found);
+ Assert.assertEquals(
+ conf.get(
+ YarnConfiguration.NM_USER_HOME_DIR,
+ YarnConfiguration.DEFAULT_NM_USER_HOME_DIR),
+ containerLaunchContext.getEnvironment()
+ .get(Environment.HOME.name()));
// Get the pid of the process
String pid = reader.readLine().trim();
@@ -538,7 +592,6 @@ public class TestContainerLaunch extends
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
- int port = 12345;
// upload the script file so that the container can run it
URL resource_alpha =
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java Tue Sep 3 18:30:05 2013
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -50,7 +51,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.webapp.ContainerLogsPage.ContainersLogsBlock;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.test.WebAppTests;
@@ -63,7 +63,7 @@ import com.google.inject.Module;
public class TestContainerLogsPage {
@Test(timeout=30000)
- public void testContainerLogDirs() throws IOException {
+ public void testContainerLogDirs() throws IOException, YarnException {
File absLogDir = new File("target",
TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
String logdirwithFile = absLogDir.toURI().toString();
@@ -86,7 +86,7 @@ public class TestContainerLogsPage {
ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
appAttemptId, 0);
List<File> files = null;
- files = ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(
+ files = ContainerLogsUtils.getContainerLogDirs(
container1, dirsHandler);
Assert.assertTrue(!(files.get(0).toString().contains("file:")));
}
@@ -146,8 +146,6 @@ public class TestContainerLogsPage {
out.write("Log file Content".getBytes());
out.close();
- ApplicationACLsManager aclsManager = mock(ApplicationACLsManager.class);
-
Context context = mock(Context.class);
ConcurrentMap<ApplicationId, Application> appMap =
new ConcurrentHashMap<ApplicationId, Application>();
@@ -157,7 +155,7 @@ public class TestContainerLogsPage {
new ConcurrentHashMap<ContainerId, Container>());
ContainersLogsBlock cLogsBlock =
- new ContainersLogsBlock(conf, context, aclsManager, dirsHandler);
+ new ContainersLogsBlock(context);
Map<String, String> params = new HashMap<String, String>();
params.put(YarnWebParams.CONTAINER_ID, container1.toString());
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Tue Sep 3 18:30:05 2013
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -76,7 +77,7 @@ public class TestNMWebServer {
}
private int startNMWebAppServer(String webAddr) {
- Context nmContext = new NodeManager.NMContext(null, null);
+ Context nmContext = new NodeManager.NMContext(null, null, null, null);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -133,8 +134,8 @@ public class TestNMWebServer {
}
@Test
- public void testNMWebApp() throws IOException {
- Context nmContext = new NodeManager.NMContext(null, null);
+ public void testNMWebApp() throws IOException, YarnException {
+ Context nmContext = new NodeManager.NMContext(null, null, null, null);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -219,10 +220,10 @@ public class TestNMWebServer {
private void writeContainerLogs(Context nmContext,
ContainerId containerId, LocalDirsHandlerService dirsHandler)
- throws IOException {
+ throws IOException, YarnException {
// ContainerLogDir should be created
File containerLogDir =
- ContainerLogsPage.ContainersLogsBlock.getContainerLogDirs(containerId,
+ ContainerLogsUtils.getContainerLogDirs(containerId,
dirsHandler).get(0);
containerLogDir.mkdirs();
for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java Tue Sep 3 18:30:05 2013
@@ -23,24 +23,38 @@ import static org.junit.Assert.assertTru
import static org.junit.Assert.fail;
import java.io.File;
+import java.io.IOException;
+import java.io.PrintWriter;
import java.io.StringReader;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
+import junit.framework.Assert;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer.NMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebApp;
@@ -86,7 +100,14 @@ public class TestNMWebServices extends J
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext(null, null);
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
+ NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
+ healthChecker.init(conf);
+ dirsHandler = healthChecker.getDiskHandler();
+ aclsManager = new ApplicationACLsManager(conf);
+ nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager);
NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
((NodeManager.NMContext)nmContext).setNodeId(nodeId);
resourceView = new ResourceView() {
@@ -110,13 +131,6 @@ public class TestNMWebServices extends J
return true;
}
};
- Configuration conf = new Configuration();
- conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
- conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
- NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
- healthChecker.init(conf);
- dirsHandler = healthChecker.getDiskHandler();
- aclsManager = new ApplicationACLsManager(conf);
nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
bind(JAXBContextResolver.class);
bind(NMWebServices.class);
@@ -292,6 +306,53 @@ public class TestNMWebServices extends J
assertEquals("incorrect number of elements", 1, nodes.getLength());
verifyNodesXML(nodes);
}
+
+ @Test
+ public void testContainerLogs() throws IOException {
+ WebResource r = resource();
+ final ContainerId containerId = BuilderUtils.newContainerId(0, 0, 0, 0);
+ final String containerIdStr = BuilderUtils.newContainerId(0, 0, 0, 0)
+ .toString();
+ final ApplicationAttemptId appAttemptId = containerId.getApplicationAttemptId();
+ final ApplicationId appId = appAttemptId.getApplicationId();
+ final String appIdStr = appId.toString();
+ final String filename = "logfile1";
+ final String logMessage = "log message\n";
+ nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
+ appId, null, nmContext));
+
+ MockContainer container = new MockContainer(appAttemptId,
+ new AsyncDispatcher(), new Configuration(), "user", appId, 1);
+ container.setState(ContainerState.RUNNING);
+ nmContext.getContainers().put(containerId, container);
+
+ // write out log file
+ Path path = dirsHandler.getLogPathForWrite(
+ ContainerLaunch.getRelativeContainerLogDir(
+ appIdStr, containerIdStr) + "/" + filename, false);
+
+ File logFile = new File(path.toUri().getPath());
+ logFile.deleteOnExit();
+ assertTrue("Failed to create log dir", logFile.getParentFile().mkdirs());
+ PrintWriter pw = new PrintWriter(logFile);
+ pw.print(logMessage);
+ pw.close();
+
+ // ask for it
+ ClientResponse response = r.path("ws").path("v1").path("node")
+ .path("containerlogs").path(containerIdStr).path(filename)
+ .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
+ String responseText = response.getEntity(String.class);
+ assertEquals(logMessage, responseText);
+
+ // ask for file that doesn't exist
+ response = r.path("ws").path("v1").path("node")
+ .path("containerlogs").path(containerIdStr).path("uhhh")
+ .accept(MediaType.TEXT_PLAIN).get(ClientResponse.class);
+ Assert.assertEquals(Status.NOT_FOUND.getStatusCode(), response.getStatus());
+ responseText = response.getEntity(String.class);
+ assertTrue(responseText.contains("Cannot find this log on the local disk."));
+ }
public void verifyNodesXML(NodeList nodes) throws JSONException, Exception {
for (int i = 0; i < nodes.getLength(); i++) {
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Tue Sep 3 18:30:05 2013
@@ -93,7 +93,13 @@ public class TestNMWebServicesApps exten
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext(null, null);
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
+ NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
+ healthChecker.init(conf);
+ dirsHandler = healthChecker.getDiskHandler();
+ aclsManager = new ApplicationACLsManager(conf);
+ nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager);
NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
((NodeManager.NMContext)nmContext).setNodeId(nodeId);
resourceView = new ResourceView() {
@@ -119,12 +125,6 @@ public class TestNMWebServicesApps exten
return true;
}
};
- conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
- conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
- NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
- healthChecker.init(conf);
- dirsHandler = healthChecker.getDiskHandler();
- aclsManager = new ApplicationACLsManager(conf);
nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
bind(JAXBContextResolver.class);
bind(NMWebServices.class);
Modified: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1519787&r1=1519786&r2=1519787&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Tue Sep 3 18:30:05 2013
@@ -93,15 +93,6 @@ public class TestNMWebServicesContainers
private Injector injector = Guice.createInjector(new ServletModule() {
@Override
protected void configureServlets() {
- nmContext = new NodeManager.NMContext(null, null) {
- public NodeId getNodeId() {
- return NodeId.newInstance("testhost.foo.com", 8042);
- };
-
- public int getHttpPort() {
- return 1234;
- };
- };
resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -131,6 +122,15 @@ public class TestNMWebServicesContainers
healthChecker.init(conf);
dirsHandler = healthChecker.getDiskHandler();
aclsManager = new ApplicationACLsManager(conf);
+ nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager) {
+ public NodeId getNodeId() {
+ return NodeId.newInstance("testhost.foo.com", 8042);
+ };
+
+ public int getHttpPort() {
+ return 1234;
+ };
+ };
nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
bind(JAXBContextResolver.class);
bind(NMWebServices.class);