You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by vi...@apache.org on 2011/04/26 06:38:19 UTC
svn commit: r1096654 - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/
mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event...
Author: vinodkv
Date: Tue Apr 26 04:38:18 2011
New Revision: 1096654
URL: http://svn.apache.org/viewvc?rev=1096654&view=rev
Log:
Adding Containers' logs' view to NodeManager UI and link it from AM UI.
Add some of the last few major commits to CHANGES.txt MR-279 section.
Added:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMLauncher.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Tue Apr 26 04:38:18 2011
@@ -3,8 +3,13 @@ Hadoop MapReduce Change Log
Trunk (unreleased changes)
MAPREDUCE-279
+
+ Add Containers' logs' view to NM UI and link it from AM UI. (vinodkv)
+
Fix race condition in TestJobHistoryEvents and TestJobHistoryParsing. (sharad)
+ Introducing web-UI for NodeManager and linking it from RM UI. (vinodkv)
+
Added functionality to stop/start queues. (acmurthy)
Added functionality to refresh queues at runtime via the 'bin/yarn
@@ -22,11 +27,11 @@ Trunk (unreleased changes)
Add HistoryCleanerService to Job History server. (Krishna Ramachandran
via sharad)
- Install sanitized poms for downstream sanity (Luke Lu via mahadev)
+ Install sanitized poms for downstream sanity (Luke Lu via mahadev)
- WebApp for Job History (Krishna Ramachandran via mahadev)
+ WebApp for Job History (Krishna Ramachandran via mahadev)
- Fix YarnRemoteException to give more details. (siddharth seth via mahadev)
+ Fix YarnRemoteException to give more details. (siddharth seth via mahadev)
MAPREDUCE-2405. Implement uber-AppMaster (in-cluster LocalJobRunner for
MRv2). (Greg Roelofs and Sharad Agarwal via mahadev)
@@ -38,6 +43,14 @@ Trunk (unreleased changes)
MAPREDUCE-2440. Name clashes in TypeConverter (luke via mahadev)
+ Implementing Containers' memory monitoring. (vinodkv)
+
+ MAPREDUCE-2405: Implement uber-AppMaster (in-cluster LocalJobRunner for MRv2) (Greg Roelofs via mahadev)
+
+ Implement health-checks for the node - server side(ResourceManager) changes. (vinodkv)
+
+ MAPREDUCE-2414. Change MRv2 to use generic interfaces. (Siddharth Seth via acmurthy)
+
INCOMPATIBLE CHANGES
MAPREDUCE-1866. Removes deprecated class
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/TaskAttempt.java Tue Apr 26 04:38:18 2011
@@ -46,9 +46,9 @@ public interface TaskAttempt {
*/
ContainerId getAssignedContainerID();
- /**If container Assigned then return container mgr address, otherwise null.
+ /**If container Assigned then return the node's http address, otherwise null.
*/
- String getAssignedContainerMgrAddress();
+ String getNodeHttpAddress();
/** Returns time at which container is launched. If container is not launched
* yet, returns 0.
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java Tue Apr 26 04:38:18 2011
@@ -26,16 +26,19 @@ import org.apache.hadoop.yarn.api.record
public class TaskAttemptContainerAssignedEvent extends TaskAttemptEvent {
- private ContainerId containerID;
- private String containerManagerAddress;
- private ContainerToken containerToken;
+ private final ContainerId containerID;
+ private final String containerManagerAddress;
+ private final String nodeHttpAddress;
+ private final ContainerToken containerToken;
public TaskAttemptContainerAssignedEvent(TaskAttemptId id,
ContainerId containerID, String containerManagerAddress,
+ String nodeHttpAddress,
ContainerToken containerToken) {
super(id, TaskAttemptEventType.TA_ASSIGNED);
this.containerID = containerID;
this.containerManagerAddress = containerManagerAddress;
+ this.nodeHttpAddress = nodeHttpAddress;
this.containerToken = containerToken;
}
@@ -50,4 +53,8 @@ public class TaskAttemptContainerAssigne
public ContainerToken getContainerToken() {
return this.containerToken;
}
+
+ public String getNodeHttpAddress() {
+ return this.nodeHttpAddress;
+ }
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Apr 26 04:38:18 2011
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.v2.a
import java.io.File;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.nio.ByteBuffer;
import java.util.ArrayList;
@@ -77,6 +78,7 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.app.speculate.SpeculatorEvent;
import org.apache.hadoop.mapreduce.v2.app.taskclean.TaskCleanupEvent;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.net.NetworkTopology;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -366,6 +368,7 @@ public abstract class TaskAttemptImpl im
private ContainerId containerID;
private String containerMgrAddress;
+ private String nodeHttpAddress;
private WrappedJvmID jvmID;
private ContainerToken containerToken;
@@ -679,13 +682,13 @@ public abstract class TaskAttemptImpl im
}
}
- /**If container Assigned then return container mgr address, otherwise null.
+ /**If container Assigned then return the node's address, otherwise null.
*/
@Override
- public String getAssignedContainerMgrAddress() {
+ public String getNodeHttpAddress() {
readLock.lock();
try {
- return containerMgrAddress;
+ return nodeHttpAddress;
} finally {
readLock.unlock();
}
@@ -844,6 +847,7 @@ public abstract class TaskAttemptImpl im
(TaskAttemptContainerAssignedEvent) event;
taskAttempt.containerID = cEvent.getContainerID();
taskAttempt.containerMgrAddress = cEvent.getContainerManagerAddress();
+ taskAttempt.nodeHttpAddress = cEvent.getNodeHttpAddress();
taskAttempt.containerToken = cEvent.getContainerToken();
// this is a _real_ Task (classic Hadoop mapred flavor):
taskAttempt.remoteTask = taskAttempt.createRemoteTask();
@@ -925,11 +929,14 @@ public abstract class TaskAttemptImpl im
// for it
taskAttempt.taskAttemptListener.register(
taskAttempt.attemptId, taskAttempt.remoteTask, taskAttempt.jvmID);
+ InetSocketAddress nodeHttpInetAddr =
+ NetUtils.createSocketAddr(taskAttempt.nodeHttpAddress); // TODO:
+ // Costly?
TaskAttemptStartedEvent tase =
new TaskAttemptStartedEvent(TypeConverter.fromYarn(taskAttempt.attemptId),
TypeConverter.fromYarn(taskAttempt.attemptId.getTaskId().getTaskType()),
taskAttempt.launchTime,
- "tracker", 0);
+ nodeHttpInetAddr.getHostName(), nodeHttpInetAddr.getPort());
taskAttempt.eventHandler.handle
(new JobHistoryEvent(taskAttempt.attemptId.getTaskId().getJobId(), tase));
taskAttempt.eventHandler.handle
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskImpl.java Tue Apr 26 04:38:18 2011
@@ -502,11 +502,11 @@ public abstract class TaskImpl implement
TaskAttempt attempt = attempts.get(attemptId);
//raise the completion event only if the container is assigned
// to nextAttemptNumber
- if (attempt.getAssignedContainerMgrAddress() != null) {
+ if (attempt.getNodeHttpAddress() != null) {
TaskAttemptCompletionEvent tce = recordFactory.newRecordInstance(TaskAttemptCompletionEvent.class);
tce.setEventId(-1);
//TODO: XXXXXX hardcoded port
- tce.setMapOutputServerAddress("http://" + attempt.getAssignedContainerMgrAddress().split(":")[0] + ":8080");
+ tce.setMapOutputServerAddress("http://" + attempt.getNodeHttpAddress().split(":")[0] + ":8080");
tce.setStatus(status);
tce.setAttemptId(attempt.getID());
tce.setAttemptRunTime(0); // TODO: set the exact run time of the task.
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/local/LocalContainerAllocator.java Tue Apr 26 04:38:18 2011
@@ -70,6 +70,7 @@ public class LocalContainerAllocator ext
eventHandler.handle(new TaskAttemptContainerAssignedEvent(
event.getAttemptID(), cID,
"localhost",//put the AppMaster hostname (TODO)
+ "localhost:9999",// put the httpAddress
null));
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java Tue Apr 26 04:38:18 2011
@@ -280,7 +280,7 @@ public class RMContainerAllocator extend
private void assign(Priority priority, List<Container> allocatedContainers) {
for (Iterator<Container> i=allocatedContainers.iterator(); i.hasNext();) {
Container allocatedContainer = i.next();
- String host = allocatedContainer.getHostName();
+ String host = allocatedContainer.getContainerManagerAddress();
Resource capability = allocatedContainer.getResource();
LinkedList<ContainerRequestEvent> requestList =
@@ -322,14 +322,15 @@ public class RMContainerAllocator extend
// send the container-assigned event to task attempt
eventHandler.handle(new TaskAttemptContainerAssignedEvent(
assigned.getAttemptID(), allocatedContainer.getId(),
- allocatedContainer.getHostName(),
+ allocatedContainer.getContainerManagerAddress(),
+ allocatedContainer.getNodeHttpAddress(),
allocatedContainer.getContainerToken()));
assignedMap.put(allocatedContainer.getId(), assigned.getAttemptID());
LOG.info("Assigned container (" + allocatedContainer + ") " +
" to task " + assigned.getAttemptID() + " at priority " + priority +
- " on node " + allocatedContainer.getHostName());
+ " on node " + allocatedContainer.getContainerManagerAddress());
}
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/StaticContainerAllocator.java Tue Apr 26 04:38:18 2011
@@ -143,7 +143,7 @@ public class StaticContainerAllocator ex
context.getEventHandler().handle(
new TaskAttemptContainerAssignedEvent(
event.getAttemptID(),
- containerID, containerMgr, null));
+ containerID, containerMgr, "localhost:9999", null));
}
}
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java Tue Apr 26 04:38:18 2011
@@ -24,6 +24,8 @@ import com.google.inject.Inject;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.*;
@@ -56,7 +58,7 @@ public class TaskPage extends AppView {
th(".id", "Attempt").
th(".progress", "Progress").
th(".state", "State").
- th(".node", "Node").
+ th(".logs", "Logs").
th(".tsh", "Started").
th(".tsh", "Finished").
th(".tsh", "Elapsed").
@@ -65,15 +67,21 @@ public class TaskPage extends AppView {
for (TaskAttempt ta : app.task.getAttempts().values()) {
String taid = MRApps.toString(ta.getID());
String progress = percent(ta.getProgress());
- String node = ta.getAssignedContainerMgrAddress();
+ ContainerId containerId = ta.getAssignedContainerID();
+ String nodeHttpAddr = ta.getNodeHttpAddress();
TaskAttemptReport report = ta.getReport();
long elapsed = Times.elapsed(report.getStartTime(), report.getFinishTime());
+ String containerIdStr = ConverterUtils.toString(containerId);
tbody.
tr().
td(".id", taid).
td(".progress", progress).
td(".state", ta.getState().toString()).
- td(".node", node).
+ td()
+ .a(".logs",
+ url("http://", nodeHttpAddr, "yarn", "containerlogs",
+ containerIdStr),
+ "Logs for " + containerIdStr)._().
td(".ts", String.valueOf(report.getStartTime())).
td(".ts", String.valueOf(report.getFinishTime())).
td(".dt", String.valueOf(elapsed)).
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java Tue Apr 26 04:38:18 2011
@@ -257,7 +257,7 @@ public class MRApp extends MRAppMaster {
cId.setId(containerCount++);
getContext().getEventHandler().handle(
new TaskAttemptContainerAssignedEvent(event.getAttemptID(), cId,
- "dummy", null));
+ "dummy", "localhost:9999",null));
}
};
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRAppBenchmark.java Tue Apr 26 04:38:18 2011
@@ -126,7 +126,8 @@ public class MRAppBenchmark {
//System.out.println("Allocating " + containerCount);
getContext().getEventHandler().handle(
new TaskAttemptContainerAssignedEvent(event
- .getAttemptID(), cId, "dummy", null));
+ .getAttemptID(), cId, "dummy", "localhost:9999",
+ null));
concurrentRunningTasks++;
} else {
Thread.sleep(1000);
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java Tue Apr 26 04:38:18 2011
@@ -232,8 +232,8 @@ public class MockJobs extends MockApps {
}
@Override
- public String getAssignedContainerMgrAddress() {
- return "localhost";
+ public String getNodeHttpAddress() {
+ return "localhost:9999";
}
@Override
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestRuntimeEstimators.java Tue Apr 26 04:38:18 2011
@@ -676,7 +676,7 @@ public class TestRuntimeEstimators {
}
@Override
- public String getAssignedContainerMgrAddress() {
+ public String getNodeHttpAddress() {
throw new UnsupportedOperationException("Not supported yet.");
}
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/CompletedTaskAttempt.java Tue Apr 26 04:38:18 2011
@@ -72,8 +72,8 @@ public class CompletedTaskAttempt implem
}
@Override
- public String getAssignedContainerMgrAddress() {
- return attemptInfo.getHostname();
+ public String getNodeHttpAddress() {
+ return attemptInfo.getHostname() + ":" + attemptInfo.getHttpPort();
}
@Override
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Container.java Tue Apr 26 04:38:18 2011
@@ -1,17 +1,37 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
package org.apache.hadoop.yarn.api.records;
public interface Container extends Comparable<Container> {
- public abstract ContainerId getId();
- public abstract String getHostName();
- public abstract Resource getResource();
- public abstract ContainerState getState();
- public abstract ContainerToken getContainerToken();
+ ContainerId getId();
+ String getContainerManagerAddress();
+ String getNodeHttpAddress();
+ Resource getResource();
+ ContainerState getState();
+ ContainerToken getContainerToken();
- public abstract void setId(ContainerId id);
- public abstract void setHostName(String hostName);
- public abstract void setResource(Resource resource);
- public abstract void setState(ContainerState state);
- public abstract void setContainerToken(ContainerToken containerToken);
+ void setId(ContainerId id);
+ void setContainerManagerAddress(String containerManagerAddress);
+ void setNodeHttpAddress(String nodeHttpAddress);
+ void setResource(Resource resource);
+ void setState(ContainerState state);
+ void setContainerToken(ContainerToken containerToken);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerPBImpl.java Tue Apr 26 04:38:18 2011
@@ -114,23 +114,43 @@ public class ContainerPBImpl extends Pro
this.containerId = id;
}
@Override
- public String getHostName() {
+ public String getContainerManagerAddress() {
ContainerProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasHostName()) {
+ if (!p.hasContainerManagerAddress()) {
return null;
}
- return (p.getHostName());
+ return (p.getContainerManagerAddress());
}
@Override
- public void setHostName(String hostName) {
+ public void setContainerManagerAddress(String containerManagerAddress) {
maybeInitBuilder();
- if (hostName == null) {
- builder.clearHostName();
+ if (containerManagerAddress == null) {
+ builder.clearContainerManagerAddress();
return;
}
- builder.setHostName((hostName));
+ builder.setContainerManagerAddress((containerManagerAddress));
}
+
+ @Override
+ public String getNodeHttpAddress() {
+ ContainerProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNodeHttpAddress()) {
+ return null;
+ }
+ return (p.getNodeHttpAddress());
+ }
+
+ @Override
+ public void setNodeHttpAddress(String nodeHttpAddress) {
+ maybeInitBuilder();
+ if (nodeHttpAddress == null) {
+ builder.clearNodeHttpAddress();
+ return;
+ }
+ builder.setNodeHttpAddress(nodeHttpAddress);
+ }
+
@Override
public Resource getResource() {
ContainerProtoOrBuilder p = viaProto ? proto : builder;
@@ -208,7 +228,7 @@ public class ContainerPBImpl extends Pro
@Override
public int compareTo(Container other) {
if (this.getId().compareTo(other.getId()) == 0) {
- if (this.getHostName().compareTo(other.getHostName()) == 0) {
+ if (this.getContainerManagerAddress().compareTo(other.getContainerManagerAddress()) == 0) {
if (this.getResource().compareTo(other.getResource()) == 0) {
if (this.getState().compareTo(other.getState()) == 0) {
//ContainerToken
@@ -220,7 +240,7 @@ public class ContainerPBImpl extends Pro
return this.getResource().compareTo(other.getResource());
}
} else {
- return this.getHostName().compareTo(other.getHostName());
+ return this.getContainerManagerAddress().compareTo(other.getContainerManagerAddress());
}
} else {
return this.getId().compareTo(other.getId());
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/proto/yarn_protos.proto Tue Apr 26 04:38:18 2011
@@ -39,10 +39,11 @@ message ContainerTokenProto {
message ContainerProto {
optional ContainerIdProto id = 1;
- optional string host_name = 2;
- optional ResourceProto resource = 3;
- optional ContainerStateProto state = 4;
- optional ContainerTokenProto container_token =5;
+ optional string container_manager_address = 2;
+ optional string node_http_address = 3;
+ optional ResourceProto resource = 4;
+ optional ContainerStateProto state = 5;
+ optional ContainerTokenProto container_token = 6;
}
enum ApplicationStateProto {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMConfig.java Tue Apr 26 04:38:18 2011
@@ -32,7 +32,7 @@ public class NMConfig {
/** host:port address to which to bind to **/
public static final String NM_BIND_ADDRESS = NM_PREFIX + "address";
- public static final String DEFAULT_NM_HTTP_BIND_ADDRESS = "localhost:9999";
+ public static final String DEFAULT_NM_HTTP_BIND_ADDRESS = "0.0.0.0:9999";
/** host:port address to which webserver has to bind to **/
public static final String NM_HTTP_BIND_ADDRESS = NM_PREFIX + "http-address";
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Tue Apr 26 04:38:18 2011
@@ -72,6 +72,7 @@ public class NodeStatusUpdaterImpl exten
private String rmAddress;
private Resource totalResource;
private String containerManagerBindAddress;
+ private String nodeHttpAddress;
private String hostName;
private int containerManagerPort;
private int httpPort;
@@ -122,6 +123,7 @@ public class NodeStatusUpdaterImpl exten
this.httpPort = httpBindAddress.getPort();
this.containerManagerBindAddress =
this.hostName + ":" + this.containerManagerPort;
+ this.nodeHttpAddress = this.hostName + ":" + this.httpPort;
LOG.info("Configured ContainerManager Address is "
+ this.containerManagerBindAddress);
// Registration has to be in start so that ContainerManager can get the
@@ -207,7 +209,8 @@ public class NodeStatusUpdaterImpl exten
// Clone the container to send it to the RM
org.apache.hadoop.yarn.api.records.Container c = container.cloneAndGetContainer();
- c.setHostName(this.containerManagerBindAddress);
+ c.setContainerManagerAddress(this.containerManagerBindAddress);
+ c.setNodeHttpAddress(this.nodeHttpAddress); // TODO: don't set everytime.
applicationContainers.add(c);
++numActiveContainers;
LOG.info("Sending out status for container: " + c);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/AllContainersPage.java Tue Apr 26 04:38:18 2011
@@ -77,7 +77,7 @@ public class AllContainersPage extends N
.tr()
.td()._("ContainerId")._()
.td()._("ContainerState")._()
- .td()._("logs-url")._()
+ .td()._("logs")._()
._()
._().tbody();
for (Entry<ContainerId, Container> entry : this.nmContext
@@ -90,7 +90,9 @@ public class AllContainersPage extends N
.td().a(url($(NM_HTTP_URL), "yarn", "container", containerIdStr),
containerIdStr)._()
.td()._(container.getContainerState())._()
- .td()._("TODO")._()
+ .td()
+ .a(url($(NM_HTTP_URL), "yarn", "containerlogs", containerIdStr),
+ "logs")._()
._();
}
tableBody._()._()._();
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java?rev=1096654&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java Tue Apr 26 04:38:18 2011
@@ -0,0 +1,175 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.webapp;
+
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.DEFAULT_NM_LOCAL_DIR;
+import static org.apache.hadoop.yarn.server.nodemanager.NMConfig.NM_LOCAL_DIR;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.EnumSet;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ApplicationLocalizer;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.SubView;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+
+import com.google.inject.Inject;
+
+public class ContainerLogsPage extends NMView {
+ @Override
+ protected Class<? extends SubView> content() {
+ return ContainersLogsBlock.class;
+ }
+
+ public static class ContainersLogsBlock extends HtmlBlock implements
+ NMWebParams {
+
+ private final Configuration conf;
+ private final Context nmContext;
+ private final RecordFactory recordFactory;
+
+ @Inject
+ public ContainersLogsBlock(Configuration conf, Context context) {
+ this.conf = conf;
+ this.nmContext = context;
+ this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
+ }
+
+ @Override
+ protected void render(Block html) {
+ DIV<Hamlet> div = html.div("#content");
+
+ ContainerId containerId =
+ ConverterUtils.toContainerId(this.recordFactory, $(CONTAINER_ID));
+ Container container = this.nmContext.getContainers().get(containerId);
+
+ if (EnumSet.of(ContainerState.NEW, ContainerState.LOCALIZING,
+ ContainerState.LOCALIZING).contains(container.getContainerState())) {
+ div.h1("Container is not yet running. Current state is "
+ + container.getContainerState())
+ ._();
+ } else if (EnumSet.of(ContainerState.RUNNING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerState.EXITED_WITH_SUCCESS).contains(
+ container.getContainerState())) {
+
+ if (!$(CONTAINER_LOG_TYPE).isEmpty()) {
+ // TODO: Get the following from logs' owning component.
+ File containerLogsDir =
+ getContainerLogDir(this.conf, this.nmContext, containerId);
+ File logFile = new File(containerLogsDir, $(CONTAINER_LOG_TYPE));
+ div.h1(logFile.getName());
+ long start =
+ $("start").isEmpty() ? -4 * 1024 : Long.parseLong($("start"));
+ start = start < 0 ? logFile.length() + start : start;
+ start = start < 0 ? 0 : start;
+ long end =
+ $("end").isEmpty() ? logFile.length() : Long
+ .parseLong($("end"));
+ end = end < 0 ? logFile.length() + end : end;
+ end = end < 0 ? logFile.length() : end;
+ if (start > end) {
+ writer().write("Invalid start and end values!");
+ } else {
+ try {
+ long toRead = end - start;
+ if (toRead < logFile.length()) {
+ div._("Showing " + toRead + " bytes. Click ")
+ .a(url($(NM_HTTP_URL), "yarn", "containerlogs",
+ $(CONTAINER_ID),
+ logFile.getName()), "here")
+ ._(" for full log").br()._();
+ }
+ // TODO: Use secure IO Utils to avoid symlink attacks.
+ FileReader reader = new FileReader(logFile);
+ char[] cbuf = new char[65536];
+ reader.skip(start);
+ int len = 0;
+ int totalRead = 0;
+ writer().write("<pre>");
+ while ((len = reader.read(cbuf, 0, (int) toRead)) > 0
+ && totalRead < (end - start)) {
+ writer().write(cbuf, 0, len); // TODO: HTMl Quoting?
+ totalRead += len;
+ toRead = toRead - totalRead;
+ }
+ writer().write("</pre>");
+ } catch (IOException e) {
+ writer().write(
+ "Exception reading log-file "
+ + StringUtils.stringifyException(e));
+ }
+ }
+ div._();
+ } else {
+ // Just print out the log-types
+ File containerLogsDir =
+ getContainerLogDir(this.conf, this.nmContext, containerId);
+ // TODO: No nested dir structure. Fix MR userlogs.
+ for (File logFile : containerLogsDir.listFiles()) {
+ div
+ .p()
+ .a(
+ url($(NM_HTTP_URL), "yarn", "containerlogs",
+ $(CONTAINER_ID),
+ logFile.getName(), "?start=-4076"),
+ logFile.getName() + " : Total file length is "
+ + logFile.length() + " bytes.")
+ ._();
+ }
+ div._();
+ }
+ } else {
+ div.h1("Container is no longer running..")._();
+ }
+ }
+
+ static File getContainerLogDir(Configuration conf,
+ Context nmContext, ContainerId containerId) {
+ String[] sLocalDirs =
+ conf.getStrings(NM_LOCAL_DIR, DEFAULT_NM_LOCAL_DIR);
+ File localDir = new File(sLocalDirs[0]); // TODO: Fix
+ File userCacheDir =
+ new File(localDir, ApplicationLocalizer.USERCACHE);
+ String user =
+ nmContext.getContainers().get(containerId).getUser();
+ File userDir = new File(userCacheDir, user);
+ File appCacheDir = new File(userDir, ApplicationLocalizer.APPCACHE);
+ File containerDir =
+ new File(new File(appCacheDir,
+ ConverterUtils.toString(containerId
+ .getAppId())), ConverterUtils.toString(containerId));
+ File containerLogsDir = new File(containerDir, "logs");
+ return containerLogsDir;
+ }
+
+ }
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerPage.java Tue Apr 26 04:38:18 2011
@@ -68,7 +68,10 @@ public class ContainerPage extends NMVie
Container container = this.nmContext.getContainers().get(containerID);
info("Container information")
._("ContainerID", $(CONTAINER_ID))
- ._("ContainerState", container.getContainerState());
+ ._("ContainerState", container.getContainerState())
+ ._("logs",
+ url($(NM_HTTP_URL), "yarn", "containerlogs", $(CONTAINER_ID)),
+ "Link to logs");
html._(InfoBlock.class);
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMController.java Tue Apr 26 04:38:18 2011
@@ -66,4 +66,8 @@ public class NMController extends Contro
public void container() {
render(ContainerPage.class);
}
+
+ public void logs() {
+ render(ContainerLogsPage.class);
+ }
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebParams.java Tue Apr 26 04:38:18 2011
@@ -23,4 +23,5 @@ public interface NMWebParams {
public static final String NM_HTTP_URL = "nm.http-url";
public static final String APPLICATION_ID = "nm.appId";
public static final String CONTAINER_ID = "nm.containerId";
+ public static final String CONTAINER_LOG_TYPE= "nm.containerLogType";
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java Tue Apr 26 04:38:18 2011
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.ACCORDION;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
+import java.util.Date;
+
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.webapp.SubView;
@@ -63,8 +65,8 @@ public class NodePage extends NMView {
this.resourceView.getPmemAllocatedForContainers() + "bytes")
._("NodeHealthyStatus",
this.context.getNodeHealthStatus().getIsNodeHealthy())
- ._("LastNodeHealthTime",
- this.context.getNodeHealthStatus().getLastHealthReportTime())
+ ._("LastNodeHealthTime", new Date(
+ this.context.getNodeHealthStatus().getLastHealthReportTime()))
._("NodeHealthReport",
this.context.getNodeHealthStatus().getHealthReport());
html._(InfoBlock.class);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java Tue Apr 26 04:38:18 2011
@@ -96,6 +96,8 @@ public class WebServer extends AbstractS
"application");
route(pajoin("/container", CONTAINER_ID), NMController.class,
"container");
+ route(pajoin("/containerlogs", CONTAINER_ID, CONTAINER_LOG_TYPE),
+ NMController.class, "logs");
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Tue Apr 26 04:38:18 2011
@@ -18,6 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.Writer;
import java.nio.ByteBuffer;
import java.util.HashMap;
import java.util.Map;
@@ -32,19 +36,31 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.Before;
import org.junit.Test;
public class TestNMWebServer {
+ private static final File testRootDir = new File("target-"
+ + TestNMWebServer.class.getName());
+
+ @Before
+ public void setup() {
+ testRootDir.mkdirs();
+ }
+
@Test
- public void testNMWebApp() throws InterruptedException {
+ public void testNMWebApp() throws InterruptedException, IOException {
Context nmContext = new NodeManager.NMContext();
ResourceView resourceView = new ResourceView() {
@Override
@@ -58,10 +74,11 @@ public class TestNMWebServer {
};
WebServer server = new WebServer(nmContext, resourceView);
Configuration conf = new Configuration();
+ conf.set(NMConfig.NM_LOCAL_DIR, testRootDir.getAbsolutePath());
server.init(conf);
server.start();
- // Add a couple of applications and the corresponding containers
+ // Add an application and the corresponding containers
RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(conf);
Dispatcher dispatcher = new AsyncDispatcher();
@@ -71,35 +88,52 @@ public class TestNMWebServer {
Map<String, LocalResource> resources =
new HashMap<String, LocalResource>();
ByteBuffer containerTokens = ByteBuffer.allocate(0);
- ApplicationId appId1 =
+ ApplicationId appId =
BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
- ApplicationId appId2 =
- BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 2);
- for (ApplicationId appId : new ApplicationId[] { appId1, appId2 }) {
Application app =
new ApplicationImpl(dispatcher, user, appId, env, resources,
containerTokens);
nmContext.getApplications().put(appId, app);
- }
- ContainerId container11 =
- BuilderUtils.newContainerId(recordFactory, appId1, 0);
- ContainerId container12 =
- BuilderUtils.newContainerId(recordFactory, appId1, 1);
- ContainerId container21 =
- BuilderUtils.newContainerId(recordFactory, appId2, 0);
- for (ContainerId containerId : new ContainerId[] { container11,
- container12, container21 }) {
+ ContainerId container1 =
+ BuilderUtils.newContainerId(recordFactory, appId, 0);
+ ContainerId container2 =
+ BuilderUtils.newContainerId(recordFactory, appId, 1);
+ for (ContainerId containerId : new ContainerId[] { container1,
+ container2}) {
// TODO: Use builder utils
ContainerLaunchContext launchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
launchContext.setContainerId(containerId);
- Container container = new ContainerImpl(dispatcher, launchContext);
+ launchContext.setUser(user);
+ Container container = new ContainerImpl(dispatcher, launchContext) {
+ public ContainerState getContainerState() {
+ return ContainerState.RUNNING;
+ };
+ };
nmContext.getContainers().put(containerId, container);
//TODO: Gross hack. Fix in code.
nmContext.getApplications().get(containerId.getAppId()).getContainers()
.put(containerId, container);
+ writeContainerLogs(conf, nmContext, containerId);
}
+ // TODO: Pull logs and test contents.
// Thread.sleep(1000000);
}
+
+ private void writeContainerLogs(Configuration conf, Context nmContext,
+ ContainerId containerId)
+ throws IOException {
+ // ContainerLogDir should be created
+ File containerLogDir =
+ ContainerLogsPage.ContainersLogsBlock.getContainerLogDir(conf,
+ nmContext, containerId);
+ containerLogDir.mkdirs();
+ for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
+ Writer writer = new FileWriter(new File(containerLogDir, fileType));
+ writer.write(ConverterUtils.toString(containerId) + "\n Hello "
+ + fileType + "!");
+ writer.close();
+ }
+ }
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Tue Apr 26 04:38:18 2011
@@ -234,7 +234,7 @@ public class ResourceManager extends Com
resourceManager = new ResourceManager();
resourceManager.init(conf);
resourceManager.start();
- } catch (Exception e) {
+ } catch (Throwable e) {
LOG.error("Error starting RM", e);
} finally {
if (resourceManager != null) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMLauncher.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMLauncher.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/AMLauncher.java Tue Apr 26 04:38:18 2011
@@ -138,7 +138,7 @@ public class AMLauncher implements Runna
Container container = master.getMasterContainer();
- final String containerManagerBindAddress = container.getHostName();
+ final String containerManagerBindAddress = container.getContainerManagerAddress();
final YarnRPC rpc = YarnRPC.create(conf); // TODO: Don't create again and again.
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Container.java Tue Apr 26 04:38:18 2011
@@ -18,10 +18,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.resource;
-import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.BuilderUtils;
@@ -34,7 +34,8 @@ public class Container {
org.apache.hadoop.yarn.api.records.Container c) {
org.apache.hadoop.yarn.api.records.Container container = recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Container.class);
container.setId(c.getId());
- container.setHostName(c.getHostName());
+ container.setContainerManagerAddress(c.getContainerManagerAddress());
+ container.setNodeHttpAddress(c.getNodeHttpAddress());
container.setResource(c.getResource());
container.setState(c.getState());
return container;
@@ -42,19 +43,24 @@ public class Container {
public static org.apache.hadoop.yarn.api.records.Container create(
RecordFactory recordFactory, ApplicationId applicationId,
- int containerId, String hostName, Resource resource) {
+ int containerId, String containerManagerAddress, String nodeHttpAddress,
+ Resource resource) {
ContainerId containerID =
BuilderUtils
.newContainerId(recordFactory, applicationId, containerId);
- return create(containerID, hostName, resource);
+ return create(containerID, containerManagerAddress, nodeHttpAddress,
+ resource);
}
public static org.apache.hadoop.yarn.api.records.Container create(
- ContainerId containerId,
- String hostName, Resource resource) {
- org.apache.hadoop.yarn.api.records.Container container = recordFactory.newRecordInstance(org.apache.hadoop.yarn.api.records.Container.class);
+ ContainerId containerId, String containerManagerAddress,
+ String nodeHttpAddress, Resource resource) {
+ org.apache.hadoop.yarn.api.records.Container container =
+ recordFactory
+ .newRecordInstance(org.apache.hadoop.yarn.api.records.Container.class);
container.setId(containerId);
- container.setHostName(hostName);
+ container.setContainerManagerAddress(containerManagerAddress);
+ container.setNodeHttpAddress(nodeHttpAddress);
container.setResource(resource);
container.setState(ContainerState.INITIALIZING);
return container;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/Application.java Tue Apr 26 04:38:18 2011
@@ -297,7 +297,7 @@ public class Application {
allocated.add(container);
LOG.debug("allocate: applicationId=" + applicationId +
- " container=" + container.getId() + " host=" + container.getHostName());
+ " container=" + container.getId() + " host=" + container.getContainerManagerAddress());
}
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Tue Apr 26 04:38:18 2011
@@ -447,7 +447,7 @@ implements ResourceScheduler, CapacitySc
Container container) {
// Reap containers
LOG.info("Application " + applicationId + " released container " + container);
- NodeManager nodeManager = nodes.get(container.getHostName());
+ NodeManager nodeManager = nodes.get(container.getContainerManagerAddress());
return nodeManager.releaseContainer(container);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Tue Apr 26 04:38:18 2011
@@ -683,22 +683,22 @@ public class LeafQueue implements Queue
List<Container> containers =
new ArrayList<Container>();
Container container =
- org.apache.hadoop.yarn.server.resourcemanager.resource.Container
- .create(recordFactory, application.getApplicationId(),
- application.getNewContainerId(),
- node.getNodeAddress(), capability);
+ org.apache.hadoop.yarn.server.resourcemanager.resource.Container
+ .create(recordFactory, application.getApplicationId(),
+ application.getNewContainerId(), node.getNodeAddress(),
+ node.getHttpAddress(), capability);
// If security is enabled, send the container-tokens too.
if (UserGroupInformation.isSecurityEnabled()) {
ContainerToken containerToken = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerToken.class);
ContainerTokenIdentifier tokenidentifier =
new ContainerTokenIdentifier(container.getId(),
- container.getHostName(), container.getResource());
+ container.getContainerManagerAddress(), container.getResource());
containerToken.setIdentifier(ByteBuffer.wrap(tokenidentifier.getBytes()));
containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
containerToken.setPassword(ByteBuffer.wrap(containerTokenSecretManager
.createPassword(tokenidentifier)));
- containerToken.setService(container.getHostName()); // TODO: port
+ containerToken.setService(container.getContainerManagerAddress());
container.setContainerToken(containerToken);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Tue Apr 26 04:38:18 2011
@@ -383,22 +383,22 @@ public class FifoScheduler implements Re
for (int i=0; i < assignedContainers; ++i) {
Container container =
org.apache.hadoop.yarn.server.resourcemanager.resource.Container
- .create(recordFactory, application.getApplicationId(),
- application.getNewContainerId(),
- node.getNodeAddress(), capability);
+ .create(recordFactory, application.getApplicationId(),
+ application.getNewContainerId(), node.getNodeAddress(),
+ node.getHttpAddress(), capability);
// If security is enabled, send the container-tokens too.
if (UserGroupInformation.isSecurityEnabled()) {
ContainerToken containerToken = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerToken.class);
ContainerTokenIdentifier tokenidentifier =
new ContainerTokenIdentifier(container.getId(),
- container.getHostName(), container.getResource());
+ container.getContainerManagerAddress(), container.getResource());
containerToken.setIdentifier(
ByteBuffer.wrap(tokenidentifier.getBytes()));
containerToken.setKind(ContainerTokenIdentifier.KIND.toString());
containerToken.setPassword(
ByteBuffer.wrap(containerTokenSecretManager
.createPassword(tokenidentifier)));
- containerToken.setService(container.getHostName()); // TODO: port
+ containerToken.setService(container.getContainerManagerAddress());
container.setContainerToken(containerToken);
}
containers.add(container);
@@ -515,7 +515,7 @@ public class FifoScheduler implements Re
Container container) {
// Reap containers
LOG.info("Application " + applicationId + " released container " + container);
- NodeManager nodeManager = nodes.get(container.getHostName());
+ NodeManager nodeManager = nodes.get(container.getContainerManagerAddress());
return nodeManager.releaseContainer(container);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java Tue Apr 26 04:38:18 2011
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import java.util.Date;
+
import com.google.inject.Inject;
import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo;
@@ -62,8 +64,8 @@ class NodesPage extends RmView {
td().a("http://" + ni.getHttpAddress(), ni.getHttpAddress())._().
td(ni.getNodeHealthStatus().getIsNodeHealthy() ? "Healthy"
: "Unhealthy").
- td(String.valueOf(ni.getNodeHealthStatus()
- .getLastHealthReportTime())).
+ td(new Date(ni.getNodeHealthStatus()
+ .getLastHealthReportTime()).toString()).
td(String.valueOf(ni.getNodeHealthStatus().getHealthReport())).
td(String.valueOf(ni.getNumContainers())).
td(String.valueOf(ni.getUsedResource().getMemory())).
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Tue Apr 26 04:38:18 2011
@@ -322,7 +322,7 @@ public class Application {
List<Container> containers) throws IOException {
for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
Container container = i.next();
- String host = container.getHostName();
+ String host = container.getContainerManagerAddress();
if (org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.equals(
requestSpec.get(priority), container.getResource())) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Tue Apr 26 04:38:18 2011
@@ -61,7 +61,8 @@ public class NodeManager implements Cont
private static final Log LOG = LogFactory.getLog(NodeManager.class);
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
- final private String hostName;
+ final private String containerManagerAddress;
+ final private String nodeHttpAddress;
final private String rackName;
final private NodeId nodeId;
final private Resource capability;
@@ -76,7 +77,8 @@ public class NodeManager implements Cont
public NodeManager(String hostName, int containerManagerPort, int httpPort,
String rackName, int memory, RMResourceTrackerImpl resourceTracker)
throws IOException {
- this.hostName = hostName;
+ this.containerManagerAddress = hostName + ":" + containerManagerPort;
+ this.nodeHttpAddress = hostName + ":" + httpPort;
this.rackName = rackName;
this.resourceTracker = resourceTracker;
this.capability =
@@ -101,7 +103,7 @@ public class NodeManager implements Cont
}
public String getHostName() {
- return hostName;
+ return containerManagerAddress;
}
public String getRackName() {
@@ -161,14 +163,15 @@ public class NodeManager implements Cont
if (container.getId().compareTo(containerLaunchContext.getContainerId()) == 0) {
throw new IllegalStateException(
"Container " + containerLaunchContext.getContainerId() +
- " already setup on node " + hostName);
+ " already setup on node " + containerManagerAddress);
}
}
Container container =
- org.apache.hadoop.yarn.server.resourcemanager.resource.Container.create(
- containerLaunchContext.getContainerId(),
- hostName, containerLaunchContext.getResource());
+ org.apache.hadoop.yarn.server.resourcemanager.resource.Container
+ .create(containerLaunchContext.getContainerId(),
+ containerManagerAddress, nodeHttpAddress,
+ containerLaunchContext.getResource());
applicationContainers.add(container);
org.apache.hadoop.yarn.server.resourcemanager.resource.Resource.subtractResource(
@@ -177,7 +180,7 @@ public class NodeManager implements Cont
used, containerLaunchContext.getResource());
LOG.info("DEBUG --- startContainer:" +
- " node=" + hostName +
+ " node=" + containerManagerAddress +
" application=" + applicationId +
" container=" + container +
" available=" + available +
@@ -188,7 +191,7 @@ public class NodeManager implements Cont
}
synchronized public void checkResourceUsage() {
- LOG.info("Checking resource usage for " + hostName);
+ LOG.info("Checking resource usage for " + containerManagerAddress);
Assert.assertEquals(available.getMemory(),
nodeInfo.getAvailableResource().getMemory());
Assert.assertEquals(used.getMemory(),
@@ -233,7 +236,7 @@ public class NodeManager implements Cont
used, container.getResource());
LOG.info("DEBUG --- stopContainer:" +
- " node=" + hostName +
+ " node=" + containerManagerAddress +
" application=" + applicationId +
" container=" + containerID +
" available=" + available +
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java Tue Apr 26 04:38:18 2011
@@ -148,7 +148,8 @@ public class TestAMRestart extends TestC
List<ResourceRequest> ask, List<Container> release) throws IOException {
Container container = recordFactory.newRecordInstance(Container.class);
container.setContainerToken(recordFactory.newRecordInstance(ContainerToken.class));
- container.setHostName("localhost");
+ container.setContainerManagerAddress("localhost");
+ container.setNodeHttpAddress("localhost:9999");
container.setId(recordFactory.newRecordInstance(ContainerId.class));
container.getId().setAppId(appID);
container.getId().setId(count);
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java?rev=1096654&r1=1096653&r2=1096654&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerTokenSecretManager.java Tue Apr 26 04:38:18 2011
@@ -238,7 +238,7 @@ public class TestContainerTokenSecretMan
public Void run() {
ContainerManager client =
(ContainerManager) yarnRPC.getProxy(ContainerManager.class,
- NetUtils.createSocketAddr(allocatedContainer.getHostName()
+ NetUtils.createSocketAddr(allocatedContainer.getContainerManagerAddress()
), conf);
try {
GetContainerStatusRequest request = recordFactory.newRecordInstance(GetContainerStatusRequest.class);
@@ -277,7 +277,7 @@ public class TestContainerTokenSecretMan
public Void run() {
try {
yarnRPC.getProxy(ContainerManager.class, NetUtils
- .createSocketAddr(allocatedContainer.getHostName()), conf);
+ .createSocketAddr(allocatedContainer.getContainerManagerAddress()), conf);
fail("Connection initiation with illegally modified tokens is expected to fail.");
} catch (YarnException e) {
LOG.info("Error", e);