You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by at...@apache.org on 2013/04/13 01:05:38 UTC
svn commit: r1467511 [2/6] - in
/hadoop/common/branches/HDFS-347/hadoop-yarn-project: ./
hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api...
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Fri Apr 12 23:05:28 2013
@@ -23,7 +23,6 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
-import java.util.Random;
import java.util.concurrent.Callable;
import java.util.regex.Pattern;
@@ -36,13 +35,12 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.RunJar;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
-import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Download a single URL to the local disk.
@@ -51,8 +49,7 @@ import org.apache.hadoop.yarn.util.Conve
public class FSDownload implements Callable<Path> {
private static final Log LOG = LogFactory.getLog(FSDownload.class);
-
- private Random rand;
+
private FileContext files;
private final UserGroupInformation userUgi;
private Configuration conf;
@@ -71,13 +68,12 @@ public class FSDownload implements Calla
public FSDownload(FileContext files, UserGroupInformation ugi, Configuration conf,
- Path destDirPath, LocalResource resource, Random rand) {
+ Path destDirPath, LocalResource resource) {
this.conf = conf;
this.destDirPath = destDirPath;
this.files = files;
this.userUgi = ugi;
this.resource = resource;
- this.rand = rand;
}
LocalResource getResource() {
@@ -270,11 +266,6 @@ public class FSDownload implements Calla
} catch (URISyntaxException e) {
throw new IOException("Invalid resource", e);
}
- Path tmp;
- do {
- tmp = new Path(destDirPath, String.valueOf(rand.nextLong()));
- } while (files.util().exists(tmp));
- destDirPath = tmp;
createDir(destDirPath, cachePerms);
final Path dst_work = new Path(destDirPath + "_tmp");
createDir(dst_work, cachePerms);
@@ -305,8 +296,6 @@ public class FSDownload implements Calla
files.delete(dst_work, true);
} catch (FileNotFoundException ignore) {
}
- // clear ref to internal var
- rand = null;
conf = null;
resource = null;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Fri Apr 12 23:05:28 2013
@@ -59,9 +59,15 @@
<!-- Resource Manager Configs -->
<property>
+ <description>The hostname of the RM.</description>
+ <name>yarn.resourcemanager.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
<description>The address of the applications manager interface in the RM.</description>
<name>yarn.resourcemanager.address</name>
- <value>0.0.0.0:8032</value>
+ <value>${yarn.resourcemanager.hostname}:8032</value>
</property>
<property>
@@ -84,7 +90,7 @@
<property>
<description>The address of the scheduler interface.</description>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>0.0.0.0:8030</value>
+ <value>${yarn.resourcemanager.hostname}:8030</value>
</property>
<property>
@@ -96,12 +102,12 @@
<property>
<description>The address of the RM web application.</description>
<name>yarn.resourcemanager.webapp.address</name>
- <value>0.0.0.0:8088</value>
+ <value>${yarn.resourcemanager.hostname}:8088</value>
</property>
<property>
<name>yarn.resourcemanager.resource-tracker.address</name>
- <value>0.0.0.0:8031</value>
+ <value>${yarn.resourcemanager.hostname}:8031</value>
</property>
<property>
@@ -119,7 +125,7 @@
<property>
<description>The address of the RM admin interface.</description>
<name>yarn.resourcemanager.admin.address</name>
- <value>0.0.0.0:8033</value>
+ <value>${yarn.resourcemanager.hostname}:8033</value>
</property>
<property>
@@ -278,11 +284,23 @@
<value>86400</value>
</property>
+ <property>
+ <description>The heart-beat interval in milliseconds for every NodeManager in the cluster.</description>
+ <name>yarn.resourcemanager.nodemanagers.heartbeat-interval-ms</name>
+ <value>1000</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
+ <description>The hostname of the NM.</description>
+ <name>yarn.nodemanager.hostname</name>
+ <value>0.0.0.0</value>
+ </property>
+
+ <property>
<description>The address of the container manager in the NM.</description>
<name>yarn.nodemanager.address</name>
- <value>0.0.0.0:0</value>
+ <value>${yarn.nodemanager.hostname}:0</value>
</property>
<property>
@@ -337,12 +355,6 @@
</property>
<property>
- <description>Heartbeat interval to RM</description>
- <name>yarn.nodemanager.heartbeat.interval-ms</name>
- <value>1000</value>
- </property>
-
- <property>
<description>Keytab for NM.</description>
<name>yarn.nodemanager.keytab</name>
<value>/etc/krb5.keytab</value>
@@ -360,9 +372,28 @@
</property>
<property>
+ <description>It limits the maximum number of files which will be localized
+ in a single local directory. If the limit is reached then sub-directories
+ will be created and new files will be localized in them. If it is set to
+ a value less than or equal to 36 [which are sub-directories (0-9 and then
+ a-z)] then NodeManager will fail to start. For example; [for public
+ cache] if this is configured with a value of 40 ( 4 files +
+ 36 sub-directories) and the local-dir is "/tmp/local-dir1" then it will
+ allow 4 files to be created directly inside "/tmp/local-dir1/filecache".
+ For files that are localized further it will create a sub-directory "0"
+ inside "/tmp/local-dir1/filecache" and will localize files inside it
+ until it becomes full. If a file is removed from a sub-directory that
+ is marked full, then that sub-directory will be used back again to
+ localize files.
+ </description>
+ <name>yarn.nodemanager.local-cache.max-files-per-directory</name>
+ <value>8192</value>
+ </property>
+
+ <property>
<description>Address where the localizer IPC is.</description>
<name>yarn.nodemanager.localizer.address</name>
- <value>0.0.0.0:8040</value>
+ <value>${yarn.nodemanager.hostname}:8040</value>
</property>
<property>
@@ -493,7 +524,7 @@
<property>
<description>NM Webapp address.</description>
<name>yarn.nodemanager.webapp.address</name>
- <value>0.0.0.0:8042</value>
+ <value>${yarn.nodemanager.hostname}:8042</value>
</property>
<property>
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Fri Apr 12 23:05:28 2013
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.factory.providers.YarnRemoteExceptionFactoryProvider;
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.junit.Test;
/*
@@ -101,13 +103,14 @@ public class TestContainerLaunchRPC {
applicationAttemptId.setAttemptId(0);
containerId.setApplicationAttemptId(applicationAttemptId);
containerId.setId(100);
- containerLaunchContext.setContainerId(containerId);
- containerLaunchContext.setResource(recordFactory
- .newRecordInstance(Resource.class));
+ Container container =
+ BuilderUtils.newContainer(containerId, null, null, recordFactory
+ .newRecordInstance(Resource.class), null, null);
StartContainerRequest scRequest = recordFactory
.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
+ scRequest.setContainer(container);
try {
proxy.startContainer(scRequest);
} catch (Exception e) {
@@ -141,7 +144,6 @@ public class TestContainerLaunchRPC {
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
- ContainerLaunchContext container = request.getContainerLaunchContext();
StartContainerResponse response = recordFactory
.newRecordInstance(StartContainerResponse.class);
status = recordFactory.newRecordInstance(ContainerStatus.class);
@@ -153,7 +155,7 @@ public class TestContainerLaunchRPC {
throw new UndeclaredThrowableException(e);
}
status.setState(ContainerState.RUNNING);
- status.setContainerId(container.getContainerId());
+ status.setContainerId(request.getContainer().getId());
status.setExitStatus(0);
return response;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Fri Apr 12 23:05:28 2013
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -52,6 +53,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Test;
@@ -124,20 +126,21 @@ public class TestRPC {
applicationAttemptId.setAttemptId(0);
containerId.setApplicationAttemptId(applicationAttemptId);
containerId.setId(100);
- containerLaunchContext.setContainerId(containerId);
- containerLaunchContext.setResource(
- recordFactory.newRecordInstance(Resource.class));
+ Container mockContainer =
+ BuilderUtils.newContainer(containerId, null, null, recordFactory
+ .newRecordInstance(Resource.class), null, null);
// containerLaunchContext.env = new HashMap<CharSequence, CharSequence>();
// containerLaunchContext.command = new ArrayList<CharSequence>();
StartContainerRequest scRequest =
recordFactory.newRecordInstance(StartContainerRequest.class);
scRequest.setContainerLaunchContext(containerLaunchContext);
+ scRequest.setContainer(mockContainer);
proxy.startContainer(scRequest);
GetContainerStatusRequest gcsRequest =
recordFactory.newRecordInstance(GetContainerStatusRequest.class);
- gcsRequest.setContainerId(containerLaunchContext.getContainerId());
+ gcsRequest.setContainerId(mockContainer.getId());
GetContainerStatusResponse response = proxy.getContainerStatus(gcsRequest);
ContainerStatus status = response.getStatus();
@@ -145,7 +148,7 @@ public class TestRPC {
boolean exception = false;
try {
StopContainerRequest stopRequest = recordFactory.newRecordInstance(StopContainerRequest.class);
- stopRequest.setContainerId(containerLaunchContext.getContainerId());
+ stopRequest.setContainerId(mockContainer.getId());
proxy.stopContainer(stopRequest);
} catch (YarnRemoteException e) {
exception = true;
@@ -179,12 +182,11 @@ public class TestRPC {
@Override
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
- ContainerLaunchContext container = request.getContainerLaunchContext();
StartContainerResponse response =
recordFactory.newRecordInstance(StartContainerResponse.class);
status = recordFactory.newRecordInstance(ContainerStatus.class);
status.setState(ContainerState.RUNNING);
- status.setContainerId(container.getContainerId());
+ status.setContainerId(request.getContainer().getId());
status.setExitStatus(0);
return response;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Fri Apr 12 23:05:28 2013
@@ -35,6 +35,7 @@ import java.util.concurrent.ExecutionExc
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.atomic.AtomicLong;
import java.util.jar.JarOutputStream;
import java.util.jar.Manifest;
@@ -66,6 +67,8 @@ import org.junit.Test;
public class TestFSDownload {
private static final Log LOG = LogFactory.getLog(TestFSDownload.class);
+ private static AtomicLong uniqueNumberGenerator =
+ new AtomicLong(System.currentTimeMillis());
@AfterClass
public static void deleteTestDir() throws IOException {
@@ -267,9 +270,11 @@ public class TestFSDownload {
rsrcVis.put(rsrc, vis);
Path destPath = dirs.getLocalPathForWrite(
basedir.toString(), size, conf);
+ destPath = new Path (destPath,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsd =
new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
- destPath, rsrc, new Random(sharedSeed));
+ destPath, rsrc);
pending.put(rsrc, exec.submit(fsd));
try {
@@ -320,9 +325,11 @@ public class TestFSDownload {
rsrcVis.put(rsrc, vis);
Path destPath = dirs.getLocalPathForWrite(
basedir.toString(), sizes[i], conf);
+ destPath = new Path (destPath,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsd =
new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
- destPath, rsrc, new Random(sharedSeed));
+ destPath, rsrc);
pending.put(rsrc, exec.submit(fsd));
}
@@ -380,9 +387,10 @@ public class TestFSDownload {
Path p = new Path(basedir, "" + 1);
LocalResource rsrc = createTarFile(files, p, size, rand, vis);
Path destPath = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
+ destPath = new Path (destPath,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsd = new FSDownload(files,
- UserGroupInformation.getCurrentUser(), conf, destPath, rsrc,
- new Random(sharedSeed));
+ UserGroupInformation.getCurrentUser(), conf, destPath, rsrc);
pending.put(rsrc, exec.submit(fsd));
try {
@@ -437,9 +445,10 @@ public class TestFSDownload {
LocalResource rsrcjar = createJarFile(files, p, size, rand, vis);
rsrcjar.setType(LocalResourceType.PATTERN);
Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
+ destPathjar = new Path (destPathjar,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsdjar = new FSDownload(files,
- UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrcjar,
- new Random(sharedSeed));
+ UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrcjar);
pending.put(rsrcjar, exec.submit(fsdjar));
try {
@@ -493,9 +502,10 @@ public class TestFSDownload {
Path p = new Path(basedir, "" + 1);
LocalResource rsrczip = createZipFile(files, p, size, rand, vis);
Path destPathjar = dirs.getLocalPathForWrite(basedir.toString(), size, conf);
+ destPathjar = new Path (destPathjar,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsdzip = new FSDownload(files,
- UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrczip,
- new Random(sharedSeed));
+ UserGroupInformation.getCurrentUser(), conf, destPathjar, rsrczip);
pending.put(rsrczip, exec.submit(fsdzip));
try {
@@ -586,9 +596,11 @@ public class TestFSDownload {
rsrcVis.put(rsrc, vis);
Path destPath = dirs.getLocalPathForWrite(
basedir.toString(), conf);
+ destPath = new Path (destPath,
+ Long.toString(uniqueNumberGenerator.incrementAndGet()));
FSDownload fsd =
new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
- destPath, rsrc, new Random(sharedSeed));
+ destPath, rsrc);
pending.put(rsrc, exec.submit(fsd));
}
@@ -614,4 +626,38 @@ public class TestFSDownload {
}
-}
+
+ @Test(timeout = 1000)
+ public void testUniqueDestinationPath() throws Exception {
+ Configuration conf = new Configuration();
+ FileContext files = FileContext.getLocalFSFileContext(conf);
+ final Path basedir = files.makeQualified(new Path("target",
+ TestFSDownload.class.getSimpleName()));
+ files.mkdir(basedir, null, true);
+ conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
+
+ ExecutorService singleThreadedExec = Executors.newSingleThreadExecutor();
+
+ LocalDirAllocator dirs =
+ new LocalDirAllocator(TestFSDownload.class.getName());
+ Path destPath = dirs.getLocalPathForWrite(basedir.toString(), conf);
+ destPath =
+ new Path(destPath, Long.toString(uniqueNumberGenerator
+ .incrementAndGet()));
+ try {
+ Path p = new Path(basedir, "dir" + 0 + ".jar");
+ LocalResourceVisibility vis = LocalResourceVisibility.PRIVATE;
+ LocalResource rsrc = createJar(files, p, vis);
+ FSDownload fsd =
+ new FSDownload(files, UserGroupInformation.getCurrentUser(), conf,
+ destPath, rsrc);
+ Future<Path> rPath = singleThreadedExec.submit(fsd);
+ // Now FSDownload will not create a random directory to localize the
+ // resource. Therefore the final localizedPath for the resource should be
+ // destination directory (passed as an argument) + file name.
+ Assert.assertEquals(destPath, rPath.get().getParent());
+ } finally {
+ singleThreadedExec.shutdown();
+ }
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestYarnVersionInfo.java Fri Apr 12 23:05:28 2013
@@ -40,11 +40,11 @@ public class TestYarnVersionInfo extends
// make sure they aren't Unknown
assertTrue("getVersion returned Unknown", !YarnVersionInfo.getVersion().equals("Unknown"));
assertTrue("getUser returned Unknown", !YarnVersionInfo.getUser().equals("Unknown"));
- assertTrue("getUrl returned Unknown", !YarnVersionInfo.getUrl().equals("Unknown"));
assertTrue("getSrcChecksum returned Unknown", !YarnVersionInfo.getSrcChecksum().equals("Unknown"));
// these could be Unknown if the VersionInfo generated from code not in svn or git
// so just check that they return something
+ assertNotNull("getUrl returned null", YarnVersionInfo.getUrl());
assertNotNull("getRevision returned null", YarnVersionInfo.getRevision());
assertNotNull("getBranch returned null", YarnVersionInfo.getBranch());
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java Fri Apr 12 23:05:28 2013
@@ -42,4 +42,7 @@ public interface NodeHeartbeatResponse {
void addAllContainersToCleanup(List<ContainerId> containers);
void addAllApplicationsToCleanup(List<ApplicationId> applications);
+
+ long getNextHeartBeatInterval();
+ void setNextHeartBeatInterval(long nextHeartBeatInterval);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java Fri Apr 12 23:05:28 2013
@@ -271,6 +271,18 @@ public class NodeHeartbeatResponsePBImpl
builder.addAllApplicationsToCleanup(iterable);
}
+ @Override
+ public long getNextHeartBeatInterval() {
+ NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getNextHeartBeatInterval());
+ }
+
+ @Override
+ public void setNextHeartBeatInterval(long nextHeartBeatInterval) {
+ maybeInitBuilder();
+ builder.setNextHeartBeatInterval(nextHeartBeatInterval);
+ }
+
private ContainerIdPBImpl convertFromProtoFormat(ContainerIdProto p) {
return new ContainerIdPBImpl(p);
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java Fri Apr 12 23:05:28 2013
@@ -98,6 +98,7 @@ public class RegisterNodeManagerResponse
if (masterKey == null)
builder.clearMasterKey();
this.masterKey = masterKey;
+ rebuild = true;
}
@Override
@@ -114,9 +115,10 @@ public class RegisterNodeManagerResponse
maybeInitBuilder();
if (nodeAction == null) {
builder.clearNodeAction();
- return;
+ } else {
+ builder.setNodeAction(convertToProtoFormat(nodeAction));
}
- builder.setNodeAction(convertToProtoFormat(nodeAction));
+ rebuild = true;
}
private NodeAction convertFromProtoFormat(NodeActionProto p) {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeAction.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeAction.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeAction.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeAction.java Fri Apr 12 23:05:28 2013
@@ -24,5 +24,5 @@ package org.apache.hadoop.yarn.server.ap
*/
public enum NodeAction {
- NORMAL, REBOOT, SHUTDOWN
+ NORMAL, RESYNC, SHUTDOWN
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Fri Apr 12 23:05:28 2013
@@ -25,7 +25,7 @@ import "yarn_protos.proto";
enum NodeActionProto {
NORMAL = 0;
- REBOOT = 1;
+ RESYNC = 1;
SHUTDOWN = 2;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Fri Apr 12 23:05:28 2013
@@ -47,4 +47,5 @@ message NodeHeartbeatResponseProto {
optional NodeActionProto nodeAction = 3;
repeated ContainerIdProto containers_to_cleanup = 4;
repeated ApplicationIdProto applications_to_cleanup = 5;
+ optional int64 nextHeartBeatInterval = 6;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Fri Apr 12 23:05:28 2013
@@ -220,7 +220,7 @@ public class LinuxContainerExecutor exte
String containerIdStr = ConverterUtils.toString(containerId);
resourcesHandler.preExecute(containerId,
- container.getLaunchContext().getResource());
+ container.getResource());
String resourcesOptions = resourcesHandler.getResourcesOption(
containerId);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java Fri Apr 12 23:05:28 2013
@@ -307,7 +307,7 @@ public class LocalDirsHandlerService ext
URI uriPath = (new Path(paths[i])).toUri();
if (uriPath.getScheme() == null
|| uriPath.getScheme().equals(FILE_SCHEME)) {
- validPaths.add(uriPath.getPath());
+ validPaths.add(new Path(uriPath.getPath()).toString());
} else {
LOG.warn(paths[i] + " is not a valid path. Path should be with "
+ FILE_SCHEME + " scheme or without scheme");
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri Apr 12 23:05:28 2013
@@ -81,6 +81,7 @@ public class NodeManager extends Composi
private Context context;
private AsyncDispatcher dispatcher;
private ContainerManagerImpl containerManager;
+ private NodeStatusUpdater nodeStatusUpdater;
private static CompositeServiceShutdownHook nodeManagerShutdownHook;
private long waitForContainersOnShutdownMillis;
@@ -119,6 +120,10 @@ public class NodeManager extends Composi
return new DeletionService(exec);
}
+ protected NMContext createNMContext(NMContainerTokenSecretManager containerTokenSecretManager) {
+ return new NMContext(containerTokenSecretManager);
+ }
+
protected void doSecureLogin() throws IOException {
SecurityUtil.login(getConfig(), YarnConfiguration.NM_KEYTAB,
YarnConfiguration.NM_PRINCIPAL);
@@ -137,7 +142,7 @@ public class NodeManager extends Composi
containerTokenSecretManager = new NMContainerTokenSecretManager(conf);
}
- this.context = new NMContext(containerTokenSecretManager);
+ this.context = createNMContext(containerTokenSecretManager);
this.aclsManager = new ApplicationACLsManager(conf);
@@ -159,7 +164,7 @@ public class NodeManager extends Composi
addService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.getDiskHandler();
- NodeStatusUpdater nodeStatusUpdater =
+ nodeStatusUpdater =
createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
@@ -210,35 +215,67 @@ public class NodeManager extends Composi
if (isStopping.getAndSet(true)) {
return;
}
-
- cleanupContainers();
+
+ cleanupContainers(NodeManagerEventType.SHUTDOWN);
super.stop();
DefaultMetricsSystem.shutdown();
}
-
+
+ protected void cleanupContainersOnResync() {
+ //we do not want to block dispatcher thread here
+ new Thread() {
+ @Override
+ public void run() {
+ cleanupContainers(NodeManagerEventType.RESYNC);
+ ((NodeStatusUpdaterImpl) nodeStatusUpdater ).rebootNodeStatusUpdater();
+ }
+ }.start();
+ }
+
@SuppressWarnings("unchecked")
- protected void cleanupContainers() {
+ protected void cleanupContainers(NodeManagerEventType eventType) {
Map<ContainerId, Container> containers = context.getContainers();
if (containers.isEmpty()) {
return;
}
- LOG.info("Containers still running on shutdown: " + containers.keySet());
+ LOG.info("Containers still running on " + eventType + " : "
+ + containers.keySet());
- List<ContainerId> containerIds = new ArrayList<ContainerId>(containers.keySet());
+ List<ContainerId> containerIds =
+ new ArrayList<ContainerId>(containers.keySet());
dispatcher.getEventHandler().handle(
new CMgrCompletedContainersEvent(containerIds,
CMgrCompletedContainersEvent.Reason.ON_SHUTDOWN));
LOG.info("Waiting for containers to be killed");
- long waitStartTime = System.currentTimeMillis();
- while (!containers.isEmpty() &&
- System.currentTimeMillis() - waitStartTime < waitForContainersOnShutdownMillis) {
- try {
- Thread.sleep(1000);
- } catch (InterruptedException ex) {
- LOG.warn("Interrupted while sleeping on container kill", ex);
+ switch (eventType) {
+ case SHUTDOWN:
+ long waitStartTime = System.currentTimeMillis();
+ while (!containers.isEmpty()
+ && System.currentTimeMillis() - waitStartTime < waitForContainersOnShutdownMillis) {
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ex) {
+ LOG.warn("Interrupted while sleeping on container kill on shutdown",
+ ex);
+ }
+ }
+ break;
+ case RESYNC:
+ while (!containers.isEmpty()) {
+ try {
+ Thread.sleep(1000);
+ //to remove done containers from the map
+ nodeStatusUpdater.getNodeStatusAndUpdateContainersInContext();
+ } catch (InterruptedException ex) {
+ LOG.warn("Interrupted while sleeping on container kill on resync",
+ ex);
+ }
}
+ break;
+ default:
+ LOG.warn("Invalid eventType: " + eventType);
}
// All containers killed
@@ -338,9 +375,8 @@ public class NodeManager extends Composi
case SHUTDOWN:
stop();
break;
- case REBOOT:
- stop();
- reboot();
+ case RESYNC:
+ cleanupContainersOnResync();
break;
default:
LOG.warn("Invalid shutdown event " + event.getType() + ". Ignoring.");
@@ -357,6 +393,11 @@ public class NodeManager extends Composi
return containerManager;
}
+ //For testing
+ Dispatcher getNMDispatcher(){
+ return dispatcher;
+ }
+
@VisibleForTesting
Context getNMContext() {
return this.context;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerEventType.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerEventType.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerEventType.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManagerEventType.java Fri Apr 12 23:05:28 2013
@@ -18,5 +18,5 @@
package org.apache.hadoop.yarn.server.nodemanager;
public enum NodeManagerEventType {
- SHUTDOWN, REBOOT
+ SHUTDOWN, RESYNC
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java Fri Apr 12 23:05:28 2013
@@ -18,9 +18,11 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.service.Service;
public interface NodeStatusUpdater extends Service {
void sendOutofBandHeartBeat();
+ NodeStatus getNodeStatusAndUpdateContainersInContext();
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Fri Apr 12 23:05:28 2013
@@ -60,6 +60,8 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.service.AbstractService;
+import com.google.common.annotations.VisibleForTesting;
+
public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdater {
@@ -71,7 +73,7 @@ public class NodeStatusUpdaterImpl exten
private final Dispatcher dispatcher;
private NodeId nodeId;
- private long heartBeatInterval;
+ private long nextHeartBeatInterval;
private ResourceTracker resourceTracker;
private InetSocketAddress rmAddress;
private Resource totalResource;
@@ -87,6 +89,12 @@ public class NodeStatusUpdaterImpl exten
private final NodeHealthCheckerService healthChecker;
private final NodeManagerMetrics metrics;
+ private long rmConnectWaitMS;
+ private long rmConnectionRetryIntervalMS;
+ private boolean waitForEver;
+
+ private Runnable statusUpdaterRunnable;
+ private Thread statusUpdater;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
@@ -103,9 +111,7 @@ public class NodeStatusUpdaterImpl exten
YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS,
YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT);
- this.heartBeatInterval =
- conf.getLong(YarnConfiguration.NM_TO_RM_HEARTBEAT_INTERVAL_MS,
- YarnConfiguration.DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS);
+
int memoryMb =
conf.getInt(
YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
@@ -134,8 +140,8 @@ public class NodeStatusUpdaterImpl exten
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
LOG.info("Initialized nodemanager for " + nodeId + ":" +
- " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
- " physical-cores=" + cpuCores + " virtual-cores=" + virtualCores);
+ " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
+ " physical-cores=" + cpuCores + " virtual-cores=" + virtualCores);
super.init(conf);
}
@@ -168,6 +174,22 @@ public class NodeStatusUpdaterImpl exten
this.isStopped = true;
super.stop();
}
+
+ protected void rebootNodeStatusUpdater() {
+ // Interrupt the updater.
+ this.isStopped = true;
+
+ try {
+ statusUpdater.join();
+ registerWithRM();
+ statusUpdater = new Thread(statusUpdaterRunnable, "Node Status Updater");
+ this.isStopped = false;
+ statusUpdater.start();
+ LOG.info("NodeStatusUpdater thread is reRegistered and restarted");
+ } catch (Exception e) {
+ throw new AvroRuntimeException(e);
+ }
+ }
private boolean isSecurityEnabled() {
return UserGroupInformation.isSecurityEnabled();
@@ -187,14 +209,15 @@ public class NodeStatusUpdaterImpl exten
conf);
}
- private void registerWithRM() throws YarnRemoteException {
+ @VisibleForTesting
+ protected void registerWithRM() throws YarnRemoteException {
Configuration conf = getConfig();
- long rmConnectWaitMS =
+ rmConnectWaitMS =
conf.getInt(
YarnConfiguration.RESOURCEMANAGER_CONNECT_WAIT_SECS,
YarnConfiguration.DEFAULT_RESOURCEMANAGER_CONNECT_WAIT_SECS)
* 1000;
- long rmConnectionRetryIntervalMS =
+ rmConnectionRetryIntervalMS =
conf.getLong(
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_SECS,
YarnConfiguration
@@ -207,7 +230,7 @@ public class NodeStatusUpdaterImpl exten
" should not be negative.");
}
- boolean waitForEver = (rmConnectWaitMS == -1000);
+ waitForEver = (rmConnectWaitMS == -1000);
if(! waitForEver) {
if(rmConnectWaitMS < 0) {
@@ -311,7 +334,7 @@ public class NodeStatusUpdaterImpl exten
return appList;
}
- private NodeStatus getNodeStatus() {
+ public NodeStatus getNodeStatusAndUpdateContainersInContext() {
NodeStatus nodeStatus = recordFactory.newRecordInstance(NodeStatus.class);
nodeStatus.setNodeId(this.nodeId);
@@ -386,7 +409,7 @@ public class NodeStatusUpdaterImpl exten
protected void startStatusUpdater() {
- new Thread("Node Status Updater") {
+ statusUpdaterRunnable = new Runnable() {
@Override
@SuppressWarnings("unchecked")
public void run() {
@@ -394,10 +417,10 @@ public class NodeStatusUpdaterImpl exten
while (!isStopped) {
// Send heartbeat
try {
- synchronized (heartbeatMonitor) {
- heartbeatMonitor.wait(heartBeatInterval);
- }
- NodeStatus nodeStatus = getNodeStatus();
+ NodeHeartbeatResponse response = null;
+ int rmRetryCount = 0;
+ long waitStartTime = System.currentTimeMillis();
+ NodeStatus nodeStatus = getNodeStatusAndUpdateContainersInContext();
nodeStatus.setResponseId(lastHeartBeatID);
NodeHeartbeatRequest request = recordFactory
@@ -407,9 +430,33 @@ public class NodeStatusUpdaterImpl exten
request.setLastKnownMasterKey(NodeStatusUpdaterImpl.this.context
.getContainerTokenSecretManager().getCurrentKey());
}
- NodeHeartbeatResponse response =
- resourceTracker.nodeHeartbeat(request);
-
+ while (!isStopped) {
+ try {
+ rmRetryCount++;
+ response = resourceTracker.nodeHeartbeat(request);
+ break;
+ } catch (Throwable e) {
+ LOG.warn("Trying to heartbeat to ResourceManager, "
+ + "current no. of failed attempts is " + rmRetryCount);
+ if(System.currentTimeMillis() - waitStartTime < rmConnectWaitMS
+ || waitForEver) {
+ try {
+ LOG.info("Sleeping for " + rmConnectionRetryIntervalMS/1000
+ + " seconds before next heartbeat to RM");
+ Thread.sleep(rmConnectionRetryIntervalMS);
+ } catch(InterruptedException ex) {
+ //done nothing
+ }
+ } else {
+ String errorMessage = "Failed to heartbeat to RM, " +
+ "no. of failed attempts is "+rmRetryCount;
+ LOG.error(errorMessage,e);
+ throw new YarnException(errorMessage,e);
+ }
+ }
+ }
+ //get next heartbeat interval from response
+ nextHeartBeatInterval = response.getNextHeartBeatInterval();
// See if the master-key has rolled over
if (isSecurityEnabled()) {
MasterKey updatedMasterKey = response.getMasterKey();
@@ -423,16 +470,16 @@ public class NodeStatusUpdaterImpl exten
if (response.getNodeAction() == NodeAction.SHUTDOWN) {
LOG
.info("Recieved SHUTDOWN signal from Resourcemanager as part of heartbeat," +
- " hence shutting down.");
+ " hence shutting down.");
dispatcher.getEventHandler().handle(
new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
break;
}
- if (response.getNodeAction() == NodeAction.REBOOT) {
+ if (response.getNodeAction() == NodeAction.RESYNC) {
LOG.info("Node is out of sync with ResourceManager,"
+ " hence rebooting.");
dispatcher.getEventHandler().handle(
- new NodeManagerEvent(NodeManagerEventType.REBOOT));
+ new NodeManagerEvent(NodeManagerEventType.RESYNC));
break;
}
@@ -452,13 +499,32 @@ public class NodeStatusUpdaterImpl exten
dispatcher.getEventHandler().handle(
new CMgrCompletedAppsEvent(appsToCleanup));
}
+ } catch (YarnException e) {
+ //catch and throw the exception if tried MAX wait time to connect RM
+ dispatcher.getEventHandler().handle(
+ new NodeManagerEvent(NodeManagerEventType.SHUTDOWN));
+ throw e;
} catch (Throwable e) {
// TODO Better error handling. Thread can die with the rest of the
// NM still running.
LOG.error("Caught exception in status-updater", e);
+ } finally {
+ synchronized (heartbeatMonitor) {
+ nextHeartBeatInterval = nextHeartBeatInterval <= 0 ?
+ YarnConfiguration.DEFAULT_RM_NM_HEARTBEAT_INTERVAL_MS :
+ nextHeartBeatInterval;
+ try {
+ heartbeatMonitor.wait(nextHeartBeatInterval);
+ } catch (InterruptedException e) {
+ // Do Nothing
+ }
+ }
}
}
}
- }.start();
+ };
+ statusUpdater =
+ new Thread(statusUpdaterRunnable, "Node Status Updater");
+ statusUpdater.start();
}
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java Fri Apr 12 23:05:28 2013
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb.client;
+import java.io.Closeable;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -35,7 +36,8 @@ import org.apache.hadoop.yarn.server.nod
import com.google.protobuf.ServiceException;
-public class LocalizationProtocolPBClientImpl implements LocalizationProtocol {
+public class LocalizationProtocolPBClientImpl implements LocalizationProtocol,
+ Closeable {
private LocalizationProtocolPB proxy;
@@ -44,7 +46,14 @@ public class LocalizationProtocolPBClien
proxy = (LocalizationProtocolPB)RPC.getProxy(
LocalizationProtocolPB.class, clientVersion, addr, conf);
}
-
+
+ @Override
+ public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+
@Override
public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status)
throws YarnRemoteException {
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java Fri Apr 12 23:05:28 2013
@@ -18,18 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
import java.util.List;
-
-import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.server.nodemanager.api.*;
public interface LocalizerHeartbeatResponse {
- public LocalizerAction getLocalizerAction();
- public List<LocalResource> getAllResources();
- public LocalResource getLocalResource(int i);
+ public LocalizerAction getLocalizerAction();
public void setLocalizerAction(LocalizerAction action);
- public void addAllResources(List<LocalResource> resources);
- public void addResource(LocalResource resource);
- public void removeResource(int index);
- public void clearResources();
-}
+ public List<ResourceLocalizationSpec> getResourceSpecs();
+ public void setResourceSpecs(List<ResourceLocalizationSpec> rsrcs);
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java Fri Apr 12 23:05:28 2013
@@ -21,13 +21,14 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import org.apache.hadoop.yarn.api.records.LocalResource;
+
import org.apache.hadoop.yarn.api.records.ProtoBase;
-import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerActionProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerHeartbeatResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerHeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceLocalizationSpecProto;
+import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
+import org.apache.hadoop.yarn.server.nodemanager.api.impl.pb.ResourceLocalizationSpecPBImpl;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerAction;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
@@ -40,13 +41,14 @@ public class LocalizerHeartbeatResponseP
LocalizerHeartbeatResponseProto.Builder builder = null;
boolean viaProto = false;
- private List<LocalResource> resources;
+ private List<ResourceLocalizationSpec> resourceSpecs;
public LocalizerHeartbeatResponsePBImpl() {
builder = LocalizerHeartbeatResponseProto.newBuilder();
}
- public LocalizerHeartbeatResponsePBImpl(LocalizerHeartbeatResponseProto proto) {
+ public LocalizerHeartbeatResponsePBImpl(
+ LocalizerHeartbeatResponseProto proto) {
this.proto = proto;
viaProto = true;
}
@@ -59,7 +61,7 @@ public class LocalizerHeartbeatResponseP
}
private void mergeLocalToBuilder() {
- if (resources != null) {
+ if (resourceSpecs != null) {
addResourcesToProto();
}
}
@@ -79,6 +81,7 @@ public class LocalizerHeartbeatResponseP
viaProto = false;
}
+ @Override
public LocalizerAction getLocalizerAction() {
LocalizerHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
if (!p.hasAction()) {
@@ -87,14 +90,10 @@ public class LocalizerHeartbeatResponseP
return convertFromProtoFormat(p.getAction());
}
- public List<LocalResource> getAllResources() {
- initResources();
- return this.resources;
- }
-
- public LocalResource getLocalResource(int i) {
+ @Override
+ public List<ResourceLocalizationSpec> getResourceSpecs() {
initResources();
- return this.resources.get(i);
+ return this.resourceSpecs;
}
public void setLocalizerAction(LocalizerAction action) {
@@ -106,31 +105,39 @@ public class LocalizerHeartbeatResponseP
builder.setAction(convertToProtoFormat(action));
}
+ public void setResourceSpecs(List<ResourceLocalizationSpec> rsrcs) {
+ maybeInitBuilder();
+ if (rsrcs == null) {
+ builder.clearResources();
+ return;
+ }
+ this.resourceSpecs = rsrcs;
+ }
+
private void initResources() {
- if (this.resources != null) {
+ if (this.resourceSpecs != null) {
return;
}
LocalizerHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<LocalResourceProto> list = p.getResourcesList();
- this.resources = new ArrayList<LocalResource>();
-
- for (LocalResourceProto c : list) {
- this.resources.add(convertFromProtoFormat(c));
+ List<ResourceLocalizationSpecProto> list = p.getResourcesList();
+ this.resourceSpecs = new ArrayList<ResourceLocalizationSpec>();
+ for (ResourceLocalizationSpecProto c : list) {
+ this.resourceSpecs.add(convertFromProtoFormat(c));
}
}
private void addResourcesToProto() {
maybeInitBuilder();
builder.clearResources();
- if (this.resources == null)
+ if (this.resourceSpecs == null)
return;
- Iterable<LocalResourceProto> iterable =
- new Iterable<LocalResourceProto>() {
+ Iterable<ResourceLocalizationSpecProto> iterable =
+ new Iterable<ResourceLocalizationSpecProto>() {
@Override
- public Iterator<LocalResourceProto> iterator() {
- return new Iterator<LocalResourceProto>() {
+ public Iterator<ResourceLocalizationSpecProto> iterator() {
+ return new Iterator<ResourceLocalizationSpecProto>() {
- Iterator<LocalResource> iter = resources.iterator();
+ Iterator<ResourceLocalizationSpec> iter = resourceSpecs.iterator();
@Override
public boolean hasNext() {
@@ -138,8 +145,10 @@ public class LocalizerHeartbeatResponseP
}
@Override
- public LocalResourceProto next() {
- return convertToProtoFormat(iter.next());
+ public ResourceLocalizationSpecProto next() {
+ ResourceLocalizationSpec resource = iter.next();
+
+ return ((ResourceLocalizationSpecPBImpl)resource).getProto();
}
@Override
@@ -154,34 +163,10 @@ public class LocalizerHeartbeatResponseP
builder.addAllResources(iterable);
}
- public void addAllResources(List<LocalResource> resources) {
- if (resources == null)
- return;
- initResources();
- this.resources.addAll(resources);
- }
- public void addResource(LocalResource resource) {
- initResources();
- this.resources.add(resource);
- }
-
- public void removeResource(int index) {
- initResources();
- this.resources.remove(index);
- }
-
- public void clearResources() {
- initResources();
- this.resources.clear();
- }
-
- private LocalResource convertFromProtoFormat(LocalResourceProto p) {
- return new LocalResourcePBImpl(p);
- }
-
- private LocalResourceProto convertToProtoFormat(LocalResource s) {
- return ((LocalResourcePBImpl)s).getProto();
+ private ResourceLocalizationSpec convertFromProtoFormat(
+ ResourceLocalizationSpecProto p) {
+ return new ResourceLocalizationSpecPBImpl(p);
}
private LocalizerActionProto convertToProtoFormat(LocalizerAction a) {
@@ -191,5 +176,4 @@ public class LocalizerHeartbeatResponseP
private LocalizerAction convertFromProtoFormat(LocalizerActionProto a) {
return LocalizerAction.valueOf(a.name());
}
-
-}
+}
\ No newline at end of file
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Fri Apr 12 23:05:28 2013
@@ -308,7 +308,9 @@ public class ContainerManagerImpl extend
* @throws YarnRemoteException
*/
private void authorizeRequest(String containerIDStr,
- ContainerLaunchContext launchContext, UserGroupInformation remoteUgi)
+ ContainerLaunchContext launchContext,
+ org.apache.hadoop.yarn.api.records.Container container,
+ UserGroupInformation remoteUgi)
throws YarnRemoteException {
if (!UserGroupInformation.isSecurityEnabled()) {
@@ -369,10 +371,10 @@ public class ContainerManagerImpl extend
}
Resource resource = tokenId.getResource();
- if (!resource.equals(launchContext.getResource())) {
+ if (!resource.equals(container.getResource())) {
unauthorized = true;
messageBuilder.append("\nExpected resource " + resource
- + " but found " + launchContext.getResource());
+ + " but found " + container.getResource());
}
}
}
@@ -392,12 +394,13 @@ public class ContainerManagerImpl extend
public StartContainerResponse startContainer(StartContainerRequest request)
throws YarnRemoteException {
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
-
- ContainerId containerID = launchContext.getContainerId();
+ org.apache.hadoop.yarn.api.records.Container lauchContainer =
+ request.getContainer();
+ ContainerId containerID = lauchContainer.getId();
String containerIDStr = containerID.toString();
UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
- authorizeRequest(containerIDStr, launchContext, remoteUgi);
+ authorizeRequest(containerIDStr, launchContext, lauchContainer, remoteUgi);
LOG.info("Start request for " + containerIDStr + " by user "
+ launchContext.getUser());
@@ -424,7 +427,7 @@ public class ContainerManagerImpl extend
// //////////// End of parsing credentials
Container container = new ContainerImpl(getConfig(), this.dispatcher,
- launchContext, credentials, metrics);
+ launchContext, lauchContainer, credentials, metrics);
ApplicationId applicationID =
containerID.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerID, container) != null) {
@@ -469,7 +472,7 @@ public class ContainerManagerImpl extend
// TODO launchedContainer misplaced -> doesn't necessarily mean a container
// launch. A finished Application will not launch containers.
metrics.launchedContainer();
- metrics.allocateContainer(launchContext.getResource());
+ metrics.allocateContainer(lauchContainer.getResource());
return response;
}
@@ -487,7 +490,7 @@ public class ContainerManagerImpl extend
// TODO: Only the container's owner can kill containers today.
UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
- authorizeRequest(containerIDStr, null, remoteUgi);
+ authorizeRequest(containerIDStr, null, null, remoteUgi);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -529,7 +532,7 @@ public class ContainerManagerImpl extend
// TODO: Only the container's owner can get containers' status today.
UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
- authorizeRequest(containerIDStr, null, remoteUgi);
+ authorizeRequest(containerIDStr, null, null, remoteUgi);
LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java Fri Apr 12 23:05:28 2013
@@ -25,6 +25,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
public interface Container extends EventHandler<ContainerEvent> {
@@ -44,4 +45,6 @@ public interface Container extends Event
ContainerStatus cloneAndGetContainerStatus();
String toString();
+
+ Resource getResource();
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java Fri Apr 12 23:05:28 2013
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -76,6 +77,7 @@ public class ContainerImpl implements Co
private final Credentials credentials;
private final NodeManagerMetrics metrics;
private final ContainerLaunchContext launchContext;
+ private final org.apache.hadoop.yarn.api.records.Container container;
private int exitCode = YarnConfiguration.INVALID_CONTAINER_EXIT_STATUS;
private final StringBuilder diagnostics;
@@ -96,12 +98,13 @@ public class ContainerImpl implements Co
new ArrayList<LocalResourceRequest>();
public ContainerImpl(Configuration conf,
- Dispatcher dispatcher,
- ContainerLaunchContext launchContext, Credentials creds,
- NodeManagerMetrics metrics) {
+ Dispatcher dispatcher, ContainerLaunchContext launchContext,
+ org.apache.hadoop.yarn.api.records.Container container,
+ Credentials creds, NodeManagerMetrics metrics) {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.launchContext = launchContext;
+ this.container = container;
this.diagnostics = new StringBuilder();
this.credentials = creds;
this.metrics = metrics;
@@ -312,7 +315,7 @@ public class ContainerImpl implements Co
public ContainerId getContainerID() {
this.readLock.lock();
try {
- return this.launchContext.getContainerId();
+ return this.container.getId();
} finally {
this.readLock.unlock();
}
@@ -373,50 +376,61 @@ public class ContainerImpl implements Co
public ContainerStatus cloneAndGetContainerStatus() {
this.readLock.lock();
try {
- return BuilderUtils.newContainerStatus(this.getContainerID(),
+ return BuilderUtils.newContainerStatus(this.container.getId(),
getCurrentState(), diagnostics.toString(), exitCode);
} finally {
this.readLock.unlock();
}
}
+ @Override
+ public Resource getResource() {
+ this.readLock.lock();
+ try {
+ return this.container.getResource();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
@SuppressWarnings({"fallthrough", "unchecked"})
private void finished() {
+ ContainerId containerID = this.container.getId();
+ String user = this.launchContext.getUser();
switch (getContainerState()) {
case EXITED_WITH_SUCCESS:
metrics.endRunningContainer();
metrics.completedContainer();
- NMAuditLogger.logSuccess(getUser(),
+ NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_SUCCESS_CONTAINER, "ContainerImpl",
- getContainerID().getApplicationAttemptId().getApplicationId(),
- getContainerID());
+ containerID.getApplicationAttemptId()
+ .getApplicationId(), containerID);
break;
case EXITED_WITH_FAILURE:
metrics.endRunningContainer();
// fall through
case LOCALIZATION_FAILED:
metrics.failedContainer();
- NMAuditLogger.logFailure(getUser(),
+ NMAuditLogger.logFailure(user,
AuditConstants.FINISH_FAILED_CONTAINER, "ContainerImpl",
"Container failed with state: " + getContainerState(),
- getContainerID().getApplicationAttemptId().getApplicationId(),
- getContainerID());
+ containerID.getApplicationAttemptId()
+ .getApplicationId(), containerID);
break;
case CONTAINER_CLEANEDUP_AFTER_KILL:
metrics.endRunningContainer();
// fall through
case NEW:
metrics.killedContainer();
- NMAuditLogger.logSuccess(getUser(),
+ NMAuditLogger.logSuccess(user,
AuditConstants.FINISH_KILLED_CONTAINER, "ContainerImpl",
- getContainerID().getApplicationAttemptId().getApplicationId(),
- getContainerID());
+ containerID.getApplicationAttemptId().getApplicationId(),
+ containerID);
}
- metrics.releaseContainer(getLaunchContext().getResource());
+ metrics.releaseContainer(this.container.getResource());
// Inform the application
- ContainerId containerID = getContainerID();
@SuppressWarnings("rawtypes")
EventHandler eventHandler = dispatcher.getEventHandler();
eventHandler.handle(new ApplicationContainerFinishedEvent(containerID));
@@ -475,7 +489,7 @@ public class ContainerImpl implements Co
@Override
public ContainerState transition(ContainerImpl container,
ContainerEvent event) {
- final ContainerLaunchContext ctxt = container.getLaunchContext();
+ final ContainerLaunchContext ctxt = container.launchContext;
container.metrics.initingContainer();
// Inform the AuxServices about the opaque serviceData
@@ -486,9 +500,9 @@ public class ContainerImpl implements Co
for (Map.Entry<String,ByteBuffer> service : csd.entrySet()) {
container.dispatcher.getEventHandler().handle(
new AuxServicesEvent(AuxServicesEventType.APPLICATION_INIT,
- ctxt.getUser(),
- ctxt.getContainerId().getApplicationAttemptId().getApplicationId(),
- service.getKey().toString(), service.getValue()));
+ ctxt.getUser(), container.container.getId()
+ .getApplicationAttemptId().getApplicationId(),
+ service.getKey().toString(), service.getValue()));
}
}
@@ -571,7 +585,7 @@ public class ContainerImpl implements Co
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.getContainerID());
+ " for container " + container.container.getId());
assert false;
// fail container?
return ContainerState.LOCALIZING;
@@ -599,14 +613,14 @@ public class ContainerImpl implements Co
// Inform the ContainersMonitor to start monitoring the container's
// resource usage.
long pmemBytes =
- container.getLaunchContext().getResource().getMemory() * 1024 * 1024L;
+ container.container.getResource().getMemory() * 1024 * 1024L;
float pmemRatio = container.daemonConf.getFloat(
YarnConfiguration.NM_VMEM_PMEM_RATIO,
YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
long vmemBytes = (long) (pmemRatio * pmemBytes);
container.dispatcher.getEventHandler().handle(
- new ContainerStartMonitoringEvent(container.getContainerID(),
+ new ContainerStartMonitoringEvent(container.container.getId(),
vmemBytes, pmemBytes));
container.metrics.runningContainer();
}
@@ -740,7 +754,7 @@ public class ContainerImpl implements Co
container.pendingResources.remove(rsrcEvent.getResource());
if (null == syms) {
LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.getContainerID());
+ " for container " + container.container.getId());
assert false;
// fail container?
return;
@@ -845,10 +859,9 @@ public class ContainerImpl implements Co
public String toString() {
this.readLock.lock();
try {
- return ConverterUtils.toString(launchContext.getContainerId());
+ return ConverterUtils.toString(container.getId());
} finally {
this.readLock.unlock();
}
}
-
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java Fri Apr 12 23:05:28 2013
@@ -249,9 +249,8 @@ public class ContainerLaunch implements
} catch (Throwable e) {
LOG.warn("Failed to launch container.", e);
dispatcher.getEventHandler().handle(new ContainerExitEvent(
- launchContext.getContainerId(),
- ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
- e.getMessage()));
+ containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
+ e.getMessage()));
return ret;
} finally {
completed.set(true);
@@ -267,7 +266,7 @@ public class ContainerLaunch implements
// If the process was killed, Send container_cleanedup_after_kill and
// just break out of this method.
dispatcher.getEventHandler().handle(
- new ContainerExitEvent(launchContext.getContainerId(),
+ new ContainerExitEvent(containerID,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST, ret,
"Container exited with a non-zero exit code " + ret));
return ret;
@@ -276,15 +275,15 @@ public class ContainerLaunch implements
if (ret != 0) {
LOG.warn("Container exited with a non-zero exit code " + ret);
this.dispatcher.getEventHandler().handle(new ContainerExitEvent(
- launchContext.getContainerId(),
- ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
- "Container exited with a non-zero exit code " + ret));
+ containerID,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret,
+ "Container exited with a non-zero exit code " + ret));
return ret;
}
LOG.info("Container " + containerIdStr + " succeeded ");
dispatcher.getEventHandler().handle(
- new ContainerEvent(launchContext.getContainerId(),
+ new ContainerEvent(containerID,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
return 0;
}
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java Fri Apr 12 23:05:28 2013
@@ -51,6 +51,7 @@ import org.apache.hadoop.security.Creden
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -59,6 +60,7 @@ import org.apache.hadoop.yarn.factory.pr
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
+import org.apache.hadoop.yarn.server.nodemanager.api.ResourceLocalizationSpec;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
@@ -89,8 +91,6 @@ public class ContainerLocalizer {
private final String localizerId;
private final FileContext lfs;
private final Configuration conf;
- private final LocalDirAllocator appDirs;
- private final LocalDirAllocator userDirs;
private final RecordFactory recordFactory;
private final Map<LocalResource,Future<Path>> pendingResources;
private final String appCacheDirContextName;
@@ -112,8 +112,6 @@ public class ContainerLocalizer {
this.recordFactory = recordFactory;
this.conf = new Configuration();
this.appCacheDirContextName = String.format(APPCACHE_CTXT_FMT, appId);
- this.appDirs = new LocalDirAllocator(appCacheDirContextName);
- this.userDirs = new LocalDirAllocator(String.format(USERCACHE_CTXT_FMT, user));
this.pendingResources = new HashMap<LocalResource,Future<Path>>();
}
@@ -197,10 +195,10 @@ public class ContainerLocalizer {
return new ExecutorCompletionService<Path>(exec);
}
- Callable<Path> download(LocalDirAllocator lda, LocalResource rsrc,
+ Callable<Path> download(Path path, LocalResource rsrc,
UserGroupInformation ugi) throws IOException {
- Path destPath = lda.getLocalPathForWrite(".", getEstimatedSize(rsrc), conf);
- return new FSDownload(lfs, ugi, conf, destPath, rsrc, new Random());
+ DiskChecker.checkDir(new File(path.toUri().getRawPath()));
+ return new FSDownload(lfs, ugi, conf, path, rsrc);
}
static long getEstimatedSize(LocalResource rsrc) {
@@ -238,25 +236,12 @@ public class ContainerLocalizer {
LocalizerHeartbeatResponse response = nodemanager.heartbeat(status);
switch (response.getLocalizerAction()) {
case LIVE:
- List<LocalResource> newResources = response.getAllResources();
- for (LocalResource r : newResources) {
- if (!pendingResources.containsKey(r)) {
- final LocalDirAllocator lda;
- switch (r.getVisibility()) {
- default:
- LOG.warn("Unknown visibility: " + r.getVisibility()
- + ", Using userDirs");
- //Falling back to userDirs for unknown visibility.
- case PUBLIC:
- case PRIVATE:
- lda = userDirs;
- break;
- case APPLICATION:
- lda = appDirs;
- break;
- }
- // TODO: Synchronization??
- pendingResources.put(r, cs.submit(download(lda, r, ugi)));
+ List<ResourceLocalizationSpec> newRsrcs = response.getResourceSpecs();
+ for (ResourceLocalizationSpec newRsrc : newRsrcs) {
+ if (!pendingResources.containsKey(newRsrc.getResource())) {
+ pendingResources.put(newRsrc.getResource(), cs.submit(download(
+ new Path(newRsrc.getDestinationDirectory().getFile()),
+ newRsrc.getResource(), ugi)));
}
}
break;
Modified: hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java?rev=1467511&r1=1467510&r2=1467511&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java (original)
+++ hadoop/common/branches/HDFS-347/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java Fri Apr 12 23:05:28 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -35,6 +36,9 @@ interface LocalResourcesTracker
boolean remove(LocalizedResource req, DeletionService delService);
+ Path getPathForLocalization(LocalResourceRequest req, Path localDirPath);
+
String getUser();
+ long nextUniqueNumber();
}