You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by ma...@apache.org on 2011/05/26 23:16:08 UTC
svn commit: r1128091 - in /hadoop/mapreduce/branches/MR-279: ./
src/test/mapred/org/apache/hadoop/mapred/jobcontrol/
yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/
Author: mahadev
Date: Thu May 26 21:16:08 2011
New Revision: 1128091
URL: http://svn.apache.org/viewvc?rev=1128091&view=rev
Log:
Fix for public dist cache from non mapred hdfs. (mahadev & ddas)
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1128091&r1=1128090&r2=1128091&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Thu May 26 21:16:08 2011
@@ -4,6 +4,8 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Fix for public dist cache from non mapred hdfs. (mahadev & ddas)
+
Fixed an NPE during handling of unnecessary reservations in CS. (acmurthy)
MAPREDUCE-2532. Metrics for NodeManager (Luke Lu via mahadev)
Modified: hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java?rev=1128091&r1=1128090&r2=1128091&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java (original)
+++ hadoop/mapreduce/branches/MR-279/src/test/mapred/org/apache/hadoop/mapred/jobcontrol/TestJobControl.java Thu May 26 21:16:08 2011
@@ -18,12 +18,15 @@
package org.apache.hadoop.mapred.jobcontrol;
+import static org.mockito.Mockito.*;
+
import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobID;
/**
* This class performs unit test for Job/JobControl classes.
@@ -191,7 +194,21 @@ public class TestJobControl extends juni
public void testJobControl() throws Exception {
doJobControlTest();
}
-
+
+ public void testGetAssignedJobId() throws Exception {
+ JobConf jc = new JobConf();
+ Job j = new Job(jc);
+ //Just make sure no exception is thrown
+ assertNull(j.getAssignedJobID());
+ org.apache.hadoop.mapreduce.Job mockjob = mock(org.apache.hadoop.mapreduce.Job.class);
+ org.apache.hadoop.mapreduce.JobID jid = new org.apache.hadoop.mapreduce.JobID("test",0);
+ when(mockjob.getJobID()).thenReturn(jid);
+ j.setJob(mockjob);
+ JobID expected = new JobID("test",0);
+ assertEquals(expected, j.getAssignedJobID());
+ verify(mockjob).getJobID();
+ }
+
public static void main(String[] args) {
TestJobControl test = new TestJobControl();
try {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java?rev=1128091&r1=1128090&r2=1128091&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java Thu May 26 21:16:08 2011
@@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.util.Conve
public class FSDownload implements Callable<Path> {
private static final Log LOG = LogFactory.getLog(FSDownload.class);
-
+
private Random rand;
private FileContext files;
private final UserGroupInformation userUgi;
@@ -73,15 +73,16 @@ public class FSDownload implements Calla
}
private Path copy(Path sCopy, Path dstdir) throws IOException {
+ FileSystem sourceFs = sCopy.getFileSystem(conf);
Path dCopy = new Path(dstdir, sCopy.getName() + ".tmp");
- FileSystem fs = FileSystem.get(new Configuration());
- FileStatus sStat = fs.getFileStatus(sCopy);
+ FileStatus sStat = sourceFs.getFileStatus(sCopy);
if (sStat.getModificationTime() != resource.getTimestamp()) {
throw new IOException("Resource " + sCopy +
" changed on src filesystem (expected " + resource.getTimestamp() +
", was " + sStat.getModificationTime());
}
- fs.copyToLocalFile(sCopy, dCopy);
+
+ sourceFs.copyToLocalFile(sCopy, dCopy);
return dCopy;
}