You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/04/23 21:46:16 UTC
svn commit: r1329405 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobcl...
Author: bobby
Date: Mon Apr 23 19:46:16 2012
New Revision: 1329405
URL: http://svn.apache.org/viewvc?rev=1329405&view=rev
Log:
svn merge -c 1329403. FIXES: MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1329405&r1=1329404&r2=1329405&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Mon Apr 23 19:46:16 2012
@@ -286,6 +286,7 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4190. Improve web UI for task attempts userlog link (Tom Graves
via bobby)
+ MAPREDUCE-4133. MR over viewfs is broken (John George via bobby)
Release 0.23.2 - UNRELEASED
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1329405&r1=1329404&r2=1329405&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Mon Apr 23 19:46:16 2012
@@ -827,7 +827,7 @@ public class JobImpl implements org.apac
|| (sysMaxReduces == 1);
long sysMaxBytes = conf.getLong(MRJobConfig.JOB_UBERTASK_MAXBYTES,
- fs.getDefaultBlockSize()); // FIXME: this is wrong; get FS from
+ fs.getDefaultBlockSize(this.remoteJobSubmitDir)); // FIXME: this is wrong; get FS from
// [File?]InputFormat and default block size
// from that
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java?rev=1329405&r1=1329404&r2=1329405&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/DistributedFSCheck.java Mon Apr 23 19:46:16 2012
@@ -128,7 +128,7 @@ public class DistributedFSCheck extends
if (rootStatus.isFile()) {
nrFiles++;
// For a regular file generate <fName,offset> pairs
- long blockSize = fs.getDefaultBlockSize();
+ long blockSize = fs.getDefaultBlockSize(rootFile);
long fileLength = rootStatus.getLen();
for(long offset = 0; offset < fileLength; offset += blockSize)
writer.append(new Text(rootFile.toString()), new LongWritable(offset));
@@ -160,15 +160,16 @@ public class DistributedFSCheck extends
) throws IOException {
// open file
FSDataInputStream in = null;
+ Path p = new Path(name);
try {
- in = fs.open(new Path(name));
+ in = fs.open(p);
} catch(IOException e) {
return name + "@(missing)";
}
in.seek(offset);
long actualSize = 0;
try {
- long blockSize = fs.getDefaultBlockSize();
+ long blockSize = fs.getDefaultBlockSize(p);
reporter.setStatus("reading " + name + "@" +
offset + "/" + blockSize);
for( int curSize = bufferSize;
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1329405&r1=1329404&r2=1329405&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Mon Apr 23 19:46:16 2012
@@ -287,7 +287,7 @@ public class JobHistory {
FSDataOutputStream out = logDirFs.create(logFile,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
+ logDirFs.getDefaultReplication(logFile),
jobHistoryBlockSize, null);
EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
jobFileOut = logDirFs.create(logDirConfPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
- logDirFs.getDefaultBlockSize(), null);
+ logDirFs.getDefaultReplication(logDirConfPath),
+ logDirFs.getDefaultBlockSize(logDirConfPath), null);
jobConf.writeXml(jobFileOut);
jobFileOut.close();
}