You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2009/12/23 11:16:48 UTC
svn commit: r893455 - in /hadoop/mapreduce/trunk: CHANGES.txt
src/tools/org/apache/hadoop/tools/DistCp.java
Author: cdouglas
Date: Wed Dec 23 10:16:48 2009
New Revision: 893455
URL: http://svn.apache.org/viewvc?rev=893455&view=rev
Log:
MAPREDUCE-1059. Use distcp.bytes.per.map when adding sync markers in distcp. Contributed by Aaron Kimball
Modified:
hadoop/mapreduce/trunk/CHANGES.txt
hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/mapreduce/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/CHANGES.txt?rev=893455&r1=893454&r2=893455&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/CHANGES.txt (original)
+++ hadoop/mapreduce/trunk/CHANGES.txt Wed Dec 23 10:16:48 2009
@@ -1070,3 +1070,6 @@
MAPREDUCE-1286. Remove quoting from client opts in TaskRunner. (Yuri
Pradkin via cdouglas)
+
+ MAPREDUCE-1059. Use distcp.bytes.per.map when adding sync markers in
+ distcp. (Aaron Kimball via cdouglas)
Modified: hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=893455&r1=893454&r2=893455&view=diff
==============================================================================
--- hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/mapreduce/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Wed Dec 23 10:16:48 2009
@@ -1215,6 +1215,8 @@
FileSystem.mkdirs(jClient.getFs(), jobDirectory, mapredSysPerms);
jobConf.set(JOB_DIR_LABEL, jobDirectory.toString());
+ long maxBytesPerMap = conf.getLong(BYTES_PER_MAP_LABEL, BYTES_PER_MAP);
+
FileSystem dstfs = args.dst.getFileSystem(conf);
boolean dstExists = dstfs.exists(args.dst);
boolean dstIsDir = false;
@@ -1381,7 +1383,7 @@
++cnsyncf;
cbsyncs += child.getLen();
- if (cnsyncf > SYNC_FILE_MAX || cbsyncs > BYTES_PER_MAP) {
+ if (cnsyncf > SYNC_FILE_MAX || cbsyncs > maxBytesPerMap) {
src_writer.sync();
dst_writer.sync();
cnsyncf = 0;