You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by to...@apache.org on 2012/05/03 04:14:30 UTC
svn commit: r1333291 [4/4] - in
/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project: ./ conf/
dev-support/ hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/
hado...
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Thu May 3 02:14:01 2012
@@ -288,23 +288,23 @@ class LoadJob extends GridmixJob {
final long[] reduceBytes = split.getOutputBytes();
final long[] reduceRecords = split.getOutputRecords();
- // enable gridmix map output record for compression
- final boolean emulateMapOutputCompression =
- CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
- && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
- float compressionRatio = 1.0f;
- if (emulateMapOutputCompression) {
- compressionRatio =
- CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
- LOG.info("GridMix is configured to use a compression ratio of "
- + compressionRatio + " for the map output data.");
- key.setCompressibility(true, compressionRatio);
- val.setCompressibility(true, compressionRatio);
- }
-
long totalRecords = 0L;
final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
+ // enable gridmix map output record for compression
+ boolean emulateMapOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
+ float compressionRatio = 1.0f;
+ if (emulateMapOutputCompression) {
+ compressionRatio =
+ CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the map output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+ }
+
int idx = 0;
int id = split.getId();
for (int i = 0; i < nReduces; ++i) {
@@ -332,7 +332,21 @@ class LoadJob extends GridmixJob {
}
} else {
long mapOutputBytes = reduceBytes[0];
- if (emulateMapOutputCompression) {
+
+ // enable gridmix job output compression
+ boolean emulateJobOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(FileOutputFormat.COMPRESS, false);
+
+ if (emulateJobOutputCompression) {
+ float compressionRatio =
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the job output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+
+ // set the output size accordingly
mapOutputBytes /= compressionRatio;
}
reduces.add(new AvgRecordFactory(mapOutputBytes, reduceRecords[0],
@@ -387,9 +401,13 @@ class LoadJob extends GridmixJob {
@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
+ LOG.info("Starting the cleanup phase.");
for (RecordFactory factory : reduces) {
key.setSeed(r.nextLong());
while (factory.next(key, val)) {
+ // send the progress update (maybe make this a thread)
+ context.progress();
+
context.write(key, val);
key.setSeed(r.nextLong());
@@ -462,7 +480,7 @@ class LoadJob extends GridmixJob {
&& FileOutputFormat.getCompressOutput(context)) {
float compressionRatio =
CompressionEmulationUtil
- .getReduceOutputCompressionEmulationRatio(conf);
+ .getJobOutputCompressionEmulationRatio(conf);
LOG.info("GridMix is configured to use a compression ratio of "
+ compressionRatio + " for the reduce output data.");
val.setCompressibility(true, compressionRatio);
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java Thu May 3 02:14:01 2012
@@ -322,10 +322,9 @@ public class TestCompressionEmulationUti
public void testOutputCompressionRatioConfiguration() throws Exception {
Configuration conf = new Configuration();
float ratio = 0.567F;
- CompressionEmulationUtil.setReduceOutputCompressionEmulationRatio(conf,
- ratio);
+ CompressionEmulationUtil.setJobOutputCompressionEmulationRatio(conf, ratio);
assertEquals(ratio,
- CompressionEmulationUtil.getReduceOutputCompressionEmulationRatio(conf),
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf),
0.0D);
}
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1327719-1333290
Modified: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Thu May 3 02:14:01 2012
@@ -287,7 +287,7 @@ public class JobHistory {
FSDataOutputStream out = logDirFs.create(logFile,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
+ logDirFs.getDefaultReplication(logFile),
jobHistoryBlockSize, null);
EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
jobFileOut = logDirFs.create(logDirConfPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
- logDirFs.getDefaultBlockSize(), null);
+ logDirFs.getDefaultReplication(logDirConfPath),
+ logDirFs.getDefaultBlockSize(logDirConfPath), null);
jobConf.writeXml(jobFileOut);
jobFileOut.close();
}
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1327719-1333290
Propchange: hadoop/common/branches/HDFS-3042/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1327719-1333290