You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by tg...@apache.org on 2013/03/26 20:21:44 UTC
svn commit: r1461275 - in
/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src:
main/java/org/apache/hadoop/mapred/gridmix/
test/java/org/apache/hadoop/mapred/gridmix/
Author: tgraves
Date: Tue Mar 26 19:21:43 2013
New Revision: 1461275
URL: http://svn.apache.org/r1461275
Log:
MAPREDUCE-4100. [Gridmix] Bug fixed in compression emulation feature for map only jobs. (amarrk via tgraves)
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java?rev=1461275&r1=1461274&r2=1461275&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/CompressionEmulationUtil.java Tue Mar 26 19:21:43 2013
@@ -85,10 +85,10 @@ class CompressionEmulationUtil {
"gridmix.compression-emulation.map-output.compression-ratio";
/**
- * Configuration property for setting the compression ratio of reduce output.
+ * Configuration property for setting the compression ratio of job output.
*/
- private static final String GRIDMIX_REDUCE_OUTPUT_COMPRESSION_RATIO =
- "gridmix.compression-emulation.reduce-output.compression-ratio";
+ private static final String GRIDMIX_JOB_OUTPUT_COMPRESSION_RATIO =
+ "gridmix.compression-emulation.job-output.compression-ratio";
/**
* Default compression ratio.
@@ -434,20 +434,20 @@ class CompressionEmulationUtil {
}
/**
- * Set the reduce output data compression ratio in the given configuration.
+ * Set the job output data compression ratio in the given configuration.
*/
- static void setReduceOutputCompressionEmulationRatio(Configuration conf,
- float ratio) {
- conf.setFloat(GRIDMIX_REDUCE_OUTPUT_COMPRESSION_RATIO, ratio);
+ static void setJobOutputCompressionEmulationRatio(Configuration conf,
+ float ratio) {
+ conf.setFloat(GRIDMIX_JOB_OUTPUT_COMPRESSION_RATIO, ratio);
}
/**
- * Get the reduce output data compression ratio using the given configuration.
+ * Get the job output data compression ratio using the given configuration.
* If the compression ratio is not set in the configuration then use the
* default value i.e {@value #DEFAULT_COMPRESSION_RATIO}.
*/
- static float getReduceOutputCompressionEmulationRatio(Configuration conf) {
- return conf.getFloat(GRIDMIX_REDUCE_OUTPUT_COMPRESSION_RATIO,
+ static float getJobOutputCompressionEmulationRatio(Configuration conf) {
+ return conf.getFloat(GRIDMIX_JOB_OUTPUT_COMPRESSION_RATIO,
DEFAULT_COMPRESSION_RATIO);
}
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1461275&r1=1461274&r2=1461275&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Tue Mar 26 19:21:43 2013
@@ -288,23 +288,23 @@ class LoadJob extends GridmixJob {
final long[] reduceBytes = split.getOutputBytes();
final long[] reduceRecords = split.getOutputRecords();
- // enable gridmix map output record for compression
- final boolean emulateMapOutputCompression =
- CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
- && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
- float compressionRatio = 1.0f;
- if (emulateMapOutputCompression) {
- compressionRatio =
- CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
- LOG.info("GridMix is configured to use a compression ratio of "
- + compressionRatio + " for the map output data.");
- key.setCompressibility(true, compressionRatio);
- val.setCompressibility(true, compressionRatio);
- }
-
long totalRecords = 0L;
final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
+ // enable gridmix map output record for compression
+ boolean emulateMapOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
+ float compressionRatio = 1.0f;
+ if (emulateMapOutputCompression) {
+ compressionRatio =
+ CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the map output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+ }
+
int idx = 0;
int id = split.getId();
for (int i = 0; i < nReduces; ++i) {
@@ -332,7 +332,21 @@ class LoadJob extends GridmixJob {
}
} else {
long mapOutputBytes = reduceBytes[0];
- if (emulateMapOutputCompression) {
+
+ // enable gridmix job output compression
+ boolean emulateJobOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(FileOutputFormat.COMPRESS, false);
+
+ if (emulateJobOutputCompression) {
+ float compressionRatio =
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the job output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+
+ // set the output size accordingly
mapOutputBytes /= compressionRatio;
}
reduces.add(new AvgRecordFactory(mapOutputBytes, reduceRecords[0],
@@ -387,9 +401,13 @@ class LoadJob extends GridmixJob {
@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
+ LOG.info("Starting the cleanup phase.");
for (RecordFactory factory : reduces) {
key.setSeed(r.nextLong());
while (factory.next(key, val)) {
+ // send the progress update (maybe make this a thread)
+ context.progress();
+
context.write(key, val);
key.setSeed(r.nextLong());
@@ -462,7 +480,7 @@ class LoadJob extends GridmixJob {
&& FileOutputFormat.getCompressOutput(context)) {
float compressionRatio =
CompressionEmulationUtil
- .getReduceOutputCompressionEmulationRatio(conf);
+ .getJobOutputCompressionEmulationRatio(conf);
LOG.info("GridMix is configured to use a compression ratio of "
+ compressionRatio + " for the reduce output data.");
val.setCompressibility(true, compressionRatio);
Modified: hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java?rev=1461275&r1=1461274&r2=1461275&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java Tue Mar 26 19:21:43 2013
@@ -322,10 +322,9 @@ public class TestCompressionEmulationUti
public void testOutputCompressionRatioConfiguration() throws Exception {
Configuration conf = new Configuration();
float ratio = 0.567F;
- CompressionEmulationUtil.setReduceOutputCompressionEmulationRatio(conf,
- ratio);
+ CompressionEmulationUtil.setJobOutputCompressionEmulationRatio(conf, ratio);
assertEquals(ratio,
- CompressionEmulationUtil.getReduceOutputCompressionEmulationRatio(conf),
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf),
0.0D);
}