You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/04/24 21:05:20 UTC
svn commit: r1329947 [4/4] - in
/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ conf/
hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/
hadoop-mapreduce-...
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/java/org/apache/hadoop/mapred/gridmix/LoadJob.java Tue Apr 24 19:05:09 2012
@@ -288,23 +288,23 @@ class LoadJob extends GridmixJob {
final long[] reduceBytes = split.getOutputBytes();
final long[] reduceRecords = split.getOutputRecords();
- // enable gridmix map output record for compression
- final boolean emulateMapOutputCompression =
- CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
- && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
- float compressionRatio = 1.0f;
- if (emulateMapOutputCompression) {
- compressionRatio =
- CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
- LOG.info("GridMix is configured to use a compression ratio of "
- + compressionRatio + " for the map output data.");
- key.setCompressibility(true, compressionRatio);
- val.setCompressibility(true, compressionRatio);
- }
-
long totalRecords = 0L;
final int nReduces = ctxt.getNumReduceTasks();
if (nReduces > 0) {
+ // enable gridmix map output record for compression
+ boolean emulateMapOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false);
+ float compressionRatio = 1.0f;
+ if (emulateMapOutputCompression) {
+ compressionRatio =
+ CompressionEmulationUtil.getMapOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the map output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+ }
+
int idx = 0;
int id = split.getId();
for (int i = 0; i < nReduces; ++i) {
@@ -332,7 +332,21 @@ class LoadJob extends GridmixJob {
}
} else {
long mapOutputBytes = reduceBytes[0];
- if (emulateMapOutputCompression) {
+
+ // enable gridmix job output compression
+ boolean emulateJobOutputCompression =
+ CompressionEmulationUtil.isCompressionEmulationEnabled(conf)
+ && conf.getBoolean(FileOutputFormat.COMPRESS, false);
+
+ if (emulateJobOutputCompression) {
+ float compressionRatio =
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf);
+ LOG.info("GridMix is configured to use a compression ratio of "
+ + compressionRatio + " for the job output data.");
+ key.setCompressibility(true, compressionRatio);
+ val.setCompressibility(true, compressionRatio);
+
+ // set the output size accordingly
mapOutputBytes /= compressionRatio;
}
reduces.add(new AvgRecordFactory(mapOutputBytes, reduceRecords[0],
@@ -387,9 +401,13 @@ class LoadJob extends GridmixJob {
@Override
public void cleanup(Context context)
throws IOException, InterruptedException {
+ LOG.info("Starting the cleanup phase.");
for (RecordFactory factory : reduces) {
key.setSeed(r.nextLong());
while (factory.next(key, val)) {
+ // send the progress update (maybe make this a thread)
+ context.progress();
+
context.write(key, val);
key.setSeed(r.nextLong());
@@ -462,7 +480,7 @@ class LoadJob extends GridmixJob {
&& FileOutputFormat.getCompressOutput(context)) {
float compressionRatio =
CompressionEmulationUtil
- .getReduceOutputCompressionEmulationRatio(conf);
+ .getJobOutputCompressionEmulationRatio(conf);
LOG.info("GridMix is configured to use a compression ratio of "
+ compressionRatio + " for the reduce output data.");
val.setCompressibility(true, compressionRatio);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/gridmix/src/test/org/apache/hadoop/mapred/gridmix/TestCompressionEmulationUtils.java Tue Apr 24 19:05:09 2012
@@ -322,10 +322,9 @@ public class TestCompressionEmulationUti
public void testOutputCompressionRatioConfiguration() throws Exception {
Configuration conf = new Configuration();
float ratio = 0.567F;
- CompressionEmulationUtil.setReduceOutputCompressionEmulationRatio(conf,
- ratio);
+ CompressionEmulationUtil.setJobOutputCompressionEmulationRatio(conf, ratio);
assertEquals(ratio,
- CompressionEmulationUtil.getReduceOutputCompressionEmulationRatio(conf),
+ CompressionEmulationUtil.getJobOutputCompressionEmulationRatio(conf),
0.0D);
}
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/index/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/index:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/contrib/vaidya/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/contrib/vaidya:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/examples/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/examples:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/java:r1327258-1329943
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/mapred-default.xml Tue Apr 24 19:05:09 2012
@@ -403,9 +403,6 @@
For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
- The configuration variable mapred.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
</description>
</property>
@@ -420,20 +417,6 @@
</property>
<property>
- <name>mapred.child.ulimit</name>
- <value></value>
- <description>The maximum virtual memory, in KB, of a process launched by the
- Map-Reduce framework. This can be used to control both the Mapper/Reducer
- tasks and applications using Hadoop Pipes, Hadoop Streaming etc.
- By default it is left unspecified to let cluster admins control it via
- limits.conf and other such relevant mechanisms.
-
- Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to
- JavaVM, else the VM might not start.
- </description>
-</property>
-
-<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>
<description>The logging level for the map task. The allowed levels are:
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java Tue Apr 24 19:05:09 2012
@@ -33,12 +33,6 @@ class MapTaskRunner extends TaskRunner {
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Tue Apr 24 19:05:09 2012
@@ -41,12 +41,6 @@ class ReduceTaskRunner extends TaskRunne
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java Tue Apr 24 19:05:09 2012
@@ -110,20 +110,7 @@ abstract class TaskRunner extends Thread
public String getChildJavaOpts(JobConf jobConf, String defaultValue) {
return jobConf.get(JobConf.MAPRED_TASK_JAVA_OPTS, defaultValue);
}
-
- /**
- * Get the maximum virtual memory of the child map/reduce tasks.
- * @param jobConf job configuration
- * @return the maximum virtual memory of the child task or <code>-1</code> if
- * none is specified
- * @deprecated Use limits specific to the map or reduce tasks set via
- * {@link JobConf#MAPRED_MAP_TASK_ULIMIT} or
- * {@link JobConf#MAPRED_REDUCE_TASK_ULIMIT}
- */
- @Deprecated
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_TASK_ULIMIT, -1);
- }
+
/**
* Get the environment variables for the child map/reduce tasks.
@@ -188,8 +175,7 @@ abstract class TaskRunner extends Thread
tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf);
- // set memory limit using ulimit if feasible and necessary ...
- List<String> setup = getVMSetupCmd();
+ List<String> setup = new ArrayList<String>();
// Set up the redirection of the task's stdout and stderr streams
File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask());
@@ -311,26 +297,6 @@ abstract class TaskRunner extends Thread
}
/**
- * @return
- */
- private List<String> getVMSetupCmd() {
-
- int ulimit = getChildUlimit(conf);
- if (ulimit <= 0) {
- return null;
- }
- List<String> setup = null;
- String[] ulimitCmd = Shell.getUlimitMemoryCommand(ulimit);
- if (ulimitCmd != null) {
- setup = new ArrayList<String>();
- for (String arg : ulimitCmd) {
- setup.add(arg);
- }
- }
- return setup;
- }
-
- /**
* Parse the given string and return an array of individual java opts. Split
* on whitespace and replace the special string "@taskid@" with the task ID
* given.
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java?rev=1329947&r1=1329946&r2=1329947&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapreduce/jobhistory/JobHistory.java Tue Apr 24 19:05:09 2012
@@ -287,7 +287,7 @@ public class JobHistory {
FSDataOutputStream out = logDirFs.create(logFile,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
+ logDirFs.getDefaultReplication(logFile),
jobHistoryBlockSize, null);
EventWriter writer = new EventWriter(out);
@@ -306,8 +306,8 @@ public class JobHistory {
jobFileOut = logDirFs.create(logDirConfPath,
new FsPermission(JobHistory.HISTORY_FILE_PERMISSION),
true, defaultBufferSize,
- logDirFs.getDefaultReplication(),
- logDirFs.getDefaultBlockSize(), null);
+ logDirFs.getDefaultReplication(logDirConfPath),
+ logDirFs.getDefaultBlockSize(logDirConfPath), null);
jobConf.writeXml(jobFileOut);
jobFileOut.close();
}
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/fs:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/hdfs:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/ipc:r1327258-1329943
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/src/webapps/job/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/src/webapps/job:r1327258-1329943