You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by bo...@apache.org on 2012/04/18 00:13:53 UTC
svn commit: r1327306 - in
/hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/...
Author: bobby
Date: Tue Apr 17 22:13:52 2012
New Revision: 1327306
URL: http://svn.apache.org/viewvc?rev=1327306&view=rev
Log:
svn merge -c 1327304 from trunk. FIXES: MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they are not being used any more (Ravi Prakash via bobby)
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Apr 17 22:13:52 2012
@@ -164,6 +164,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4151. RM scheduler web page should filter apps to those that
are relevant to scheduling (Jason Lowe via tgraves)
+ MAPREDUCE-4134. Remove references of mapred.child.ulimit etc. since they
+ are not being used any more (Ravi Prakash via bobby)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JobConf.java Tue Apr 17 22:13:52 2012
@@ -187,9 +187,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_TASK_ULIMIT} can be used to
- * control the maximum virtual memory of the child processes.
- *
* The configuration variable {@link #MAPRED_TASK_ENV} can be used to pass
* other environment variables to the child processes.
*
@@ -210,9 +207,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_MAP_TASK_ULIMIT} can be used to
- * control the maximum virtual memory of the map processes.
- *
* The configuration variable {@link #MAPRED_MAP_TASK_ENV} can be used to pass
* other environment variables to the map processes.
*/
@@ -230,9 +224,6 @@ public class JobConf extends Configurati
* /tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
* -Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
*
- * The configuration variable {@link #MAPRED_REDUCE_TASK_ULIMIT} can be used
- * to control the maximum virtual memory of the reduce processes.
- *
* The configuration variable {@link #MAPRED_REDUCE_TASK_ENV} can be used to
* pass process environment variables to the reduce processes.
*/
@@ -242,36 +233,32 @@ public class JobConf extends Configurati
public static final String DEFAULT_MAPRED_TASK_JAVA_OPTS = "-Xmx200m";
/**
- * Configuration key to set the maximum virutal memory available to the child
- * map and reduce tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_TASK_JAVA_OPTS}, else the VM might not start.
- *
- * @deprecated Use {@link #MAPRED_MAP_TASK_ULIMIT} or
- * {@link #MAPRED_REDUCE_TASK_ULIMIT}
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the child
+ * map and reduce tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
@Deprecated
public static final String MAPRED_TASK_ULIMIT = "mapred.child.ulimit";
/**
- * Configuration key to set the maximum virutal memory available to the
- * map tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_MAP_TASK_JAVA_OPTS}, else the VM might not start.
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the
+ * map tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
- public static final String MAPRED_MAP_TASK_ULIMIT = JobContext.MAP_ULIMIT;
+ @Deprecated
+ public static final String MAPRED_MAP_TASK_ULIMIT = "mapreduce.map.ulimit";
/**
- * Configuration key to set the maximum virutal memory available to the
- * reduce tasks (in kilo-bytes).
- *
- * Note: This must be greater than or equal to the -Xmx passed to the JavaVM
- * via {@link #MAPRED_REDUCE_TASK_JAVA_OPTS}, else the VM might not start.
+ * @deprecated
+ * Configuration key to set the maximum virtual memory available to the
+ * reduce tasks (in kilo-bytes). This has been deprecated and will no
+ * longer have any effect.
*/
- public static final String MAPRED_REDUCE_TASK_ULIMIT =
- JobContext.REDUCE_ULIMIT;
+ @Deprecated
+ public static final String MAPRED_REDUCE_TASK_ULIMIT =
+ "mapreduce.reduce.ulimit";
/**
@@ -1966,6 +1953,15 @@ public class JobConf extends Configurati
+ " Instead use " + JobConf.MAPRED_JOB_MAP_MEMORY_MB_PROPERTY
+ " and " + JobConf.MAPRED_JOB_REDUCE_MEMORY_MB_PROPERTY);
}
+ if(get(JobConf.MAPRED_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_TASK_ULIMIT));
+ }
+ if(get(JobConf.MAPRED_MAP_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_MAP_TASK_ULIMIT));
+ }
+ if(get(JobConf.MAPRED_REDUCE_TASK_ULIMIT) != null ) {
+ LOG.warn(JobConf.deprecatedString(JobConf.MAPRED_REDUCE_TASK_ULIMIT));
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java Tue Apr 17 22:13:52 2012
@@ -183,8 +183,6 @@ public interface MRJobConfig {
public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts";
- public static final String MAP_ULIMIT = "mapreduce.map.ulimit";
-
public static final String MAP_MAX_ATTEMPTS = "mapreduce.map.maxattempts";
public static final String MAP_DEBUG_SCRIPT = "mapreduce.map.debug.script";
@@ -243,8 +241,6 @@ public interface MRJobConfig {
public static final String REDUCE_JAVA_OPTS = "mapreduce.reduce.java.opts";
- public static final String REDUCE_ULIMIT = "mapreduce.reduce.ulimit";
-
public static final String MAPREDUCE_JOB_DIR = "mapreduce.job.dir";
public static final String REDUCE_MAX_ATTEMPTS = "mapreduce.reduce.maxattempts";
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java Tue Apr 17 22:13:52 2012
@@ -314,8 +314,6 @@ public class ConfigUtil {
new String[] {MRJobConfig.MAP_ENV});
Configuration.addDeprecation("mapred.map.child.java.opts",
new String[] {MRJobConfig.MAP_JAVA_OPTS});
- Configuration.addDeprecation("mapred.map.child.ulimit",
- new String[] {MRJobConfig.MAP_ULIMIT});
Configuration.addDeprecation("mapred.map.max.attempts",
new String[] {MRJobConfig.MAP_MAX_ATTEMPTS});
Configuration.addDeprecation("mapred.map.task.debug.script",
@@ -362,8 +360,6 @@ public class ConfigUtil {
new String[] {MRJobConfig.REDUCE_ENV});
Configuration.addDeprecation("mapred.reduce.child.java.opts",
new String[] {MRJobConfig.REDUCE_JAVA_OPTS});
- Configuration.addDeprecation("mapred.reduce.child.ulimit",
- new String[] {MRJobConfig.REDUCE_ULIMIT});
Configuration.addDeprecation("mapred.reduce.max.attempts",
new String[] {MRJobConfig.REDUCE_MAX_ATTEMPTS});
Configuration.addDeprecation("mapred.reduce.parallel.copies",
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml Tue Apr 17 22:13:52 2012
@@ -411,9 +411,6 @@
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
- The configuration variable mapred.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
-
Usage of -Djava.library.path can cause programs to no longer function if
hadoop native libraries are used. These values should instead be set as part
of LD_LIBRARY_PATH in the map / reduce JVM env using the mapreduce.map.env and
@@ -432,20 +429,6 @@
</property>
<property>
- <name>mapred.child.ulimit</name>
- <value></value>
- <description>The maximum virtual memory, in KB, of a process launched by the
- Map-Reduce framework. This can be used to control both the Mapper/Reducer
- tasks and applications using Hadoop Pipes, Hadoop Streaming etc.
- By default it is left unspecified to let cluster admins control it via
- limits.conf and other such relevant mechanisms.
-
- Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to
- JavaVM, else the VM might not start.
- </description>
-</property>
-
-<property>
<name>mapreduce.admin.user.env</name>
<value>LD_LIBRARY_PATH=$HADOOP_COMMON_HOME/lib/native</value>
<description>Expert: Additional execution environment entries for
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/mapred-default.xml Tue Apr 17 22:13:52 2012
@@ -403,9 +403,6 @@
For example, to enable verbose gc logging to a file named for the taskid in
/tmp and to set the heap maximum to be a gigabyte, pass a 'value' of:
-Xmx1024m -verbose:gc -Xloggc:/tmp/@taskid@.gc
-
- The configuration variable mapred.child.ulimit can be used to control the
- maximum virtual memory of the child processes.
</description>
</property>
@@ -420,20 +417,6 @@
</property>
<property>
- <name>mapred.child.ulimit</name>
- <value></value>
- <description>The maximum virtual memory, in KB, of a process launched by the
- Map-Reduce framework. This can be used to control both the Mapper/Reducer
- tasks and applications using Hadoop Pipes, Hadoop Streaming etc.
- By default it is left unspecified to let cluster admins control it via
- limits.conf and other such relevant mechanisms.
-
- Note: mapred.child.ulimit must be greater than or equal to the -Xmx passed to
- JavaVM, else the VM might not start.
- </description>
-</property>
-
-<property>
<name>mapreduce.map.log.level</name>
<value>INFO</value>
<description>The logging level for the map task. The allowed levels are:
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/MapTaskRunner.java Tue Apr 17 22:13:52 2012
@@ -33,12 +33,6 @@ class MapTaskRunner extends TaskRunner {
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_MAP_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/ReduceTaskRunner.java Tue Apr 17 22:13:52 2012
@@ -41,12 +41,6 @@ class ReduceTaskRunner extends TaskRunne
super.getChildJavaOpts(jobConf,
JobConf.DEFAULT_MAPRED_TASK_JAVA_OPTS));
}
-
- @Override
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_REDUCE_TASK_ULIMIT,
- super.getChildUlimit(jobConf));
- }
@Override
public String getChildEnv(JobConf jobConf) {
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java?rev=1327306&r1=1327305&r2=1327306&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java (original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/java/org/apache/hadoop/mapred/TaskRunner.java Tue Apr 17 22:13:52 2012
@@ -110,20 +110,7 @@ abstract class TaskRunner extends Thread
public String getChildJavaOpts(JobConf jobConf, String defaultValue) {
return jobConf.get(JobConf.MAPRED_TASK_JAVA_OPTS, defaultValue);
}
-
- /**
- * Get the maximum virtual memory of the child map/reduce tasks.
- * @param jobConf job configuration
- * @return the maximum virtual memory of the child task or <code>-1</code> if
- * none is specified
- * @deprecated Use limits specific to the map or reduce tasks set via
- * {@link JobConf#MAPRED_MAP_TASK_ULIMIT} or
- * {@link JobConf#MAPRED_REDUCE_TASK_ULIMIT}
- */
- @Deprecated
- public int getChildUlimit(JobConf jobConf) {
- return jobConf.getInt(JobConf.MAPRED_TASK_ULIMIT, -1);
- }
+
/**
* Get the environment variables for the child map/reduce tasks.
@@ -188,8 +175,7 @@ abstract class TaskRunner extends Thread
tracker.addToMemoryManager(t.getTaskID(), t.isMapTask(), conf);
- // set memory limit using ulimit if feasible and necessary ...
- List<String> setup = getVMSetupCmd();
+ List<String> setup = new ArrayList<String>();
// Set up the redirection of the task's stdout and stderr streams
File[] logFiles = prepareLogFiles(taskid, t.isTaskCleanupTask());
@@ -311,26 +297,6 @@ abstract class TaskRunner extends Thread
}
/**
- * @return
- */
- private List<String> getVMSetupCmd() {
-
- int ulimit = getChildUlimit(conf);
- if (ulimit <= 0) {
- return null;
- }
- List<String> setup = null;
- String[] ulimitCmd = Shell.getUlimitMemoryCommand(ulimit);
- if (ulimitCmd != null) {
- setup = new ArrayList<String>();
- for (String arg : ulimitCmd) {
- setup.add(arg);
- }
- }
- return setup;
- }
-
- /**
* Parse the given string and return an array of individual java opts. Split
* on whitespace and replace the special string "@taskid@" with the task ID
* given.