You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by tu...@apache.org on 2012/07/18 05:36:46 UTC
svn commit: r1362751 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./
hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/
hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/te...
Author: tucu
Date: Wed Jul 18 03:36:46 2012
New Revision: 1362751
URL: http://svn.apache.org/viewvc?rev=1362751&view=rev
Log:
MAPREDUCE-4406. Users should be able to specify the MiniCluster ResourceManager and JobHistoryServer ports. (ahmed via tucu)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1362751&r1=1362750&r2=1362751&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Jul 18 03:36:46 2012
@@ -139,6 +139,9 @@ Branch-2 ( Unreleased changes )
MAPREDUCE-4422. YARN_APPLICATION_CLASSPATH needs a documented default value in
YarnConfiguration. (ahmed via tucu)
+ MAPREDUCE-4406. Users should be able to specify the MiniCluster ResourceManager
+ and JobHistoryServer ports. (ahmed via tucu)
+
Release 2.1.0-alpha - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java?rev=1362751&r1=1362750&r2=1362751&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java Wed Jul 18 03:36:46 2012
@@ -134,4 +134,15 @@ public class JHAdminConfig {
*/
public static final String MR_HISTORY_STORAGE =
MR_HISTORY_PREFIX + "store.class";
+
+ /** Whether to use fixed ports with the minicluster. */
+ public static final String MR_HISTORY_MINICLUSTER_FIXED_PORTS = MR_HISTORY_PREFIX
+ + "minicluster.fixed.ports";
+
+ /**
+ * Default is false to be able to run tests concurrently without port
+ * conflicts.
+ */
+ public static boolean DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS = false;
+
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java?rev=1362751&r1=1362750&r2=1362751&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java Wed Jul 18 03:36:46 2012
@@ -113,10 +113,6 @@ public class MiniMRYarnCluster extends M
// for corresponding uberized tests.
conf.setBoolean(MRJobConfig.JOB_UBERTASK_ENABLE, false);
- // Set config for JH Server
- conf.set(JHAdminConfig.MR_HISTORY_ADDRESS,
- JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
-
super.init(conf);
}
@@ -128,10 +124,15 @@ public class MiniMRYarnCluster extends M
@Override
public synchronized void start() {
try {
- getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
+ if (!getConfig().getBoolean(
+ JHAdminConfig.MR_HISTORY_MINICLUSTER_FIXED_PORTS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_MINICLUSTER_FIXED_PORTS)) {
+ // pick free random ports.
+ getConfig().set(JHAdminConfig.MR_HISTORY_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ getConfig().set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ }
historyServer = new JobHistoryServer();
historyServer.init(getConfig());
new Thread() {
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1362751&r1=1362750&r2=1362751&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Jul 18 03:36:46 2012
@@ -565,6 +565,16 @@ public class YarnConfiguration extends C
public static final String IS_MINI_YARN_CLUSTER = YARN_PREFIX + ".is.minicluster";
+ /** Whether to use fixed ports with the minicluster. */
+ public static final String YARN_MINICLUSTER_FIXED_PORTS = YARN_PREFIX
+ + "minicluster.fixed.ports";
+
+ /**
+ * Default is false to be able to run tests concurrently without port
+ * conflicts.
+ */
+ public static boolean DEFAULT_YARN_MINICLUSTER_FIXED_PORTS = false;
+
public YarnConfiguration() {
super();
}
Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1362751&r1=1362750&r2=1362751&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Wed Jul 18 03:36:46 2012
@@ -135,18 +135,21 @@ public class MiniYARNCluster extends Com
public synchronized void start() {
try {
getConfig().setBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, true);
- getConfig().set(YarnConfiguration.RM_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(YarnConfiguration.RM_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
- getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS,
- MiniYARNCluster.getHostname() + ":0");
+ if (!getConfig().getBoolean(
+ YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS,
+ YarnConfiguration.DEFAULT_YARN_MINICLUSTER_FIXED_PORTS)) {
+ // pick free random ports.
+ getConfig().set(YarnConfiguration.RM_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ getConfig().set(YarnConfiguration.RM_ADMIN_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ getConfig().set(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ getConfig().set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ getConfig().set(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
+ }
Store store = StoreFactory.getStore(getConfig());
resourceManager = new ResourceManager(store) {
@Override