You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/05/04 22:23:17 UTC
svn commit: r1334158 [2/3] - in
/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project: ./ conf/
dev-support/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/
hadoop-mapreduce-client/hadoop-mapreduce-clien...
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project:r1332460-1334157
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt Fri May 4 20:22:57 2012
@@ -160,6 +160,12 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-3883. Document yarn.nodemanager.delete.debug-delay-sec
configuration property (Eugene Koontz via tgraves)
+ MAPREDUCE-4219. make default container-executor.conf.dir be a path
+ relative to the container-executor binary. (rvs via tucu)
+
+ MAPREDUCE-4205. retrofit all JVM shutdown hooks to use ShutdownHookManager
+ (tucu)
+
OPTIMIZATIONS
BUG FIXES
@@ -269,6 +275,15 @@ Release 2.0.0 - UNRELEASED
MAPREDUCE-4193. broken doc link for yarn-default.xml in site.xml.
(phunt via tomwhite)
+ MAPREDUCE-4202. TestYarnClientProtocolProvider is broken (Daryn Sharp via
+ bobby)
+
+ MAPREDUCE-3173. MRV2 UI doesn't work properly without internet (Devaraj K
+ via bobby)
+
+ MAPREDUCE-3958. RM: Remove RMNodeState and replace it with NodeState
+ (Bikas Saha via bobby)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -304,6 +319,8 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4079. Allow MR AppMaster to limit ephemeral port range.
(bobby via tgraves)
+ MAPREDUCE-4210. Expose listener address for WebApp (Daryn Sharp via bobby)
+
OPTIMIZATIONS
BUG FIXES
@@ -430,6 +447,17 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4206. Sorting by Last Health-Update on the RM nodes page sorts
does not work correctly (Jonathon Eagles via tgraves)
+ MAPREDUCE-4212. TestJobClientGetJob sometimes fails
+ (Daryn Sharp via tgraves)
+
+ MAPREDUCE-4211. Error conditions (missing appid, appid not found) are
+ masked in the RM app page (Jonathan Eagles via bobby)
+
+ MAPREDUCE-4163. consistently set the bind address (Daryn Sharp via bobby)
+
+ MAPREDUCE-4048. NullPointerException exception while accessing the
+ Application Master UI (Devaraj K via bobby)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/CHANGES.txt
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt:r1332460-1334157
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/conf/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/conf:r1332460-1334157
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java Fri May 4 20:22:57 2012
@@ -19,7 +19,6 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collections;
@@ -127,10 +126,7 @@ public class TaskAttemptListenerImpl ext
}
server.start();
- InetSocketAddress listenerAddress = server.getListenerAddress();
- listenerAddress.getAddress();
- this.address = NetUtils.createSocketAddr(InetAddress.getLocalHost()
- .getCanonicalHostName() + ":" + listenerAddress.getPort());
+ this.address = NetUtils.getConnectAddress(server);
} catch (IOException e) {
throw new YarnException(e);
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java Fri May 4 20:22:57 2012
@@ -90,6 +90,7 @@ import org.apache.hadoop.security.UserGr
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.ClusterInfo;
import org.apache.hadoop.yarn.SystemClock;
@@ -130,6 +131,11 @@ public class MRAppMaster extends Composi
private static final Log LOG = LogFactory.getLog(MRAppMaster.class);
+ /**
+ * Priority of the MRAppMaster shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
private Clock clock;
private final long startTime;
private final long appSubmitTime;
@@ -990,8 +996,8 @@ public class MRAppMaster extends Composi
new MRAppMaster(applicationAttemptId, containerId, nodeHostString,
Integer.parseInt(nodePortString),
Integer.parseInt(nodeHttpPortString), appSubmitTime);
- Runtime.getRuntime().addShutdownHook(
- new MRAppMasterShutdownHook(appMaster));
+ ShutdownHookManager.get().addShutdownHook(
+ new MRAppMasterShutdownHook(appMaster), SHUTDOWN_HOOK_PRIORITY);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
conf.addResource(new Path(MRJobConfig.JOB_CONF_FILE));
String jobUserName = System
@@ -1010,7 +1016,7 @@ public class MRAppMaster extends Composi
// The shutdown hook that runs when a signal is received AND during normal
// close of the JVM.
- static class MRAppMasterShutdownHook extends Thread {
+ static class MRAppMasterShutdownHook implements Runnable {
MRAppMaster appMaster;
MRAppMasterShutdownHook(MRAppMaster appMaster) {
this.appMaster = appMaster;
@@ -1028,12 +1034,6 @@ public class MRAppMaster extends Composi
appMaster.jobHistoryEventHandler.setSignalled(true);
}
appMaster.stop();
- try {
- //Close all the FileSystem objects
- FileSystem.closeAll();
- } catch (IOException ioe) {
- LOG.warn("Failed to close all FileSystem objects", ioe);
- }
}
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java Fri May 4 20:22:57 2012
@@ -18,9 +18,7 @@
package org.apache.hadoop.mapreduce.v2.app.client;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.Collection;
@@ -78,7 +76,6 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
-import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -116,13 +113,7 @@ public class MRClientService extends Abs
public void start() {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address = NetUtils.createSocketAddr("0.0.0.0:0");
- InetAddress hostNameResolved = null;
- try {
- hostNameResolved = InetAddress.getLocalHost();
- } catch (UnknownHostException e) {
- throw new YarnException(e);
- }
+ InetSocketAddress address = new InetSocketAddress(0);
ClientToAMSecretManager secretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
@@ -150,9 +141,7 @@ public class MRClientService extends Abs
}
server.start();
- this.bindAddress =
- NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
- + ":" + server.getPort());
+ this.bindAddress = NetUtils.getConnectAddress(server);
LOG.info("Instantiated MRClientService at " + this.bindAddress);
try {
webApp = WebApps.$for("mapreduce", AppContext.class, appContext, "ws").with(conf).
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/AppController.java Fri May 4 20:22:57 2012
@@ -27,6 +27,8 @@ import java.util.Locale;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
@@ -47,6 +49,8 @@ import com.google.inject.Inject;
* This class renders the various pages that the web app supports.
*/
public class AppController extends Controller implements AMParams {
+ private static final Log LOG = LogFactory.getLog(AppController.class);
+
protected final App app;
protected AppController(App app, Configuration conf, RequestContext ctx,
@@ -220,6 +224,8 @@ public class AppController extends Contr
toString().toLowerCase(Locale.US));
setTitle(join(tt, " Tasks for ", $(JOB_ID)));
} catch (Exception e) {
+ LOG.error("Failed to render tasks page with task type : "
+ + $(TASK_TYPE) + " for job id : " + $(JOB_ID), e);
badRequest(e.getMessage());
}
}
@@ -283,6 +289,8 @@ public class AppController extends Contr
render(attemptsPage());
} catch (Exception e) {
+ LOG.error("Failed to render attempts page with task type : "
+ + $(TASK_TYPE) + " for job id : " + $(JOB_ID), e);
badRequest(e.getMessage());
}
}
@@ -316,7 +324,8 @@ public class AppController extends Contr
*/
void badRequest(String s) {
setStatus(HttpServletResponse.SC_BAD_REQUEST);
- setTitle(join("Bad request: ", s));
+ String title = "Bad request: ";
+ setTitle((s != null) ? join(title, s) : title);
}
/**
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java Fri May 4 20:22:57 2012
@@ -356,7 +356,7 @@ public class TestContainerLauncher {
// make proxy connect to our local containerManager server
ContainerManager proxy = (ContainerManager) rpc.getProxy(
ContainerManager.class,
- NetUtils.createSocketAddr("localhost:" + server.getPort()), conf);
+ NetUtils.getConnectAddress(server), conf);
return proxy;
}
};
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalClientProtocolProvider.java Fri May 4 20:22:57 2012
@@ -33,8 +33,9 @@ public class LocalClientProtocolProvider
@Override
public ClientProtocol create(Configuration conf) throws IOException {
- String framework = conf.get(MRConfig.FRAMEWORK_NAME);
- if (framework != null && !framework.equals(MRConfig.LOCAL_FRAMEWORK_NAME)) {
+ String framework =
+ conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.LOCAL_FRAMEWORK_NAME);
+ if (!MRConfig.LOCAL_FRAMEWORK_NAME.equals(framework)) {
return null;
}
String tracker = conf.get(JTConfig.JT_IPC_ADDRESS, "local");
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestJobClientGetJob.java Fri May 4 20:22:57 2012
@@ -49,8 +49,9 @@ public class TestJobClientGetJob {
JobConf conf = new JobConf();
conf.set("mapreduce.framework.name", "local");
FileInputFormat.addInputPath(conf, createTempFile("in", "hello"));
- FileOutputFormat.setOutputPath(conf,
- new Path(TEST_ROOT_DIR, getClass().getSimpleName()));
+ Path outputDir = new Path(TEST_ROOT_DIR, getClass().getSimpleName());
+ outputDir.getFileSystem(conf).delete(outputDir, true);
+ FileOutputFormat.setOutputPath(conf, outputDir);
JobClient jc = new JobClient(conf);
RunningJob runningJob = jc.submitJob(conf);
assertNotNull("Running job", runningJob);
Propchange: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml:r1332460-1334157
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java Fri May 4 20:22:57 2012
@@ -19,9 +19,7 @@
package org.apache.hadoop.mapreduce.v2.hs;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.security.AccessControlException;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
@@ -76,7 +74,6 @@ import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.DelegationToken;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
@@ -117,17 +114,10 @@ public class HistoryClientService extend
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
initializeWebApp(conf);
- String serviceAddr = conf.get(JHAdminConfig.MR_HISTORY_ADDRESS,
- JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
- InetSocketAddress address = NetUtils.createSocketAddr(serviceAddr,
- JHAdminConfig.DEFAULT_MR_HISTORY_PORT,
- JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS);
- InetAddress hostNameResolved = null;
- try {
- hostNameResolved = InetAddress.getLocalHost();
- } catch (UnknownHostException e) {
- throw new YarnException(e);
- }
+ InetSocketAddress address = conf.getSocketAddr(
+ JHAdminConfig.MR_HISTORY_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_PORT);
server =
rpc.getServer(HSClientProtocol.class, protocolHandler, address,
@@ -143,31 +133,24 @@ public class HistoryClientService extend
}
server.start();
- this.bindAddress =
- NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
- + ":" + server.getPort());
+ this.bindAddress = conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_ADDRESS,
+ server.getListenerAddress());
LOG.info("Instantiated MRClientService at " + this.bindAddress);
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String resolvedAddress = bindAddress.getHostName() + ":" + bindAddress.getPort();
- conf.set(JHAdminConfig.MR_HISTORY_ADDRESS, resolvedAddress);
-
- String hostname = getConfig().get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
- hostname = (hostname.contains(":")) ? hostname.substring(0, hostname.indexOf(":")) : hostname;
- int port = webApp.port();
- resolvedAddress = hostname + ":" + port;
- conf.set(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS, resolvedAddress);
- }
-
super.start();
}
private void initializeWebApp(Configuration conf) {
webApp = new HsWebApp(history);
- String bindAddress = conf.get(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
- JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS);
- WebApps.$for("jobhistory", HistoryClientService.class, this, "ws").with(conf).at(bindAddress).start(webApp);
+ InetSocketAddress bindAddress = conf.getSocketAddr(
+ JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
+ JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);
+ // NOTE: there should be a .at(InetSocketAddress)
+ WebApps.$for("jobhistory", HistoryClientService.class, this, "ws")
+ .with(conf).at(NetUtils.getHostPortString(bindAddress)).start(webApp);
+ conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
+ webApp.getListenerAddress());
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java Fri May 4 20:22:57 2012
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -40,6 +41,12 @@ import org.apache.hadoop.yarn.service.Co
*
*****************************************************************/
public class JobHistoryServer extends CompositeService {
+
+ /**
+ * Priority of the JobHistoryServer shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
private static final Log LOG = LogFactory.getLog(JobHistoryServer.class);
private HistoryContext historyContext;
private HistoryClientService clientService;
@@ -118,8 +125,9 @@ public class JobHistoryServer extends Co
StringUtils.startupShutdownMessage(JobHistoryServer.class, args, LOG);
try {
JobHistoryServer jobHistoryServer = new JobHistoryServer();
- Runtime.getRuntime().addShutdownHook(
- new CompositeServiceShutdownHook(jobHistoryServer));
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(jobHistoryServer),
+ SHUTDOWN_HOOK_PRIORITY);
YarnConfiguration conf = new YarnConfiguration(new JobConf());
jobHistoryServer.init(conf);
jobHistoryServer.start();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientRedirect.java Fri May 4 20:22:57 2012
@@ -390,9 +390,7 @@ public class TestClientRedirect {
rpc.getServer(protocol, this, address,
conf, null, 1);
server.start();
- this.bindAddress =
- NetUtils.createSocketAddr(hostNameResolved.getHostAddress()
- + ":" + server.getPort());
+ this.bindAddress = NetUtils.getConnectAddress(server);
super.start();
amRunning = true;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestYarnClientProtocolProvider.java Fri May 4 20:22:57 2012
@@ -28,6 +28,7 @@ import junit.framework.TestCase;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.LocalJobRunner;
import org.apache.hadoop.mapred.ResourceMgrDelegate;
import org.apache.hadoop.mapred.YARNRunner;
import org.apache.hadoop.mapreduce.protocol.ClientProtocol;
@@ -54,17 +55,26 @@ public class TestYarnClientProtocolProvi
try {
cluster = new Cluster(conf);
- fail("Cluster should not be initialized with out any framework name");
- } catch (IOException e) {
-
+ } catch (Exception e) {
+ throw new Exception(
+ "Failed to initialize a local runner w/o a cluster framework key", e);
}
-
+
+ try {
+ assertTrue("client is not a LocalJobRunner",
+ cluster.getClient() instanceof LocalJobRunner);
+ } finally {
+ if (cluster != null) {
+ cluster.close();
+ }
+ }
+
try {
conf = new Configuration();
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
cluster = new Cluster(conf);
ClientProtocol client = cluster.getClient();
- assertTrue(client instanceof YARNRunner);
+ assertTrue("client is a YARNRunner", client instanceof YARNRunner);
} catch (IOException e) {
} finally {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeState.java Fri May 4 20:22:57 2012
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.api.records;
-// TODO NodeState is a clone of RMNodeState made for MR-3353. In a subsequent
-// patch RMNodeState should be replaced with NodeState
/**
* <p>State of a <code>Node</code>.</p>
*/
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/service/CompositeService.java Fri May 4 20:22:57 2012
@@ -107,12 +107,11 @@ public class CompositeService extends Ab
* JVM Shutdown hook for CompositeService which will stop the give
* CompositeService gracefully in case of JVM shutdown.
*/
- public static class CompositeServiceShutdownHook extends Thread {
+ public static class CompositeServiceShutdownHook implements Runnable {
private CompositeService compositeService;
public CompositeServiceShutdownHook(CompositeService compositeService) {
- super("CompositeServiceShutdownHook for " + compositeService.getName());
this.compositeService = compositeService;
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java Fri May 4 20:22:57 2012
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.webapp;
import static com.google.common.base.Preconditions.checkNotNull;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -75,6 +76,14 @@ public abstract class WebApp extends Ser
@Provides public HttpServer httpServer() { return httpServer; }
+ /**
+ * Get the address the http server is bound to
+ * @return InetSocketAddress
+ */
+ public InetSocketAddress getListenerAddress() {
+ return checkNotNull(httpServer, "httpServer").getListenerAddress();
+ }
+
public int port() {
return checkNotNull(httpServer, "httpServer").getPort();
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/JQueryUI.java Fri May 4 20:22:57 2012
@@ -79,11 +79,11 @@ public class JQueryUI extends HtmlBlock
@Override
protected void render(Block html) {
html.
- link(join("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/themes/",
- getTheme(), "/jquery-ui.css")).
+ link(root_url(join("static/jquery/themes-1.8.16/",
+ getTheme(), "/jquery-ui.css"))).
link(root_url("static/dt-1.7.5/css/jui-dt.css")).
- script("https://ajax.googleapis.com/ajax/libs/jquery/1.6.4/jquery.min.js").
- script("https://ajax.googleapis.com/ajax/libs/jqueryui/1.8.16/jquery-ui.min.js").
+ script(root_url("static/jquery/jquery.min-1.6.4.js")).
+ script(root_url("static/jquery/jquery-ui.min-1.8.16.js")).
script(root_url("static/dt-1.7.5/js/jquery.dataTables.min.js")).
script(root_url("static/yarn.dt.plugins.js")).
script(root_url("static/themeswitcher.js")).
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/themeswitcher.js.gz
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/themeswitcher.js.gz?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
Binary files - no diff available.
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Fri May 4 20:22:57 2012
@@ -85,7 +85,7 @@ public class TestContainerLaunchRPC {
ContainerManager proxy = (ContainerManager) rpc.getProxy(
ContainerManager.class,
- NetUtils.createSocketAddr("localhost:" + server.getPort()), conf);
+ server.getListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory
.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Fri May 4 20:22:57 2012
@@ -75,8 +75,7 @@ public class TestRPC {
// Any unrelated protocol would do
ClientRMProtocol proxy = (ClientRMProtocol) rpc.getProxy(
- ClientRMProtocol.class, NetUtils.createSocketAddr("localhost:"
- + server.getPort()), conf);
+ ClientRMProtocol.class, NetUtils.getConnectAddress(server), conf);
try {
proxy.getNewApplication(Records
@@ -109,7 +108,7 @@ public class TestRPC {
RPC.setProtocolEngine(conf, ContainerManagerPB.class, ProtobufRpcEngine.class);
ContainerManager proxy = (ContainerManager)
rpc.getProxy(ContainerManager.class,
- NetUtils.createSocketAddr("localhost:" + server.getPort()), conf);
+ NetUtils.getConnectAddress(server), conf);
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
containerLaunchContext.setUser("dummy-user");
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java Fri May 4 20:22:57 2012
@@ -149,6 +149,18 @@ public class TestWebApp {
app.stop();
}
+ @Test public void testCreateWithPort() {
+ // see if the ephemeral port is updated
+ WebApp app = WebApps.$for(this).at(0).start();
+ int port = app.getListenerAddress().getPort();
+ assertTrue(port > 0);
+ app.stop();
+ // try to reuse the port
+ app = WebApps.$for(this).at(port).start();
+ assertEquals(port, app.getListenerAddress().getPort());
+ app.stop();
+ }
+
@Test public void testServePaths() {
WebApp app = WebApps.$for("test", this).start();
assertEquals("/test", app.getRedirectPath());
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Fri May 4 20:22:57 2012
@@ -27,7 +27,7 @@
<properties>
<!-- Basedir eeded for generating FindBugs warnings using parent pom -->
<yarn.basedir>${project.parent.parent.basedir}</yarn.basedir>
- <container-executor.conf.dir>/etc/hadoop</container-executor.conf.dir>
+ <container-executor.conf.dir>../etc/hadoop</container-executor.conf.dir>
<container-executor.additional_cflags></container-executor.additional_cflags>
</properties>
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Fri May 4 20:22:57 2012
@@ -30,6 +30,7 @@ import org.apache.hadoop.metrics2.lib.De
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -54,6 +55,12 @@ import org.apache.hadoop.yarn.util.Recor
public class NodeManager extends CompositeService implements
ServiceStateChangeListener {
+
+ /**
+ * Priority of the NodeManager shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
private static final Log LOG = LogFactory.getLog(NodeManager.class);
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
protected ContainerTokenSecretManager containerTokenSecretManager;
@@ -250,11 +257,12 @@ public class NodeManager extends Composi
// Remove the old hook if we are rebooting.
if (hasToReboot && null != nodeManagerShutdownHook) {
- Runtime.getRuntime().removeShutdownHook(nodeManagerShutdownHook);
+ ShutdownHookManager.get().removeShutdownHook(nodeManagerShutdownHook);
}
nodeManagerShutdownHook = new CompositeServiceShutdownHook(this);
- Runtime.getRuntime().addShutdownHook(nodeManagerShutdownHook);
+ ShutdownHookManager.get().addShutdownHook(nodeManagerShutdownHook,
+ SHUTDOWN_HOOK_PRIORITY);
YarnConfiguration conf = new YarnConfiguration();
this.init(conf);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Fri May 4 20:22:57 2012
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.ser
import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
-import java.net.UnknownHostException;
import java.nio.ByteBuffer;
import java.util.Map;
@@ -33,13 +32,13 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.DataInputByteBuffer;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.ContainerManager;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
@@ -244,15 +243,10 @@ public class ContainerManagerImpl extend
}
server.start();
- try {
- resolvedAddress = InetAddress.getLocalHost();
- } catch (UnknownHostException e) {
- throw new YarnException(e);
- }
- this.context.getNodeId().setHost(resolvedAddress.getCanonicalHostName());
- this.context.getNodeId().setPort(server.getPort());
- LOG.info("ContainerManager started at "
- + this.context.getNodeId().toString());
+ InetSocketAddress connectAddress = NetUtils.getConnectAddress(server);
+ this.context.getNodeId().setHost(connectAddress.getHostName());
+ this.context.getNodeId().setPort(connectAddress.getPort());
+ LOG.info("ContainerManager started at " + connectAddress);
super.start();
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java Fri May 4 20:22:57 2012
@@ -69,7 +69,6 @@ import org.apache.hadoop.fs.CommonConfig
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -228,14 +227,9 @@ public class ResourceLocalizationService
cacheCleanupPeriod, cacheCleanupPeriod, TimeUnit.MILLISECONDS);
server = createServer();
server.start();
- String host = getConfig().get(YarnConfiguration.NM_LOCALIZER_ADDRESS)
- .split(":")[0];
- getConfig().set(YarnConfiguration.NM_LOCALIZER_ADDRESS, host + ":"
- + server.getPort());
- localizationServerAddress = getConfig().getSocketAddr(
- YarnConfiguration.NM_LOCALIZER_ADDRESS,
- YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
- YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
+ localizationServerAddress =
+ getConfig().updateConnectAddr(YarnConfiguration.NM_LOCALIZER_ADDRESS,
+ server.getListenerAddress());
LOG.info("Localizer started on port " + server.getPort());
super.start();
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c Fri May 4 20:22:57 2012
@@ -29,6 +29,7 @@
#include <string.h>
#include <sys/stat.h>
#include <sys/types.h>
+#include <limits.h>
#define MAX_SIZE 10
@@ -87,6 +88,25 @@ static int is_only_root_writable(const c
}
/**
+ * Return a string with the configuration file path name resolved via realpath(3)
+ *
+ * NOTE: relative path names are resolved relative to the second argument not getwd(3)
+ */
+char *resolve_config_path(const char* file_name, const char *root) {
+ const char *real_fname = NULL;
+ char buffer[PATH_MAX*2 + 1];
+
+ if (file_name[0] == '/') {
+ real_fname = file_name;
+ } else if (realpath(root, buffer) != NULL) {
+ strncpy(strrchr(buffer, '/') + 1, file_name, PATH_MAX);
+ real_fname = buffer;
+ }
+
+ return (real_fname == NULL) ? NULL : realpath(real_fname, NULL);
+}
+
+/**
* Ensure that the configuration file and all of the containing directories
* are only writable by root. Otherwise, an attacker can change the
* configuration and potentially cause damage.
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h Fri May 4 20:22:57 2012
@@ -24,6 +24,13 @@
*/
int check_configuration_permissions(const char* file_name);
+/**
+ * Return a string with the configuration file path name resolved via realpath(3)
+ *
+ * NOTE: relative path names are resolved relative to the second argument not getwd(3)
+ */
+char *resolve_config_path(const char* file_name, const char *root);
+
// read the given configuration file
void read_config(const char* config_file);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c Fri May 4 20:22:57 2012
@@ -33,6 +33,12 @@
#define STRINGIFY(X) _STRINGIFY(X)
#define CONF_FILENAME "container-executor.cfg"
+// When building as part of a Maven build this value gets defined by using
+// container-executor.conf.dir property. See:
+// hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+// for details.
+// NOTE: if this ends up being a relative path it gets resolved relative to
+// the location of the container-executor binary itself, not getwd(3)
#ifndef HADOOP_CONF_DIR
#error HADOOP_CONF_DIR must be defined
#endif
@@ -96,7 +102,7 @@ int main(int argc, char **argv) {
char *executable_file = get_executable();
char *orig_conf_file = STRINGIFY(HADOOP_CONF_DIR) "/" CONF_FILENAME;
- char *conf_file = realpath(orig_conf_file, NULL);
+ char *conf_file = resolve_config_path(orig_conf_file, argv[0]);
char *local_dirs, *log_dirs;
if (conf_file == NULL) {
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c Fri May 4 20:22:57 2012
@@ -196,6 +196,18 @@ void test_check_user() {
}
}
+void test_resolve_config_path() {
+ printf("\nTesting resolve_config_path\n");
+ if (strcmp(resolve_config_path("/etc/passwd", NULL), "/etc/passwd") != 0) {
+ printf("FAIL: failed to resolve config_name on an absolute path name: /etc/passwd\n");
+ exit(1);
+ }
+ if (strcmp(resolve_config_path("../etc/passwd", "/etc/passwd"), "/etc/passwd") != 0) {
+ printf("FAIL: failed to resolve config_name on a relative path name: ../etc/passwd (relative to /etc/passwd)");
+ exit(1);
+ }
+}
+
void test_check_configuration_permissions() {
printf("\nTesting check_configuration_permissions\n");
if (check_configuration_permissions("/etc/passwd") != 0) {
@@ -668,7 +680,9 @@ int main(int argc, char **argv) {
int my_username = 0;
// clean up any junk from previous run
- system("chmod -R u=rwx " TEST_ROOT "; rm -fr " TEST_ROOT);
+ if (system("chmod -R u=rwx " TEST_ROOT "; rm -fr " TEST_ROOT)) {
+ exit(1);
+ }
if (mkdirs(TEST_ROOT "/logs/userlogs", 0755) != 0) {
exit(1);
@@ -700,6 +714,9 @@ int main(int argc, char **argv) {
printf("\nStarting tests\n");
+ printf("\nTesting resolve_config_path()\n");
+ test_resolve_config_path();
+
printf("\nTesting get_user_directory()\n");
test_get_user_directory();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java Fri May 4 20:22:57 2012
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.no
import static org.mockito.Mockito.mock;
import java.io.IOException;
-import java.net.InetAddress;
-import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -88,7 +86,7 @@ public class TestNodeStatusUpdater {
.getRecordFactory(null);
int heartBeatID = 0;
- volatile Error nmStartError = null;
+ volatile Throwable nmStartError = null;
private final List<NodeId> registeredNodes = new ArrayList<NodeId>();
private final Configuration conf = new YarnConfiguration();
private NodeManager nm;
@@ -118,12 +116,8 @@ public class TestNodeStatusUpdater {
NodeId nodeId = request.getNodeId();
Resource resource = request.getResource();
LOG.info("Registering " + nodeId.toString());
- try {
- Assert.assertEquals(InetAddress.getLocalHost().getCanonicalHostName()
- + ":12345", nodeId.toString());
- } catch (UnknownHostException e) {
- Assert.fail(e.getMessage());
- }
+ // NOTE: this really should be checking against the config value
+ Assert.assertEquals("localhost:12345", nodeId.toString());
Assert.assertEquals(5 * 1024, resource.getMemory());
registeredNodes.add(nodeId);
RegistrationResponse regResponse = recordFactory
@@ -421,8 +415,9 @@ public class TestNodeStatusUpdater {
public void run() {
try {
nm.start();
- } catch (Error e) {
+ } catch (Throwable e) {
TestNodeStatusUpdater.this.nmStartError = e;
+ throw new YarnException(e);
}
}
}.start();
@@ -433,11 +428,11 @@ public class TestNodeStatusUpdater {
int waitCount = 0;
while (nm.getServiceState() == STATE.INITED && waitCount++ != 20) {
LOG.info("Waiting for NM to start..");
+ if (nmStartError != null) {
+ Assert.fail(nmStartError.getCause().getMessage());
+ }
Thread.sleep(1000);
}
- if (nmStartError != null) {
- throw nmStartError;
- }
if (nm.getServiceState() != STATE.STARTED) {
// NM could have failed.
Assert.fail("NodeManager failed to start");
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java Fri May 4 20:22:57 2012
@@ -95,6 +95,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEventType;
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.ArgumentMatcher;
@@ -103,7 +104,14 @@ public class TestResourceLocalizationSer
static final Path basedir =
new Path("target", TestResourceLocalizationService.class.getName());
-
+ static Server mockServer;
+
+ @BeforeClass
+ public static void setup() {
+ mockServer = mock(Server.class);
+ doReturn(new InetSocketAddress(123)).when(mockServer).getListenerAddress();
+ }
+
@Test
public void testLocalizationInit() throws Exception {
final Configuration conf = new Configuration();
@@ -178,7 +186,6 @@ public class TestResourceLocalizationSer
}
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
- Server ignore = mock(Server.class);
LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.init(conf);
@@ -203,7 +210,7 @@ public class TestResourceLocalizationSer
new ResourceLocalizationService(dispatcher, exec, delService,
dirsHandler);
ResourceLocalizationService spyService = spy(rawService);
- doReturn(ignore).when(spyService).createServer();
+ doReturn(mockServer).when(spyService).createServer();
doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(
isA(Configuration.class));
doReturn(lfs).when(spyService)
@@ -366,7 +373,6 @@ public class TestResourceLocalizationSer
}
conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
- Server ignore = mock(Server.class);
DrainDispatcher dispatcher = new DrainDispatcher();
dispatcher.init(conf);
dispatcher.start();
@@ -388,7 +394,7 @@ public class TestResourceLocalizationSer
new ResourceLocalizationService(dispatcher, exec, delService,
dirsHandler);
ResourceLocalizationService spyService = spy(rawService);
- doReturn(ignore).when(spyService).createServer();
+ doReturn(mockServer).when(spyService).createServer();
doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
try {
spyService.init(conf);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java Fri May 4 20:22:57 2012
@@ -120,11 +120,8 @@ public class AdminService extends Abstra
}
this.server.start();
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String resolvedAddress =
- this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort();
- conf.set(YarnConfiguration.RM_ADMIN_ADDRESS, resolvedAddress);
- }
+ conf.updateConnectAddr(YarnConfiguration.RM_ADMIN_ADDRESS,
+ server.getListenerAddress());
super.start();
}
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Fri May 4 20:22:57 2012
@@ -31,7 +31,6 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.util.StringUtils;
@@ -57,7 +56,6 @@ import org.apache.hadoop.yarn.ipc.RPCUti
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -120,13 +118,8 @@ public class ApplicationMasterService ex
this.server.start();
this.bindAddress =
- NetUtils.createSocketAddr(masterServiceAddress.getHostName(),
- this.server.getPort());
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String resolvedAddress =
- this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort();
- conf.set(YarnConfiguration.RM_SCHEDULER_ADDRESS, resolvedAddress);
- }
+ conf.updateConnectAddr(YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ server.getListenerAddress());
super.start();
}
@@ -308,7 +301,7 @@ public class ApplicationMasterService ex
numContainers = schedulerNodeReport.getNumContainers();
}
NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(),
- RMNodeState.toNodeState(rmNode.getState()),
+ rmNode.getState(),
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getNodeHealthStatus());
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Fri May 4 20:22:57 2012
@@ -79,7 +79,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
@@ -150,11 +149,8 @@ public class ClientRMService extends Abs
}
this.server.start();
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String resolvedAddress =
- this.server.getListenerAddress().getHostName() + ":" + this.server.getListenerAddress().getPort();
- conf.set(YarnConfiguration.RM_ADDRESS, resolvedAddress);
- }
+ clientBindAddress = conf.updateConnectAddr(YarnConfiguration.RM_ADDRESS,
+ server.getListenerAddress());
super.start();
}
@@ -417,7 +413,7 @@ public class ClientRMService extends Abs
}
NodeReport report = BuilderUtils.newNodeReport(rmNode.getNodeID(),
- RMNodeState.toNodeState(rmNode.getState()),
+ rmNode.getState(),
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getNodeHealthStatus());
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Fri May 4 20:22:57 2012
@@ -32,6 +32,7 @@ import org.apache.hadoop.metrics2.lib.De
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -87,6 +88,12 @@ import org.apache.hadoop.yarn.webapp.Web
*/
@SuppressWarnings("unchecked")
public class ResourceManager extends CompositeService implements Recoverable {
+
+ /**
+ * Priority of the ResourceManager shutdown hook.
+ */
+ public static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
private static final Log LOG = LogFactory.getLog(ResourceManager.class);
public static final long clusterTimeStamp = System.currentTimeMillis();
@@ -613,8 +620,9 @@ public class ResourceManager extends Com
Configuration conf = new YarnConfiguration();
Store store = StoreFactory.getStore(conf);
ResourceManager resourceManager = new ResourceManager(store);
- Runtime.getRuntime().addShutdownHook(
- new CompositeServiceShutdownHook(resourceManager));
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(resourceManager),
+ SHUTDOWN_HOOK_PRIORITY);
resourceManager.init(conf);
//resourceManager.recover(store.restore());
//store.doneWithRecovery();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Fri May 4 20:22:57 2012
@@ -133,11 +133,8 @@ public class ResourceTrackerService exte
}
this.server.start();
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
- String resolvedAddress =
- server.getListenerAddress().getHostName() + ":" + server.getListenerAddress().getPort();
- conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, resolvedAddress);
- }
+ conf.updateConnectAddr(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS,
+ server.getListenerAddress());
}
@Override
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Fri May 4 20:22:57 2012
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -56,7 +57,6 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.state.InvalidStateTransitonException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -498,7 +498,7 @@ public class RMAppImpl implements RMApp
}
private void processNodeUpdate(RMAppNodeUpdateType type, RMNode node) {
- RMNodeState nodeState = node.getState();
+ NodeState nodeState = node.getState();
updatedNodes.add(node);
LOG.debug("Received node update event:" + type + " for node:" + node
+ " with state:" + nodeState);
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java Fri May 4 20:22:57 2012
@@ -23,10 +23,10 @@ import java.util.List;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
/**
@@ -99,7 +99,7 @@ public interface RMNode {
*/
public Node getNode();
- public RMNodeState getState();
+ public NodeState getState();
public List<ContainerId> getContainersToCleanUp();
Modified: hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1334158&r1=1334157&r2=1334158&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Fri May 4 20:22:57 2012
@@ -40,6 +40,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -104,53 +105,53 @@ public class RMNodeImpl implements RMNod
.newRecordInstance(HeartbeatResponse.class);
private static final StateMachineFactory<RMNodeImpl,
- RMNodeState,
+ NodeState,
RMNodeEventType,
RMNodeEvent> stateMachineFactory
= new StateMachineFactory<RMNodeImpl,
- RMNodeState,
+ NodeState,
RMNodeEventType,
- RMNodeEvent>(RMNodeState.NEW)
+ RMNodeEvent>(NodeState.NEW)
//Transitions from NEW state
- .addTransition(RMNodeState.NEW, RMNodeState.RUNNING,
+ .addTransition(NodeState.NEW, NodeState.RUNNING,
RMNodeEventType.STARTED, new AddNodeTransition())
//Transitions from RUNNING state
- .addTransition(RMNodeState.RUNNING,
- EnumSet.of(RMNodeState.RUNNING, RMNodeState.UNHEALTHY),
+ .addTransition(NodeState.RUNNING,
+ EnumSet.of(NodeState.RUNNING, NodeState.UNHEALTHY),
RMNodeEventType.STATUS_UPDATE, new StatusUpdateWhenHealthyTransition())
- .addTransition(RMNodeState.RUNNING, RMNodeState.DECOMMISSIONED,
+ .addTransition(NodeState.RUNNING, NodeState.DECOMMISSIONED,
RMNodeEventType.DECOMMISSION,
- new DeactivateNodeTransition(RMNodeState.DECOMMISSIONED))
- .addTransition(RMNodeState.RUNNING, RMNodeState.LOST,
+ new DeactivateNodeTransition(NodeState.DECOMMISSIONED))
+ .addTransition(NodeState.RUNNING, NodeState.LOST,
RMNodeEventType.EXPIRE,
- new DeactivateNodeTransition(RMNodeState.LOST))
- .addTransition(RMNodeState.RUNNING, RMNodeState.REBOOTED,
+ new DeactivateNodeTransition(NodeState.LOST))
+ .addTransition(NodeState.RUNNING, NodeState.REBOOTED,
RMNodeEventType.REBOOTING,
- new DeactivateNodeTransition(RMNodeState.REBOOTED))
- .addTransition(RMNodeState.RUNNING, RMNodeState.RUNNING,
+ new DeactivateNodeTransition(NodeState.REBOOTED))
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition())
- .addTransition(RMNodeState.RUNNING, RMNodeState.RUNNING,
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
- .addTransition(RMNodeState.RUNNING, RMNodeState.RUNNING,
+ .addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.RECONNECTED, new ReconnectNodeTransition())
//Transitions from UNHEALTHY state
- .addTransition(RMNodeState.UNHEALTHY,
- EnumSet.of(RMNodeState.UNHEALTHY, RMNodeState.RUNNING),
+ .addTransition(NodeState.UNHEALTHY,
+ EnumSet.of(NodeState.UNHEALTHY, NodeState.RUNNING),
RMNodeEventType.STATUS_UPDATE, new StatusUpdateWhenUnHealthyTransition())
- .addTransition(RMNodeState.UNHEALTHY, RMNodeState.UNHEALTHY,
+ .addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
RMNodeEventType.RECONNECTED, new ReconnectNodeTransition())
- .addTransition(RMNodeState.UNHEALTHY, RMNodeState.UNHEALTHY,
+ .addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
RMNodeEventType.CLEANUP_APP, new CleanUpAppTransition())
- .addTransition(RMNodeState.UNHEALTHY, RMNodeState.UNHEALTHY,
+ .addTransition(NodeState.UNHEALTHY, NodeState.UNHEALTHY,
RMNodeEventType.CLEANUP_CONTAINER, new CleanUpContainerTransition())
// create the topology tables
.installTopology();
- private final StateMachine<RMNodeState, RMNodeEventType,
+ private final StateMachine<NodeState, RMNodeEventType,
RMNodeEvent> stateMachine;
public RMNodeImpl(NodeId nodeId, RMContext context, String hostName,
@@ -252,7 +253,7 @@ public class RMNodeImpl implements RMNod
}
@Override
- public RMNodeState getState() {
+ public NodeState getState() {
this.readLock.lock();
try {
@@ -302,7 +303,7 @@ public class RMNodeImpl implements RMNod
LOG.debug("Processing " + event.getNodeId() + " of type " + event.getType());
try {
writeLock.lock();
- RMNodeState oldState = getState();
+ NodeState oldState = getState();
try {
stateMachine.doTransition(event.getType(), event);
} catch (InvalidStateTransitonException e) {
@@ -321,7 +322,7 @@ public class RMNodeImpl implements RMNod
}
}
- private void updateMetricsForRejoinedNode(RMNodeState previousNodeState) {
+ private void updateMetricsForRejoinedNode(NodeState previousNodeState) {
ClusterMetrics metrics = ClusterMetrics.getMetrics();
metrics.incrNumActiveNodes();
@@ -341,7 +342,7 @@ public class RMNodeImpl implements RMNod
}
}
- private void updateMetricsForDeactivatedNode(RMNodeState finalState) {
+ private void updateMetricsForDeactivatedNode(NodeState finalState) {
ClusterMetrics metrics = ClusterMetrics.getMetrics();
metrics.decrNumActiveNodes();
@@ -440,8 +441,8 @@ public class RMNodeImpl implements RMNod
public static class DeactivateNodeTransition
implements SingleArcTransition<RMNodeImpl, RMNodeEvent> {
- private final RMNodeState finalState;
- public DeactivateNodeTransition(RMNodeState finalState) {
+ private final NodeState finalState;
+ public DeactivateNodeTransition(NodeState finalState) {
this.finalState = finalState;
}
@@ -466,9 +467,9 @@ public class RMNodeImpl implements RMNod
}
public static class StatusUpdateWhenHealthyTransition implements
- MultipleArcTransition<RMNodeImpl, RMNodeEvent, RMNodeState> {
+ MultipleArcTransition<RMNodeImpl, RMNodeEvent, NodeState> {
@Override
- public RMNodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
@@ -486,8 +487,8 @@ public class RMNodeImpl implements RMNod
new NodesListManagerEvent(
NodesListManagerEventType.NODE_UNUSABLE, rmNode));
// Update metrics
- rmNode.updateMetricsForDeactivatedNode(RMNodeState.UNHEALTHY);
- return RMNodeState.UNHEALTHY;
+ rmNode.updateMetricsForDeactivatedNode(NodeState.UNHEALTHY);
+ return NodeState.UNHEALTHY;
}
// Filter the map to only obtain just launched containers and finished
@@ -541,15 +542,15 @@ public class RMNodeImpl implements RMNod
rmNode.containersToClean.clear();
rmNode.finishedApplications.clear();
- return RMNodeState.RUNNING;
+ return NodeState.RUNNING;
}
}
public static class StatusUpdateWhenUnHealthyTransition implements
- MultipleArcTransition<RMNodeImpl, RMNodeEvent, RMNodeState> {
+ MultipleArcTransition<RMNodeImpl, RMNodeEvent, NodeState> {
@Override
- public RMNodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
+ public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;
// Switch the last heartbeatresponse.
@@ -566,11 +567,11 @@ public class RMNodeImpl implements RMNod
// notifiers get update metadata because they will very likely query it
// upon notification
// Update metrics
- rmNode.updateMetricsForRejoinedNode(RMNodeState.UNHEALTHY);
- return RMNodeState.RUNNING;
+ rmNode.updateMetricsForRejoinedNode(NodeState.UNHEALTHY);
+ return NodeState.RUNNING;
}
- return RMNodeState.UNHEALTHY;
+ return NodeState.UNHEALTHY;
}
}
}