You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sz...@apache.org on 2012/01/16 05:24:34 UTC
svn commit: r1231834 [6/7] - in
/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project: ./ bin/
conf/ hadoop-mapreduce-client/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/
hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/ja...
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockContainer.java Mon Jan 16 04:24:24 2012
@@ -1,120 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
-import org.apache.hadoop.yarn.util.BuilderUtils;
-
-public class MockContainer implements Container {
-
- private ContainerId id;
- private ContainerState state;
- private String user;
- private ContainerLaunchContext launchContext;
- private final Map<Path, String> resource = new HashMap<Path, String>();
- private RecordFactory recordFactory;
-
- public MockContainer(ApplicationAttemptId appAttemptId,
- Dispatcher dispatcher, Configuration conf, String user,
- ApplicationId appId, int uniqId) {
-
- this.user = user;
- this.recordFactory = RecordFactoryProvider.getRecordFactory(conf);
- this.id = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId,
- uniqId);
- this.launchContext = recordFactory
- .newRecordInstance(ContainerLaunchContext.class);
- launchContext.setContainerId(id);
- launchContext.setUser(user);
- this.state = ContainerState.NEW;
-
- }
-
- public void setState(ContainerState state) {
- this.state = state;
- }
-
- @Override
- public ContainerId getContainerID() {
- return id;
- }
-
- @Override
- public String getUser() {
- return user;
- }
-
- @Override
- public ContainerState getContainerState() {
- return state;
- }
-
- @Override
- public ContainerLaunchContext getLaunchContext() {
- return launchContext;
- }
-
- @Override
- public Credentials getCredentials() {
- return null;
- }
-
- @Override
- public Map<Path, String> getLocalizedResources() {
- return resource;
- }
-
- @Override
- public ContainerStatus cloneAndGetContainerStatus() {
- ContainerStatus containerStatus = recordFactory
- .newRecordInstance(ContainerStatus.class);
- containerStatus
- .setState(org.apache.hadoop.yarn.api.records.ContainerState.RUNNING);
- containerStatus.setContainerId(this.launchContext.getContainerId());
- containerStatus.setDiagnostics("testing");
- containerStatus.setExitStatus(0);
- return containerStatus;
- }
-
- @Override
- public String toString() {
- return "";
- }
-
- @Override
- public void handle(ContainerEvent event) {
- }
-
-}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java Mon Jan 16 04:24:24 2012
@@ -26,6 +26,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.server.sec
import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -72,6 +74,45 @@ public class TestNMWebServer {
FileUtil.fullyDelete(testRootDir);
FileUtil.fullyDelete(testLogDir);
}
+
+ private String startNMWebAppServer(String webAddr) {
+ Context nmContext = new NodeManager.NMContext();
+ ResourceView resourceView = new ResourceView() {
+ @Override
+ public long getVmemAllocatedForContainers() {
+ return 0;
+ }
+ @Override
+ public long getPmemAllocatedForContainers() {
+ return 0;
+ }
+ };
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
+ conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
+ NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
+ healthChecker.init(conf);
+ LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
+ conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
+ WebServer server = new WebServer(nmContext, resourceView,
+ new ApplicationACLsManager(conf), dirsHandler);
+ server.init(conf);
+ server.start();
+ String webAppAddr = conf.get(YarnConfiguration.NM_WEBAPP_ADDRESS);
+ return StringUtils.split(webAppAddr, ':')[1];
+ }
+
+ @Test
+ public void testNMWebAppWithOutPort() throws IOException {
+ String port = startNMWebAppServer("0.0.0.0");
+ Assert.assertTrue("Port is not updated", Integer.parseInt(port) > 0);
+ }
+
+ @Test
+ public void testNMWebAppWithEphemeralPort() throws IOException {
+ String port = startNMWebAppServer("0.0.0.0:0");
+ Assert.assertTrue("Port is not updated", Integer.parseInt(port) > 0);
+ }
@Test
public void testNMWebApp() throws IOException {
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java Mon Jan 16 04:24:24 2012
@@ -38,8 +38,6 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.MockApp;
-import org.apache.hadoop.yarn.server.nodemanager.MockContainer;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
@@ -384,18 +382,91 @@ public class TestNMWebServicesApps exten
String message = exception.getString("message");
String type = exception.getString("exception");
String classname = exception.getString("javaClassName");
- WebServicesTestUtils
- .checkStringMatch(
- "exception message",
- "No enum const class org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE",
- message);
- WebServicesTestUtils.checkStringMatch("exception type",
- "IllegalArgumentException", type);
- WebServicesTestUtils.checkStringMatch("exception classname",
- "java.lang.IllegalArgumentException", classname);
+ verifyStatInvalidException(message, type, classname);
}
}
+ // verify the exception object default format is JSON
+ @Test
+ public void testNodeAppsStateInvalidDefault() throws JSONException, Exception {
+ WebResource r = resource();
+ Application app = new MockApp(1);
+ nmContext.getApplications().put(app.getAppId(), app);
+ addAppContainers(app);
+ Application app2 = new MockApp("foo", 1234, 2);
+ nmContext.getApplications().put(app2.getAppId(), app2);
+ addAppContainers(app2);
+
+ try {
+ r.path("ws").path("v1").path("node").path("apps")
+ .queryParam("state", "FOO_STATE").get(JSONObject.class);
+ fail("should have thrown exception on invalid user query");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject msg = response.getEntity(JSONObject.class);
+ JSONObject exception = msg.getJSONObject("RemoteException");
+ assertEquals("incorrect number of elements", 3, exception.length());
+ String message = exception.getString("message");
+ String type = exception.getString("exception");
+ String classname = exception.getString("javaClassName");
+ verifyStatInvalidException(message, type, classname);
+ }
+ }
+
+ // test that the exception output also returns XML
+ @Test
+ public void testNodeAppsStateInvalidXML() throws JSONException, Exception {
+ WebResource r = resource();
+ Application app = new MockApp(1);
+ nmContext.getApplications().put(app.getAppId(), app);
+ addAppContainers(app);
+ Application app2 = new MockApp("foo", 1234, 2);
+ nmContext.getApplications().put(app2.getAppId(), app2);
+ addAppContainers(app2);
+
+ try {
+ r.path("ws").path("v1").path("node").path("apps")
+ .queryParam("state", "FOO_STATE").accept(MediaType.APPLICATION_XML)
+ .get(JSONObject.class);
+ fail("should have thrown exception on invalid user query");
+ } catch (UniformInterfaceException ue) {
+ ClientResponse response = ue.getResponse();
+
+ assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+ assertEquals(MediaType.APPLICATION_XML_TYPE, response.getType());
+ String msg = response.getEntity(String.class);
+
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+ DocumentBuilder db = dbf.newDocumentBuilder();
+ InputSource is = new InputSource();
+ is.setCharacterStream(new StringReader(msg));
+ Document dom = db.parse(is);
+ NodeList nodes = dom.getElementsByTagName("RemoteException");
+ Element element = (Element) nodes.item(0);
+ String message = WebServicesTestUtils.getXmlString(element, "message");
+ String type = WebServicesTestUtils.getXmlString(element, "exception");
+ String classname = WebServicesTestUtils.getXmlString(element,
+ "javaClassName");
+ verifyStatInvalidException(message, type, classname);
+ }
+ }
+
+ private void verifyStatInvalidException(String message, String type,
+ String classname) {
+ WebServicesTestUtils
+ .checkStringMatch(
+ "exception message",
+ "No enum const class org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState.FOO_STATE",
+ message);
+ WebServicesTestUtils.checkStringMatch("exception type",
+ "IllegalArgumentException", type);
+ WebServicesTestUtils.checkStringMatch("exception classname",
+ "java.lang.IllegalArgumentException", classname);
+ }
+
@Test
public void testNodeSingleApps() throws JSONException, Exception {
testNodeSingleAppHelper(MediaType.APPLICATION_JSON);
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java Mon Jan 16 04:24:24 2012
@@ -39,8 +39,6 @@ import org.apache.hadoop.yarn.event.Asyn
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
-import org.apache.hadoop.yarn.server.nodemanager.MockApp;
-import org.apache.hadoop.yarn.server.nodemanager.MockContainer;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml Mon Jan 16 04:24:24 2012
@@ -139,7 +139,6 @@
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
- <version>1.2</version>
<executions>
<execution>
<phase>compile</phase>
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Mon Jan 16 04:24:24 2012
@@ -1,20 +1,20 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.resourcemanager;
@@ -28,12 +28,15 @@ import java.util.concurrent.atomic.Atomi
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.PolicyProvider;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.ClientRMProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetAllApplicationsResponse;
@@ -43,6 +46,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterMetricsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
@@ -53,8 +58,8 @@ import org.apache.hadoop.yarn.api.protoc
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -67,6 +72,8 @@ import org.apache.hadoop.yarn.factories.
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -78,6 +85,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.util.BuilderUtils;
/**
@@ -97,18 +105,22 @@ public class ClientRMService extends Abs
private String clientServiceBindAddress;
private Server server;
+ private RMDelegationTokenSecretManager rmDTSecretManager;
+
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
InetSocketAddress clientBindAddress;
private final ApplicationACLsManager applicationsACLsManager;
-
+
public ClientRMService(RMContext rmContext, YarnScheduler scheduler,
- RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager) {
+ RMAppManager rmAppManager, ApplicationACLsManager applicationACLsManager,
+ RMDelegationTokenSecretManager rmDTSecretManager) {
super(ClientRMService.class.getName());
this.scheduler = scheduler;
this.rmContext = rmContext;
this.rmAppManager = rmAppManager;
this.applicationsACLsManager = applicationACLsManager;
+ this.rmDTSecretManager = rmDTSecretManager;
}
@Override
@@ -118,21 +130,19 @@ public class ClientRMService extends Abs
YarnConfiguration.DEFAULT_RM_ADDRESS);
clientBindAddress =
NetUtils.createSocketAddr(clientServiceBindAddress,
- YarnConfiguration.DEFAULT_RM_PORT,
- YarnConfiguration.RM_ADDRESS);
+ YarnConfiguration.DEFAULT_RM_PORT,
+ YarnConfiguration.RM_ADDRESS);
super.init(conf);
}
@Override
public void start() {
- // All the clients to appsManager are supposed to be authenticated via
- // Kerberos if security is enabled, so no secretManager.
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
this.server =
rpc.getServer(ClientRMProtocol.class, this,
clientBindAddress,
- conf, null,
+ conf, this.rmDTSecretManager,
conf.getInt(YarnConfiguration.RM_CLIENT_THREAD_COUNT,
YarnConfiguration.DEFAULT_RM_CLIENT_THREAD_COUNT));
@@ -423,6 +433,49 @@ public class ClientRMService extends Abs
return response;
}
+
+ @Override
+ public GetDelegationTokenResponse getDelegationToken(
+ GetDelegationTokenRequest request) throws YarnRemoteException {
+ try {
+
+ // Verify that the connection is kerberos authenticated
+ AuthenticationMethod authMethod = UserGroupInformation
+ .getRealAuthenticationMethod(UserGroupInformation.getCurrentUser());
+ if (UserGroupInformation.isSecurityEnabled()
+ && (authMethod != AuthenticationMethod.KERBEROS)) {
+ throw new IOException(
+ "Delegation Token can be issued only with kerberos authentication");
+ }
+
+ GetDelegationTokenResponse response =
+ recordFactory.newRecordInstance(GetDelegationTokenResponse.class);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ Text owner = new Text(ugi.getUserName());
+ Text realUser = null;
+ if (ugi.getRealUser() != null) {
+ realUser = new Text(ugi.getRealUser().getUserName());
+ }
+ RMDelegationTokenIdentifier tokenIdentifier =
+ new RMDelegationTokenIdentifier(owner, new Text(request.getRenewer()),
+ realUser);
+ Token<RMDelegationTokenIdentifier> realRMDTtoken =
+ new Token<RMDelegationTokenIdentifier>(tokenIdentifier,
+ this.rmDTSecretManager);
+ response.setRMDelegationToken(
+ BuilderUtils.newDelegationToken(
+ realRMDTtoken.getIdentifier(),
+ realRMDTtoken.getKind().toString(),
+ realRMDTtoken.getPassword(),
+ clientBindAddress.getAddress().getHostAddress() + ":"
+ + clientBindAddress.getPort()
+ ));
+ return response;
+ } catch(IOException io) {
+ throw RPCUtil.getRemoteException(io);
+ }
+ }
+
void refreshServiceAcls(Configuration configuration,
PolicyProvider policyProvider) {
this.server.refreshServiceAcl(configuration, policyProvider);
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java Mon Jan 16 04:24:24 2012
@@ -1,20 +1,20 @@
/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.yarn.server.resourcemanager;
@@ -41,12 +41,13 @@ import org.apache.hadoop.yarn.event.Disp
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager;
import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
+import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.StoreFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -67,9 +68,9 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServlet;
-import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
@@ -107,7 +108,7 @@ public class ResourceManager extends Com
private EventHandler<SchedulerEvent> schedulerDispatcher;
protected RMAppManager rmAppManager;
protected ApplicationACLsManager applicationACLsManager;
-
+ protected RMDelegationTokenSecretManager rmDTSecretManager;
private WebApp webApp;
private RMContext rmContext;
private final Store store;
@@ -193,7 +194,7 @@ public class ResourceManager extends Com
// Register event handler for RMAppManagerEvents
this.rmDispatcher.register(RMAppManagerEventType.class,
this.rmAppManager);
-
+ this.rmDTSecretManager = createRMDelegationTokenSecretManager();
clientRM = createClientRMService();
addService(clientRM);
@@ -255,7 +256,7 @@ public class ResourceManager extends Com
}
@Private
- public static final class SchedulerEventDispatcher extends AbstractService
+ public static class SchedulerEventDispatcher extends AbstractService
implements EventHandler<SchedulerEvent> {
private final ResourceScheduler scheduler;
@@ -435,7 +436,12 @@ public class ResourceManager extends Com
startWepApp();
DefaultMetricsSystem.initialize("ResourceManager");
-
+ try {
+ rmDTSecretManager.startThreads();
+ } catch(IOException ie) {
+ throw new YarnException("Failed to start secret manager threads", ie);
+ }
+
super.start();
/*synchronized(shutdown) {
@@ -459,6 +465,7 @@ public class ResourceManager extends Com
if (webApp != null) {
webApp.stop();
}
+ rmDTSecretManager.stopThreads();
/*synchronized(shutdown) {
shutdown.set(true);
@@ -475,9 +482,25 @@ public class ResourceManager extends Com
this.nmLivelinessMonitor, this.containerTokenSecretManager);
}
+ protected RMDelegationTokenSecretManager
+ createRMDelegationTokenSecretManager() {
+ long secretKeyInterval =
+ conf.getLong(YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_KEY,
+ YarnConfiguration.DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT);
+ long tokenMaxLifetime =
+ conf.getLong(YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_KEY,
+ YarnConfiguration.DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT);
+ long tokenRenewInterval =
+ conf.getLong(YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ YarnConfiguration.DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+
+ return new RMDelegationTokenSecretManager(secretKeyInterval,
+ tokenMaxLifetime, tokenRenewInterval, 3600000);
+ }
+
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
- this.applicationACLsManager);
+ this.applicationACLsManager, this.rmDTSecretManager);
}
protected ApplicationMasterService createApplicationMasterService() {
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Mon Jan 16 04:24:24 2012
@@ -265,8 +265,8 @@ public class ResourceTrackerService exte
HeartbeatResponse latestResponse = recordFactory
.newRecordInstance(HeartbeatResponse.class);
latestResponse.setResponseId(lastHeartbeatResponse.getResponseId() + 1);
- latestResponse.addAllContainersToCleanup(rmNode.pullContainersToCleanUp());
- latestResponse.addAllApplicationsToCleanup(rmNode.pullAppsToCleanup());
+ latestResponse.addAllContainersToCleanup(rmNode.getContainersToCleanUp());
+ latestResponse.addAllApplicationsToCleanup(rmNode.getAppsToCleanup());
latestResponse.setNodeAction(NodeAction.NORMAL);
// 4. Send status to RMNode, saving the latest response.
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java Mon Jan 16 04:24:24 2012
@@ -101,9 +101,9 @@ public interface RMNode {
public RMNodeState getState();
- public List<ContainerId> pullContainersToCleanUp();
+ public List<ContainerId> getContainersToCleanUp();
- public List<ApplicationId> pullAppsToCleanup();
+ public List<ApplicationId> getAppsToCleanup();
public HeartbeatResponse getLastHeartBeatResponse();
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java Mon Jan 16 04:24:24 2012
@@ -89,7 +89,6 @@ public class RMNodeImpl implements RMNod
/* set of containers that have just launched */
private final Map<ContainerId, ContainerStatus> justLaunchedContainers =
new HashMap<ContainerId, ContainerStatus>();
-
/* set of containers that need to be cleaned */
private final Set<ContainerId> containersToClean = new TreeSet<ContainerId>(
@@ -248,54 +247,38 @@ public class RMNodeImpl implements RMNod
}
@Override
- public List<ApplicationId> pullAppsToCleanup() {
- this.writeLock.lock();
-
- try {
- List<ApplicationId> lastfinishedApplications = new ArrayList<ApplicationId>();
- lastfinishedApplications.addAll(this.finishedApplications);
- this.finishedApplications.clear();
- return lastfinishedApplications;
- } finally {
- this.writeLock.unlock();
- }
-
- }
-
- @Private
- public List<ContainerId> getContainersToCleanUp() {
+ public List<ApplicationId> getAppsToCleanup() {
this.readLock.lock();
+
try {
- return new ArrayList<ContainerId>(containersToClean);
+ return new ArrayList<ApplicationId>(this.finishedApplications);
} finally {
this.readLock.unlock();
}
+
}
@Override
- public List<ContainerId> pullContainersToCleanUp() {
+ public List<ContainerId> getContainersToCleanUp() {
- this.writeLock.lock();
+ this.readLock.lock();
try {
- List<ContainerId> containersToCleanUp = new ArrayList<ContainerId>();
- containersToCleanUp.addAll(this.containersToClean);
- this.containersToClean.clear();
- return containersToCleanUp;
+ return new ArrayList<ContainerId>(this.containersToClean);
} finally {
- this.writeLock.unlock();
+ this.readLock.unlock();
}
};
@Override
public HeartbeatResponse getLastHeartBeatResponse() {
- this.writeLock.lock();
+ this.readLock.lock();
try {
return this.latestHeartBeatResponse;
} finally {
- this.writeLock.unlock();
+ this.readLock.unlock();
}
}
@@ -407,14 +390,22 @@ public class RMNodeImpl implements RMNod
for (ContainerStatus remoteContainer : statusEvent.getContainers()) {
ContainerId containerId = remoteContainer.getContainerId();
- // Don't bother with containers already scheduled for cleanup,
- // the scheduler doens't need to know any more about this container
+ // Don't bother with containers already scheduled for cleanup, or for
+ // applications already killed. The scheduler doens't need to know any
+ // more about this container
if (rmNode.containersToClean.contains(containerId)) {
LOG.info("Container " + containerId + " already scheduled for " +
"cleanup, no further processing");
continue;
}
-
+ if (rmNode.finishedApplications.contains(containerId
+ .getApplicationAttemptId().getApplicationId())) {
+ LOG.info("Container " + containerId
+ + " belongs to an application that is already killed,"
+ + " no further processing");
+ continue;
+ }
+
// Process running containers
if (remoteContainer.getState() == ContainerState.RUNNING) {
if (!rmNode.justLaunchedContainers.containsKey(containerId)) {
@@ -435,6 +426,12 @@ public class RMNodeImpl implements RMNod
rmNode.context.getDelegationTokenRenewer().updateKeepAliveApplications(
statusEvent.getKeepAliveAppIds());
+
+ // HeartBeat processing from our end is done, as node pulls the following
+ // lists before sending status-updates. Clear data-structures
+ rmNode.containersToClean.clear();
+ rmNode.finishedApplications.clear();
+
return RMNodeState.RUNNING;
}
}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApp.java Mon Jan 16 04:24:24 2012
@@ -39,9 +39,9 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerReservedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
@@ -61,6 +62,7 @@ import com.google.common.collect.Multise
* Each running Application in the RM corresponds to one instance
* of this class.
*/
+@SuppressWarnings("unchecked")
public class SchedulerApp {
private static final Log LOG = LogFactory.getLog(SchedulerApp.class);
@@ -174,13 +176,20 @@ public class SchedulerApp {
this.appSchedulingInfo.stop(rmAppAttemptFinalState);
}
- synchronized public void containerLaunchedOnNode(ContainerId containerId) {
+ public synchronized void containerLaunchedOnNode(ContainerId containerId,
+ NodeId nodeId) {
// Inform the container
RMContainer rmContainer =
getRMContainer(containerId);
- rmContainer.handle(
- new RMContainerEvent(containerId,
- RMContainerEventType.LAUNCHED));
+ if (rmContainer == null) {
+ // Some unknown container sneaked into the system. Kill it.
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
+ return;
+ }
+
+ rmContainer.handle(new RMContainerEvent(containerId,
+ RMContainerEventType.LAUNCHED));
}
synchronized public void containerCompleted(RMContainer rmContainer,
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Mon Jan 16 04:24:24 2012
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -76,6 +77,7 @@ import org.apache.hadoop.yarn.server.sec
@LimitedPrivate("yarn")
@Evolving
+@SuppressWarnings("unchecked")
public class CapacityScheduler
implements ResourceScheduler, CapacitySchedulerContext {
@@ -588,10 +590,12 @@ implements ResourceScheduler, CapacitySc
LOG.info("Unknown application: " + applicationAttemptId +
" launched container " + containerId +
" on node: " + node);
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
return;
}
- application.containerLaunchedOnNode(containerId);
+ application.containerLaunchedOnNode(containerId, node.getNodeID());
}
@Override
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Mon Jan 16 04:24:24 2012
@@ -65,6 +65,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
@@ -87,6 +88,7 @@ import org.apache.hadoop.yarn.util.Build
@LimitedPrivate("yarn")
@Evolving
+@SuppressWarnings("unchecked")
public class FifoScheduler implements ResourceScheduler {
private static final Log LOG = LogFactory.getLog(FifoScheduler.class);
@@ -282,7 +284,6 @@ public class FifoScheduler implements Re
return nodes.get(nodeId);
}
- @SuppressWarnings("unchecked")
private synchronized void addApplication(ApplicationAttemptId appAttemptId,
String user) {
// TODO: Fix store
@@ -655,10 +656,14 @@ public class FifoScheduler implements Re
LOG.info("Unknown application: " + applicationAttemptId +
" launched container " + containerId +
" on node: " + node);
+ // Some unknown container sneaked into the system. Kill it.
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new RMNodeCleanContainerEvent(node.getNodeID(), containerId));
+
return;
}
- application.containerLaunchedOnNode(containerId);
+ application.containerLaunchedOnNode(containerId, node.getNodeID());
}
@Lock(FifoScheduler.class)
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java Mon Jan 16 04:24:24 2012
@@ -20,67 +20,113 @@ package org.apache.hadoop.yarn.server.re
import static org.apache.hadoop.yarn.util.StringHelper.join;
+import java.util.ArrayList;
+
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
+import org.apache.hadoop.yarn.webapp.ResponseInfo;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
import com.google.inject.servlet.RequestScoped;
class CapacitySchedulerPage extends RmView {
static final String _Q = ".ui-state-default.ui-corner-all";
- static final float WIDTH_F = 0.8f;
+ static final float Q_MAX_WIDTH = 0.8f;
+ static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
static final String Q_END = "left:101%";
- static final String OVER = "font-size:1px;background:rgba(255, 140, 0, 0.8)";
- static final String UNDER = "font-size:1px;background:rgba(50, 205, 50, 0.8)";
- static final float EPSILON = 1e-8f;
+ static final String Q_GIVEN = "left:0%;background:none;border:1px dashed rgba(0,0,0,0.25)";
+ static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
+ static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
@RequestScoped
- static class Parent {
- CSQueue queue;
+ static class CSQInfo {
+ CapacitySchedulerInfo csinfo;
+ CapacitySchedulerQueueInfo qinfo;
+ }
+
+ static class LeafQueueInfoBlock extends HtmlBlock {
+ final CapacitySchedulerLeafQueueInfo lqinfo;
+
+ @Inject LeafQueueInfoBlock(ViewContext ctx, CSQInfo info) {
+ super(ctx);
+ lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo;
+ }
+
+ @Override
+ protected void render(Block html) {
+ ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status").
+ _("Queue State:", lqinfo.getQueueState()).
+ _("Capacity:", percent(lqinfo.getCapacity() / 100)).
+ _("Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
+ _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)).
+ _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)).
+ _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)).
+ _("Utilization:", percent(lqinfo.getUtilization() / 100)).
+ _("Used Resources:", lqinfo.getUsedResources().toString()).
+ _("Num Active Applications:", Integer.toString(lqinfo.getNumActiveApplications())).
+ _("Num Pending Applications:", Integer.toString(lqinfo.getNumPendingApplications())).
+ _("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
+ _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
+ _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
+ _("Max Active Applications:", Integer.toString(lqinfo.getMaxActiveApplications())).
+ _("Max Active Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())).
+ _("User Limit:", Integer.toString(lqinfo.getUserLimit()) + "%").
+ _("User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor()));
+
+ html._(InfoBlock.class);
+
+ // clear the info contents so this queue's info doesn't accumulate into another queue's info
+ ri.clear();
+ }
}
public static class QueueBlock extends HtmlBlock {
- final Parent parent;
- final CapacitySchedulerInfo sinfo;
+ final CSQInfo csqinfo;
- @Inject QueueBlock(Parent parent) {
- this.parent = parent;
- sinfo = new CapacitySchedulerInfo(parent.queue);
+ @Inject QueueBlock(CSQInfo info) {
+ csqinfo = info;
}
@Override
public void render(Block html) {
+ ArrayList<CapacitySchedulerQueueInfo> subQueues =
+ (csqinfo.qinfo == null) ? csqinfo.csinfo.getSubQueues()
+ : csqinfo.qinfo.getSubQueues();
UL<Hamlet> ul = html.ul();
- for (CapacitySchedulerQueueInfo info : sinfo.getSubQueues()) {
+ for (CapacitySchedulerQueueInfo info : subQueues) {
float used = info.getUsedCapacity() / 100;
float set = info.getCapacity() / 100;
- float delta = Math.abs(set - used) + 0.001f;
float max = info.getMaxCapacity() / 100;
LI<UL<Hamlet>> li = ul.
li().
- a(_Q).$style(width(max * WIDTH_F)).
- $title(join("used:", percent(used), " set:", percent(set),
- " max:", percent(max))).
- //span().$style(Q_END)._(absMaxPct)._().
- span().$style(join(width(delta/max), ';',
- used > set ? OVER : UNDER, ';',
- used > set ? left(set/max) : left(used/max)))._('.')._().
- span(".q", info.getQueuePath().substring(5))._();
- if (info.getQueue() instanceof ParentQueue) {
- // this could be optimized better
- parent.queue = info.getQueue();
- li.
- _(QueueBlock.class);
+ a(_Q).$style(width(max * Q_MAX_WIDTH)).
+ $title(join("capacity:", percent(set), " used:", percent(used),
+ " max capacity:", percent(max))).
+ span().$style(join(Q_GIVEN, ";font-size:1px;", width(set/max))).
+ _('.')._().
+ span().$style(join(width(used*set/max),
+ ";font-size:1px;left:0%;", used > 1 ? Q_OVER : Q_UNDER)).
+ _('.')._().
+ span(".q", info.getQueuePath().substring(5))._().
+ span().$class("qstats").$style(left(Q_STATS_POS)).
+ _(join(percent(used), " used"))._();
+
+ csqinfo.qinfo = info;
+ if (info.getSubQueues() == null) {
+ li.ul("#lq").li()._(LeafQueueInfoBlock.class)._()._();
+ } else {
+ li._(QueueBlock.class);
}
li._();
}
@@ -91,11 +137,11 @@ class CapacitySchedulerPage extends RmVi
static class QueuesBlock extends HtmlBlock {
final CapacityScheduler cs;
- final Parent parent;
+ final CSQInfo csqinfo;
- @Inject QueuesBlock(ResourceManager rm, Parent parent) {
+ @Inject QueuesBlock(ResourceManager rm, CSQInfo info) {
cs = (CapacityScheduler) rm.getResourceScheduler();
- this.parent = parent;
+ csqinfo = info;
}
@Override
@@ -110,24 +156,36 @@ class CapacitySchedulerPage extends RmVi
if (cs == null) {
ul.
li().
- a(_Q).$style(width(WIDTH_F)).
+ a(_Q).$style(width(Q_MAX_WIDTH)).
span().$style(Q_END)._("100% ")._().
span(".q", "default")._()._();
} else {
CSQueue root = cs.getRootQueue();
- parent.queue = root;
- CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(parent.queue);
+ CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root);
+ csqinfo.csinfo = sinfo;
+ csqinfo.qinfo = null;
+
float used = sinfo.getUsedCapacity() / 100;
- float set = sinfo.getCapacity() / 100;
- float delta = Math.abs(set - used) + 0.001f;
ul.
+ li().$style("margin-bottom: 1em").
+ span().$style("font-weight: bold")._("Legend:")._().
+ span().$class("qlegend ui-corner-all").$style(Q_GIVEN).
+ _("Capacity")._().
+ span().$class("qlegend ui-corner-all").$style(Q_UNDER).
+ _("Used")._().
+ span().$class("qlegend ui-corner-all").$style(Q_OVER).
+ _("Used (over capacity)")._().
+ span().$class("qlegend ui-corner-all ui-state-default").
+ _("Max Capacity")._().
+ _().
li().
- a(_Q).$style(width(WIDTH_F)).
+ a(_Q).$style(width(Q_MAX_WIDTH)).
$title(join("used:", percent(used))).
- span().$style(Q_END)._("100%")._().
- span().$style(join(width(delta), ';', used > set ? OVER : UNDER,
- ';', used > set ? left(set) : left(used)))._(".")._().
+ span().$style(join(width(used), ";left:0%;",
+ used > 1 ? Q_OVER : Q_UNDER))._(".")._().
span(".q", "root")._().
+ span().$class("qstats").$style(left(Q_STATS_POS)).
+ _(join(percent(used), " used"))._().
_(QueueBlock.class)._();
}
ul._()._().
@@ -144,13 +202,18 @@ class CapacitySchedulerPage extends RmVi
"#cs ul { list-style: none }",
"#cs a { font-weight: normal; margin: 2px; position: relative }",
"#cs a span { font-weight: normal; font-size: 80% }",
- "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }")._().
+ "#cs-wrapper .ui-widget-header { padding: 0.2em 0.5em }",
+ ".qstats { font-weight: normal; font-size: 80%; position: absolute }",
+ ".qlegend { font-weight: normal; padding: 0 1em; margin: 1em }",
+ "table.info tr th {width: 50%}")._(). // to center info table
script("/static/jt/jquery.jstree.js").
script().$type("text/javascript").
_("$(function() {",
" $('#cs a span').addClass('ui-corner-all').css('position', 'absolute');",
" $('#cs').bind('loaded.jstree', function (e, data) {",
- " data.inst.open_all(); }).",
+ " data.inst.open_all();",
+ " data.inst.close_node('#lq', true);",
+ " }).",
" jstree({",
" core: { animation: 188, html_titles: true },",
" plugins: ['themeroller', 'html_data', 'ui'],",
@@ -160,8 +223,9 @@ class CapacitySchedulerPage extends RmVi
" });",
" $('#cs').bind('select_node.jstree', function(e, data) {",
" var q = $('.q', data.rslt.obj).first().text();",
- " if (q == 'root') q = '';",
- " $('#apps').dataTable().fnFilter(q, 3);",
+ " if (q == 'root') q = '';",
+ " else q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+ " $('#apps').dataTable().fnFilter(q, 3, true);",
" });",
" $('#cs').show();",
"});")._();
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java Mon Jan 16 04:24:24 2012
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo;
+import org.apache.hadoop.yarn.webapp.RemoteExceptionData;
@Singleton
@Provider
@@ -55,7 +56,8 @@ public class JAXBContextResolver impleme
CapacitySchedulerQueueInfo.class, FifoSchedulerInfo.class,
SchedulerTypeInfo.class, NodeInfo.class, UserMetricsInfo.class,
CapacitySchedulerInfo.class, ClusterMetricsInfo.class,
- SchedulerInfo.class, AppsInfo.class, NodesInfo.class };
+ SchedulerInfo.class, AppsInfo.class, NodesInfo.class,
+ RemoteExceptionData.class};
public JAXBContextResolver() throws Exception {
this.types = new HashSet<Class>(Arrays.asList(cTypes));
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java Mon Jan 16 04:24:24 2012
@@ -26,9 +26,8 @@ import javax.xml.bind.annotation.XmlRoot
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
-import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.ParentQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
@XmlRootElement(name = "capacityScheduler")
@XmlType(name = "capacityScheduler")
@@ -83,21 +82,11 @@ public class CapacitySchedulerInfo exten
CSQueue parentQueue = parent;
ArrayList<CapacitySchedulerQueueInfo> queuesInfo = new ArrayList<CapacitySchedulerQueueInfo>();
for (CSQueue queue : parentQueue.getChildQueues()) {
- float usedCapacity = queue.getUsedCapacity() * 100;
- float capacity = queue.getCapacity() * 100;
- String queueName = queue.getQueueName();
- String queuePath = queue.getQueuePath();
- float max = queue.getMaximumCapacity();
- if (max < EPSILON || max > 1f)
- max = 1f;
- float maxCapacity = max * 100;
- QueueState state = queue.getState();
- CapacitySchedulerQueueInfo info = new CapacitySchedulerQueueInfo(
- capacity, usedCapacity, maxCapacity, queueName, state, queuePath);
-
- if (queue instanceof ParentQueue) {
- info.isParent = true;
- info.queue = queue;
+ CapacitySchedulerQueueInfo info;
+ if (queue instanceof LeafQueue) {
+ info = new CapacitySchedulerLeafQueueInfo((LeafQueue)queue);
+ } else {
+ info = new CapacitySchedulerQueueInfo(queue);
info.subQueues = getQueues(queue);
}
queuesInfo.add(info);
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java Mon Jan 16 04:24:24 2012
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import javax.xml.bind.annotation.XmlSeeAlso;
import javax.xml.bind.annotation.XmlTransient;
import org.apache.hadoop.yarn.api.records.QueueState;
@@ -29,20 +30,23 @@ import org.apache.hadoop.yarn.server.res
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
+@XmlSeeAlso({CapacitySchedulerLeafQueueInfo.class})
public class CapacitySchedulerQueueInfo {
@XmlTransient
- protected String queuePath;
- @XmlTransient
- protected Boolean isParent = false;
+ static final float EPSILON = 1e-8f;
- // bit odd to store this but makes html easier for now
@XmlTransient
- protected CSQueue queue;
+ protected String queuePath;
protected float capacity;
protected float usedCapacity;
protected float maxCapacity;
+ protected float absoluteCapacity;
+ protected float absoluteMaxCapacity;
+ protected float utilization;
+ protected int numApplications;
+ protected String usedResources;
protected String queueName;
protected QueueState state;
protected ArrayList<CapacitySchedulerQueueInfo> subQueues;
@@ -50,22 +54,23 @@ public class CapacitySchedulerQueueInfo
CapacitySchedulerQueueInfo() {
};
- CapacitySchedulerQueueInfo(float cap, float used, float max, String name,
- QueueState state, String path) {
- this.capacity = cap;
- this.usedCapacity = used;
- this.maxCapacity = max;
- this.queueName = name;
- this.state = state;
- this.queuePath = path;
- }
-
- public Boolean isParent() {
- return this.isParent;
- }
-
- public CSQueue getQueue() {
- return this.queue;
+ CapacitySchedulerQueueInfo(CSQueue q) {
+ queuePath = q.getQueuePath();
+ capacity = q.getCapacity() * 100;
+ usedCapacity = q.getUsedCapacity() * 100;
+
+ maxCapacity = q.getMaximumCapacity();
+ if (maxCapacity < EPSILON || maxCapacity > 1f)
+ maxCapacity = 1f;
+ maxCapacity *= 100;
+
+ absoluteCapacity = cap(q.getAbsoluteCapacity(), 0f, 1f) * 100;
+ absoluteMaxCapacity = cap(q.getAbsoluteMaximumCapacity(), 0f, 1f) * 100;
+ utilization = q.getUtilization() * 100;
+ numApplications = q.getNumApplications();
+ usedResources = q.getUsedResources().toString();
+ queueName = q.getQueueName();
+ state = q.getState();
}
public float getCapacity() {
@@ -80,6 +85,26 @@ public class CapacitySchedulerQueueInfo
return this.maxCapacity;
}
+ public float getAbsoluteCapacity() {
+ return absoluteCapacity;
+ }
+
+ public float getAbsoluteMaxCapacity() {
+ return absoluteMaxCapacity;
+ }
+
+ public float getUtilization() {
+ return utilization;
+ }
+
+ public int getNumApplications() {
+ return numApplications;
+ }
+
+ public String getUsedResources() {
+ return usedResources;
+ }
+
public String getQueueName() {
return this.queueName;
}
@@ -96,4 +121,14 @@ public class CapacitySchedulerQueueInfo
return this.subQueues;
}
+ /**
+ * Limit a value to a specified range.
+ * @param val the value to be capped
+ * @param low the lower bound of the range (inclusive)
+ * @param hi the upper bound of the range (inclusive)
+ * @return the capped value
+ */
+ static float cap(float val, float low, float hi) {
+ return Math.min(Math.max(val, low), hi);
+ }
}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java Mon Jan 16 04:24:24 2012
@@ -39,15 +39,17 @@ public class MockNM {
private int responseId;
private NodeId nodeId;
- private final String nodeIdStr;
private final int memory;
private final ResourceTrackerService resourceTracker;
private final int httpPort = 2;
MockNM(String nodeIdStr, int memory, ResourceTrackerService resourceTracker) {
- this.nodeIdStr = nodeIdStr;
this.memory = memory;
this.resourceTracker = resourceTracker;
+ String[] splits = nodeIdStr.split(":");
+ nodeId = Records.newRecord(NodeId.class);
+ nodeId.setHost(splits[0]);
+ nodeId.setPort(Integer.parseInt(splits[1]));
}
public NodeId getNodeId() {
@@ -63,14 +65,10 @@ public class MockNM {
new HashMap<ApplicationId, List<ContainerStatus>>();
conts.put(container.getId().getApplicationAttemptId().getApplicationId(),
Arrays.asList(new ContainerStatus[] { container.getContainerStatus() }));
- nodeHeartbeat(conts, true,nodeId);
+ nodeHeartbeat(conts, true);
}
public NodeId registerNode() throws Exception {
- String[] splits = nodeIdStr.split(":");
- nodeId = Records.newRecord(NodeId.class);
- nodeId.setHost(splits[0]);
- nodeId.setPort(Integer.parseInt(splits[1]));
RegisterNodeManagerRequest req = Records.newRecord(
RegisterNodeManagerRequest.class);
req.setNodeId(nodeId);
@@ -83,11 +81,11 @@ public class MockNM {
}
public HeartbeatResponse nodeHeartbeat(boolean b) throws Exception {
- return nodeHeartbeat(new HashMap<ApplicationId, List<ContainerStatus>>(), b,nodeId);
+ return nodeHeartbeat(new HashMap<ApplicationId, List<ContainerStatus>>(), b);
}
public HeartbeatResponse nodeHeartbeat(Map<ApplicationId,
- List<ContainerStatus>> conts, boolean isHealthy, NodeId nodeId) throws Exception {
+ List<ContainerStatus>> conts, boolean isHealthy) throws Exception {
NodeHeartbeatRequest req = Records.newRecord(NodeHeartbeatRequest.class);
NodeStatus status = Records.newRecord(NodeStatus.class);
status.setNodeId(nodeId);
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java Mon Jan 16 04:24:24 2012
@@ -152,13 +152,13 @@ public class MockNodes {
}
@Override
- public List<ApplicationId> pullAppsToCleanup() {
+ public List<ApplicationId> getAppsToCleanup() {
// TODO Auto-generated method stub
return null;
}
@Override
- public List<ContainerId> pullContainersToCleanUp() {
+ public List<ContainerId> getContainersToCleanUp() {
// TODO Auto-generated method stub
return null;
}
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java Mon Jan 16 04:24:24 2012
@@ -178,7 +178,7 @@ public class MockRM extends ResourceMana
@Override
protected ClientRMService createClientRMService() {
return new ClientRMService(getRMContext(), getResourceScheduler(),
- rmAppManager, applicationACLsManager) {
+ rmAppManager, applicationACLsManager, null) {
@Override
public void start() {
// override to not start rpc handler
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java Mon Jan 16 04:24:24 2012
@@ -89,7 +89,7 @@ public class TestApplicationACLs {
resourceManager = new MockRM(conf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(getRMContext(), this.scheduler,
- this.rmAppManager, this.applicationACLsManager);
+ this.rmAppManager, this.applicationACLsManager, null);
};
};
new Thread() {
Modified: hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java?rev=1231834&r1=1231833&r2=1231834&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java (original)
+++ hadoop/common/branches/branch-0.23-PB/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java Mon Jan 16 04:24:24 2012
@@ -19,26 +19,39 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import junit.framework.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.util.BuilderUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.junit.Test;
-import org.mortbay.log.Log;
public class TestApplicationCleanup {
+ private static final Log LOG = LogFactory
+ .getLog(TestApplicationCleanup.class);
+
@Test
public void testAppCleanup() throws Exception {
Logger rootLogger = LogManager.getRootLogger();
@@ -67,11 +80,13 @@ public class TestApplicationCleanup {
List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
int contReceived = conts.size();
- while (contReceived < request) {
+ int waitCount = 0;
+ while (contReceived < request && waitCount++ < 20) {
conts = am.allocate(new ArrayList<ResourceRequest>(),
new ArrayList<ContainerId>()).getAllocatedContainers();
contReceived += conts.size();
- Log.info("Got " + contReceived + " containers. Waiting to get " + request);
+ LOG.info("Got " + contReceived + " containers. Waiting to get "
+ + request);
Thread.sleep(2000);
}
Assert.assertEquals(request, conts.size());
@@ -86,11 +101,12 @@ public class TestApplicationCleanup {
//currently only containers are cleaned via this
//AM container is cleaned via container launcher
- while (cleanedConts < 2 || cleanedApps < 1) {
+ waitCount = 0;
+ while ((cleanedConts < 3 || cleanedApps < 1) && waitCount++ < 20) {
HeartbeatResponse resp = nm1.nodeHeartbeat(true);
contsToClean = resp.getContainersToCleanupList();
apps = resp.getApplicationsToCleanupList();
- Log.info("Waiting to get cleanup events.. cleanedConts: "
+ LOG.info("Waiting to get cleanup events.. cleanedConts: "
+ cleanedConts + " cleanedApps: " + cleanedApps);
cleanedConts += contsToClean.size();
cleanedApps += apps.size();
@@ -99,6 +115,130 @@ public class TestApplicationCleanup {
Assert.assertEquals(1, apps.size());
Assert.assertEquals(app.getApplicationId(), apps.get(0));
+ Assert.assertEquals(1, cleanedApps);
+ Assert.assertEquals(3, cleanedConts);
+
+ rm.stop();
+ }
+
+ @Test
+ public void testContainerCleanup() throws Exception {
+
+ Logger rootLogger = LogManager.getRootLogger();
+ rootLogger.setLevel(Level.DEBUG);
+ final DrainDispatcher dispatcher = new DrainDispatcher();
+ MockRM rm = new MockRM() {
+ @Override
+ protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
+ return new SchedulerEventDispatcher(this.scheduler) {
+ @Override
+ public void handle(SchedulerEvent event) {
+ scheduler.handle(event);
+ }
+ };
+ }
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("h1:1234", 5000);
+
+ RMApp app = rm.submitApp(2000);
+
+ //kick the scheduling
+ nm1.nodeHeartbeat(true);
+
+ RMAppAttempt attempt = app.getCurrentAppAttempt();
+ MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
+ am.registerAppAttempt();
+
+ //request for containers
+ int request = 2;
+ am.allocate("h1" , 1000, request,
+ new ArrayList<ContainerId>());
+ dispatcher.await();
+
+ //kick the scheduler
+ nm1.nodeHeartbeat(true);
+ List<Container> conts = am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ int contReceived = conts.size();
+ int waitCount = 0;
+ while (contReceived < request && waitCount++ < 20) {
+ conts = am.allocate(new ArrayList<ResourceRequest>(),
+ new ArrayList<ContainerId>()).getAllocatedContainers();
+ dispatcher.await();
+ contReceived += conts.size();
+ LOG.info("Got " + contReceived + " containers. Waiting to get "
+ + request);
+ Thread.sleep(2000);
+ }
+ Assert.assertEquals(request, conts.size());
+
+ // Release a container.
+ ArrayList<ContainerId> release = new ArrayList<ContainerId>();
+ release.add(conts.get(1).getId());
+ am.allocate(new ArrayList<ResourceRequest>(), release);
+ dispatcher.await();
+
+ // Send one more heartbeat with a fake running container. This is to
+ // simulate the situation that can happen if the NM reports that container
+ // is running in the same heartbeat when the RM asks it to clean it up.
+ Map<ApplicationId, List<ContainerStatus>> containerStatuses =
+ new HashMap<ApplicationId, List<ContainerStatus>>();
+ ArrayList<ContainerStatus> containerStatusList =
+ new ArrayList<ContainerStatus>();
+ containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1)
+ .getId(), ContainerState.RUNNING, "nothing", 0));
+ containerStatuses.put(app.getApplicationId(), containerStatusList);
+
+ HeartbeatResponse resp = nm1.nodeHeartbeat(containerStatuses, true);
+ dispatcher.await();
+ List<ContainerId> contsToClean = resp.getContainersToCleanupList();
+ int cleanedConts = contsToClean.size();
+ waitCount = 0;
+ while (cleanedConts < 1 && waitCount++ < 20) {
+ resp = nm1.nodeHeartbeat(true);
+ dispatcher.await();
+ contsToClean = resp.getContainersToCleanupList();
+ LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
+ cleanedConts += contsToClean.size();
+ Thread.sleep(1000);
+ }
+ LOG.info("Got cleanup for " + contsToClean.get(0));
+ Assert.assertEquals(1, cleanedConts);
+
+ // Now to test the case when RM already gave cleanup, and NM suddenly
+ // realizes that the container is running.
+ LOG.info("Testing container launch much after release and "
+ + "NM getting cleanup");
+ containerStatuses.clear();
+ containerStatusList.clear();
+ containerStatusList.add(BuilderUtils.newContainerStatus(conts.get(1)
+ .getId(), ContainerState.RUNNING, "nothing", 0));
+ containerStatuses.put(app.getApplicationId(), containerStatusList);
+
+ resp = nm1.nodeHeartbeat(containerStatuses, true);
+ dispatcher.await();
+ contsToClean = resp.getContainersToCleanupList();
+ cleanedConts = contsToClean.size();
+ // The cleanup list won't be instantaneous as it is given out by scheduler
+ // and not RMNodeImpl.
+ waitCount = 0;
+ while (cleanedConts < 1 && waitCount++ < 20) {
+ resp = nm1.nodeHeartbeat(true);
+ dispatcher.await();
+ contsToClean = resp.getContainersToCleanupList();
+ LOG.info("Waiting to get cleanup events.. cleanedConts: " + cleanedConts);
+ cleanedConts += contsToClean.size();
+ Thread.sleep(1000);
+ }
+ LOG.info("Got cleanup for " + contsToClean.get(0));
+ Assert.assertEquals(1, cleanedConts);
rm.stop();
}