You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by vi...@apache.org on 2014/11/03 22:45:00 UTC
git commit: YARN-2795. Fixed ResourceManager to not crash loading
node-label data from HDFS in secure mode. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/trunk 237890fea -> ec6cbece8
YARN-2795. Fixed ResourceManager to not crash loading node-label data from HDFS in secure mode. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec6cbece
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec6cbece
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec6cbece
Branch: refs/heads/trunk
Commit: ec6cbece8e7772868ce8ad996135d3136bd32245
Parents: 237890f
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Mon Nov 3 13:44:06 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Nov 3 13:44:06 2014 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/ResourceManager.java | 32 +++++++++-----------
.../yarn/server/resourcemanager/TestRMHA.java | 9 +++---
.../resourcemanager/TestResourceManager.java | 8 ++++-
.../security/TestDelegationTokenRenewer.java | 7 ++++-
5 files changed, 36 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec6cbece/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9566458..25d03f4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -849,6 +849,9 @@ Release 2.6.0 - UNRELEASED
that were caused when adding log-upload-time via YARN-2703. (Xuan Gong via
vinodkv)
+ YARN-2795. Fixed ResourceManager to not crash loading node-label data from
+ HDFS in secure mode. (Wangda Tan via vinodkv)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec6cbece/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 57c8fce..642c732 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -23,9 +23,7 @@ import java.io.InputStream;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
@@ -195,7 +193,17 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected void serviceInit(Configuration conf) throws Exception {
this.conf = conf;
this.rmContext = new RMContextImpl();
-
+
+ // Set UGI and do login
+ // If security is enabled, use login user
+ // If security is not enabled, use current user
+ this.rmLoginUGI = UserGroupInformation.getCurrentUser();
+ try {
+ doSecureLogin();
+ } catch(IOException ie) {
+ throw new YarnRuntimeException("Failed to login", ie);
+ }
+
this.configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
@@ -242,14 +250,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
if (this.rmContext.isHAEnabled()) {
HAUtil.verifyAndSetConfiguration(this.conf);
}
+
createAndInitActiveServices();
webAppAddress = WebAppUtils.getWebAppBindURL(this.conf,
YarnConfiguration.RM_BIND_HOST,
WebAppUtils.getRMWebAppURLWithoutScheme(this.conf));
- this.rmLoginUGI = UserGroupInformation.getCurrentUser();
-
super.serviceInit(this.conf);
}
@@ -1019,17 +1026,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
synchronized void transitionToActive() throws Exception {
- if (rmContext.getHAServiceState() ==
- HAServiceProtocol.HAServiceState.ACTIVE) {
+ if (rmContext.getHAServiceState() == HAServiceProtocol.HAServiceState.ACTIVE) {
LOG.info("Already in active state");
return;
}
LOG.info("Transitioning to active state");
- // use rmLoginUGI to startActiveServices.
- // in non-secure model, rmLoginUGI will be current UGI
- // in secure model, rmLoginUGI will be LoginUser UGI
this.rmLoginUGI.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
@@ -1071,12 +1074,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
@Override
protected void serviceStart() throws Exception {
- try {
- doSecureLogin();
- } catch(IOException ie) {
- throw new YarnRuntimeException("Failed to login", ie);
- }
-
if (this.rmContext.isHAEnabled()) {
transitionToStandby(true);
} else {
@@ -1084,7 +1081,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
startWepApp();
- if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) {
+ if (getConfig().getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER,
+ false)) {
int port = webApp.port();
WebAppUtils.setRMWebAppPort(conf, port);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec6cbece/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index e30ca29..8cef4c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -28,8 +28,6 @@ import java.net.InetSocketAddress;
import javax.ws.rs.core.MediaType;
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -37,10 +35,10 @@ import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HAServiceProtocol.StateChangeRequestInfo;
import org.apache.hadoop.ha.HealthCheckFailedException;
-import org.apache.hadoop.metrics2.MetricsSystem;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -54,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -64,7 +63,7 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
public class TestRMHA {
private Log LOG = LogFactory.getLog(TestRMHA.class);
- private final Configuration configuration = new YarnConfiguration();
+ private Configuration configuration;
private MockRM rm = null;
private RMApp app = null;
private RMAppAttempt attempt = null;
@@ -82,6 +81,8 @@ public class TestRMHA {
@Before
public void setUp() throws Exception {
+ configuration = new Configuration();
+ UserGroupInformation.setConfiguration(configuration);
configuration.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
configuration.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + ","
+ RM2_NODE_ID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec6cbece/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index 1117fbe..6735575 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -57,6 +57,7 @@ public class TestResourceManager {
@Before
public void setUp() throws Exception {
Configuration conf = new YarnConfiguration();
+ UserGroupInformation.setConfiguration(conf);
resourceManager = new ResourceManager();
resourceManager.init(conf);
resourceManager.getRMContext().getContainerTokenSecretManager().rollMasterKey();
@@ -254,7 +255,12 @@ public class TestResourceManager {
AuthenticationFilterInitializer.class.getName() + ", "
+ this.getClass().getName() };
for (String filterInitializer : filterInitializers) {
- resourceManager = new ResourceManager();
+ resourceManager = new ResourceManager() {
+ @Override
+ protected void doSecureLogin() throws IOException {
+ // Skip the login.
+ }
+ };
Configuration conf = new YarnConfiguration();
conf.set(filterInitializerConfKey, filterInitializer);
conf.set("hadoop.security.authentication", "kerberos");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec6cbece/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
index b824df7..7275089 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
@@ -808,7 +808,12 @@ public class TestDelegationTokenRenewer {
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
UserGroupInformation.setConfiguration(conf);
- MockRM rm = new MockRM(conf);
+ MockRM rm = new MockRM(conf) {
+ @Override
+ protected void doSecureLogin() throws IOException {
+ // Skip the login.
+ }
+ };
ByteBuffer tokens = ByteBuffer.wrap("BOGUS".getBytes());
ContainerLaunchContext amContainer =
ContainerLaunchContext.newInstance(