You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ka...@apache.org on 2014/02/28 18:36:04 UTC
svn commit: r1573017 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/...
Author: kasha
Date: Fri Feb 28 17:36:04 2014
New Revision: 1573017
URL: http://svn.apache.org/r1573017
Log:
YARN-1528. Allow setting auth for ZK connections. (kasha)
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMZKUtils.java
- copied unchanged from r1573014, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMZKUtils.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1573017&r1=1573016&r2=1573017&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Fri Feb 28 17:36:04 2014
@@ -226,6 +226,8 @@ Release 2.4.0 - UNRELEASED
YARN-1301. Added the INFO level log of the non-empty blacklist additions
and removals inside ApplicationMasterService. (Tsuyoshi Ozawa via zjshen)
+ YARN-1528. Allow setting auth for ZK connections. (kasha)
+
OPTIMIZATIONS
BUG FIXES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1573017&r1=1573016&r2=1573017&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Fri Feb 28 17:36:04 2014
@@ -335,6 +335,8 @@ public class YarnConfiguration extends C
public static final String RM_ZK_ACL = RM_ZK_PREFIX + "acl";
public static final String DEFAULT_RM_ZK_ACL = "world:anyone:rwcda";
+ public static final String RM_ZK_AUTH = RM_ZK_PREFIX + "auth";
+
public static final String ZK_STATE_STORE_PREFIX =
RM_PREFIX + "zk-state-store.";
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java?rev=1573017&r1=1573016&r2=1573017&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java Fri Feb 28 17:36:04 2014
@@ -31,14 +31,12 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.util.ZKUtil;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.ACL;
import java.io.IOException;
-import java.util.Collections;
import java.util.List;
@InterfaceAudience.Private
@@ -88,18 +86,8 @@ public class EmbeddedElectorService exte
long zkSessionTimeout = conf.getLong(YarnConfiguration.RM_ZK_TIMEOUT_MS,
YarnConfiguration.DEFAULT_RM_ZK_TIMEOUT_MS);
- String zkAclConf = conf.get(YarnConfiguration.RM_ZK_ACL,
- YarnConfiguration.DEFAULT_RM_ZK_ACL);
- List<ACL> zkAcls;
- try {
- zkAcls = ZKUtil.parseACLs(ZKUtil.resolveConfIndirection(zkAclConf));
- } catch (ZKUtil.BadAclFormatException bafe) {
- throw new YarnRuntimeException(
- YarnConfiguration.RM_ZK_ACL + "has ill-formatted ACLs");
- }
-
- // TODO (YARN-1528): ZKAuthInfo to be set for rm-store and elector
- List<ZKUtil.ZKAuthInfo> zkAuths = Collections.emptyList();
+ List<ACL> zkAcls = RMZKUtils.getZKAcls(conf);
+ List<ZKUtil.ZKAuthInfo> zkAuths = RMZKUtils.getZKAuths(conf);
elector = new ActiveStandbyElector(zkQuorum, (int) zkSessionTimeout,
electionZNode, zkAcls, zkAuths, this);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java?rev=1573017&r1=1573016&r2=1573017&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java Fri Feb 28 17:36:04 2014
@@ -48,6 +48,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ApplicationStateDataProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RMStateVersionProto;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.RMZKUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.RMStateVersion;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -91,6 +92,7 @@ public class ZKRMStateStore extends RMSt
private int zkSessionTimeout;
private long zkRetryInterval;
private List<ACL> zkAcl;
+ private List<ZKUtil.ZKAuthInfo> zkAuths;
/**
*
@@ -200,18 +202,9 @@ public class ZKRMStateStore extends RMSt
zkRetryInterval =
conf.getLong(YarnConfiguration.RM_ZK_RETRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_ZK_RETRY_INTERVAL_MS);
- // Parse authentication from configuration.
- String zkAclConf =
- conf.get(YarnConfiguration.RM_ZK_ACL,
- YarnConfiguration.DEFAULT_RM_ZK_ACL);
- zkAclConf = ZKUtil.resolveConfIndirection(zkAclConf);
- try {
- zkAcl = ZKUtil.parseACLs(zkAclConf);
- } catch (ZKUtil.BadAclFormatException bafe) {
- LOG.error("Invalid format for " + YarnConfiguration.RM_ZK_ACL);
- throw bafe;
- }
+ zkAcl = RMZKUtils.getZKAcls(conf);
+ zkAuths = RMZKUtils.getZKAuths(conf);
zkRootNodePath = getNodePath(znodeWorkingPath, ROOT_ZNODE_NAME);
rmAppRoot = getNodePath(zkRootNodePath, RM_APP_ROOT);
@@ -952,6 +945,9 @@ public class ZKRMStateStore extends RMSt
retries++) {
try {
zkClient = getNewZooKeeper();
+ for (ZKUtil.ZKAuthInfo zkAuth : zkAuths) {
+ zkClient.addAuthInfo(zkAuth.getScheme(), zkAuth.getAuth());
+ }
if (useDefaultFencingScheme) {
zkClient.addAuthInfo(zkRootNodeAuthScheme,
(zkRootNodeUsername + ":" + zkRootNodePassword).getBytes());
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java?rev=1573017&r1=1573016&r2=1573017&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java Fri Feb 28 17:36:04 2014
@@ -32,10 +32,12 @@ import org.apache.zookeeper.WatchedEvent
import org.apache.zookeeper.Watcher;
import org.apache.zookeeper.ZooDefs;
import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.server.auth.DigestAuthenticationProvider;
import org.junit.Assert;
import org.junit.Test;
import java.io.IOException;
+import java.security.NoSuchAlgorithmException;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -49,6 +51,20 @@ public class TestZKRMStateStoreZKClientC
private Log LOG =
LogFactory.getLog(TestZKRMStateStoreZKClientConnections.class);
+ private static final String DIGEST_USER_PASS="test-user:test-password";
+ private static final String TEST_AUTH_GOOD = "digest:" + DIGEST_USER_PASS;
+ private static final String DIGEST_USER_HASH;
+ static {
+ try {
+ DIGEST_USER_HASH = DigestAuthenticationProvider.generateDigest(
+ DIGEST_USER_PASS);
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ private static final String TEST_ACL = "digest:" + DIGEST_USER_HASH + ":rwcda";
+
+
class TestZKClient {
ZKRMStateStore store;
@@ -252,4 +268,16 @@ public class TestZKRMStateStoreZKClientC
fail(error);
}
}
+
+ @Test
+ public void testZKAuths() throws Exception {
+ TestZKClient zkClientTester = new TestZKClient();
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setInt(YarnConfiguration.RM_ZK_NUM_RETRIES, 1);
+ conf.setInt(YarnConfiguration.RM_ZK_TIMEOUT_MS, 100);
+ conf.set(YarnConfiguration.RM_ZK_ACL, TEST_ACL);
+ conf.set(YarnConfiguration.RM_ZK_AUTH, TEST_AUTH_GOOD);
+
+ zkClientTester.getRMStateStore(conf);
+ }
}