You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2014/10/16 23:23:52 UTC
[1/3] git commit: YARN-2689 TestSecureRMRegistryOperations failing on
windows: secure ZK won't start
Repository: hadoop
Updated Branches:
refs/heads/branch-2 aef8dbde9 -> fddbf52ca
refs/heads/branch-2.6 fd036896a -> b0a72f354
refs/heads/trunk 289443333 -> 6f43491c0
YARN-2689 TestSecureRMRegistryOperations failing on windows: secure ZK won't start
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0a72f35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0a72f35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0a72f35
Branch: refs/heads/branch-2.6
Commit: b0a72f354bcfe68fd227323a52e916adc6bd3b1e
Parents: fd03689
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 16 14:21:38 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 16 14:21:38 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 5 +++
.../registry/client/impl/zk/CuratorService.java | 12 +++----
.../client/impl/zk/RegistrySecurity.java | 16 +++++----
.../secure/AbstractSecureRegistryTest.java | 8 ++++-
.../registry/secure/TestSecureLogins.java | 25 ++++++++++----
.../registry/secure/TestSecureRegistry.java | 34 ++++++++++++++++++++
6 files changed, 81 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 404d066..9f72c0d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -573,6 +573,11 @@ Release 2.6.0 - UNRELEASED
YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
YARN-2668 yarn-registry JAR won't link against ZK 3.4.5. (stevel)
+
+ YARN-2689 TestSecureRMRegistryOperations failing on windows:
+ secure ZK won't start (stevel)
+
+ ---
YARN-2598 GHS should show N/A instead of null for the inaccessible information
(Zhijie Shen via mayank)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index a0e6365..0b68b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -249,9 +249,6 @@ public class CuratorService extends CompositeService
synchronized (CuratorService.class) {
// set the security options
- //log them
- securityConnectionDiagnostics = buildSecurityDiagnostics();
-
// build up the curator itself
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
builder.ensembleProvider(ensembleProvider)
@@ -264,7 +261,8 @@ public class CuratorService extends CompositeService
// set up the builder AND any JVM context
registrySecurity.applySecurityEnvironment(builder);
-
+ //log them
+ securityConnectionDiagnostics = buildSecurityDiagnostics();
framework = builder.build();
framework.start();
}
@@ -275,7 +273,7 @@ public class CuratorService extends CompositeService
@Override
public String toString() {
return super.toString()
- + bindingDiagnosticDetails();
+ + " " + bindingDiagnosticDetails();
}
/**
@@ -386,7 +384,9 @@ public class CuratorService extends CompositeService
ioe = new PathIsNotEmptyDirectoryException(path);
} else if (exception instanceof KeeperException.AuthFailedException) {
ioe = new AuthenticationFailedException(path,
- "Authentication Failed: " + exception, exception);
+ "Authentication Failed: " + exception
+ + "; " + securityConnectionDiagnostics,
+ exception);
} else if (exception instanceof KeeperException.NoChildrenForEphemeralsException) {
ioe = new NoChildrenForEphemeralsException(path,
"Cannot create a path under an ephemeral node: " + exception,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 6d5792e..5370880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -596,6 +596,7 @@ public class RegistrySecurity extends AbstractService {
+ " %s required\n"
// kerberos module
+ " keyTab=\"%s\"\n"
+ + " debug=true\n"
+ " principal=\"%s\"\n"
+ " useKeyTab=true\n"
+ " useTicketCache=false\n"
@@ -621,12 +622,15 @@ public class RegistrySecurity extends AbstractService {
"invalid context");
Preconditions.checkArgument(keytab != null && keytab.isFile(),
"Keytab null or missing: ");
+ String keytabpath = keytab.getAbsolutePath();
+ // fix up for windows; no-op on unix
+ keytabpath = keytabpath.replace('\\', '/');
return String.format(
Locale.ENGLISH,
JAAS_ENTRY,
context,
getKerberosAuthModuleForJVM(),
- keytab.getAbsolutePath(),
+ keytabpath,
principal);
}
@@ -846,11 +850,11 @@ public class RegistrySecurity extends AbstractService {
StringBuilder builder = new StringBuilder();
builder.append(secureRegistry ? "secure registry; "
: "insecure registry; ");
- builder.append("Access policy: ").append(access);
+ builder.append("Curator service access policy: ").append(access);
- builder.append(", System ACLs: ").append(aclsToString(systemACLs));
- builder.append(UgiInfo.fromCurrentUser());
- builder.append(" Kerberos Realm: ").append(kerberosRealm).append(" ; ");
+ builder.append("; System ACLs: ").append(aclsToString(systemACLs));
+ builder.append("User: ").append(UgiInfo.fromCurrentUser());
+ builder.append("; Kerberos Realm: ").append(kerberosRealm);
builder.append(describeProperty(Environment.JAAS_CONF_KEY));
String sasl =
System.getProperty(PROP_ZK_ENABLE_SASL_CLIENT,
@@ -859,7 +863,7 @@ public class RegistrySecurity extends AbstractService {
builder.append(describeProperty(PROP_ZK_ENABLE_SASL_CLIENT,
DEFAULT_ZK_ENABLE_SASL_CLIENT));
if (saslEnabled) {
- builder.append("JAAS Client Identity")
+ builder.append("; JAAS Client Identity")
.append("=")
.append(jaasClientIdentity)
.append("; ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
index ca3f9c9..7fdd261 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
@@ -46,6 +46,7 @@ import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.Principal;
import java.util.HashSet;
@@ -319,11 +320,16 @@ public class AbstractSecureRegistryTest extends RegistryTestHelper {
* @param keytab keytab
* @return the logged in context
* @throws LoginException failure to log in
+ * @throws FileNotFoundException no keytab
*/
protected LoginContext login(String principal,
- String context, File keytab) throws LoginException {
+ String context, File keytab) throws LoginException,
+ FileNotFoundException {
LOG.info("Logging in as {} in context {} with keytab {}",
principal, context, keytab);
+ if (!keytab.exists()) {
+ throw new FileNotFoundException(keytab.getAbsolutePath());
+ }
Set<Principal> principals = new HashSet<Principal>();
principals.add(new KerberosPrincipal(principal));
Subject subject = new Subject(false, principals, new HashSet<Object>(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
index ab9d490..9a90a45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
@@ -99,12 +99,25 @@ public class TestSecureLogins extends AbstractSecureRegistryTest {
ALICE_CLIENT_CONTEXT,
keytab_alice);
- logLoginDetails(ALICE_LOCALHOST, client);
- String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
- assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
- String config = FileUtils.readFileToString(new File(confFilename));
- LOG.info("{}=\n{}", confFilename, config);
- RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ try {
+ logLoginDetails(ALICE_LOCALHOST, client);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
+ String config = FileUtils.readFileToString(new File(confFilename));
+ LOG.info("{}=\n{}", confFilename, config);
+ RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ } finally {
+ client.logout();
+ }
+ }
+
+ @Test
+ public void testZKServerContextLogin() throws Throwable {
+ LoginContext client = login(ZOOKEEPER_LOCALHOST,
+ ZOOKEEPER_SERVER_CONTEXT,
+ keytab_zk);
+ logLoginDetails(ZOOKEEPER_LOCALHOST, client);
+
client.logout();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0a72f35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
index 2dad4bd..083f7f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
@@ -24,12 +24,16 @@ import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
import org.apache.hadoop.registry.client.impl.zk.CuratorService;
import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Login;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+import org.apache.zookeeper.server.auth.SaslServerCallbackHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
@@ -52,6 +56,36 @@ public class TestSecureRegistry extends AbstractSecureRegistryTest {
RegistrySecurity.clearZKSaslClientProperties();
}
+ /**
+ * this is a cut and paste of some of the ZK internal code that was
+ * failing on windows and swallowing its exceptions
+ */
+ @Test
+ public void testLowlevelZKSaslLogin() throws Throwable {
+ RegistrySecurity.bindZKToServerJAASContext(ZOOKEEPER_SERVER_CONTEXT);
+ String serverSection =
+ System.getProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
+ ZooKeeperSaslServer.DEFAULT_LOGIN_CONTEXT_NAME);
+ assertEquals(ZOOKEEPER_SERVER_CONTEXT, serverSection);
+
+ AppConfigurationEntry entries[];
+ entries = javax.security.auth.login.Configuration.getConfiguration()
+ .getAppConfigurationEntry(
+ serverSection);
+
+ assertNotNull("null entries", entries);
+
+ SaslServerCallbackHandler saslServerCallbackHandler =
+ new SaslServerCallbackHandler(
+ javax.security.auth.login.Configuration.getConfiguration());
+ Login login = new Login(serverSection, saslServerCallbackHandler);
+ try {
+ login.startThreadIfNeeded();
+ } finally {
+ login.shutdown();
+ }
+ }
+
@Test
public void testCreateSecureZK() throws Throwable {
startSecureZK();
[2/3] git commit: YARN-2689 TestSecureRMRegistryOperations failing on
windows: secure ZK won't start
Posted by st...@apache.org.
YARN-2689 TestSecureRMRegistryOperations failing on windows: secure ZK won't start
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fddbf52c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fddbf52c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fddbf52c
Branch: refs/heads/branch-2
Commit: fddbf52caa91af28f2764f0b2f80916c522bde39
Parents: aef8dbd
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 16 14:21:38 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 16 14:21:49 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 5 +++
.../registry/client/impl/zk/CuratorService.java | 12 +++----
.../client/impl/zk/RegistrySecurity.java | 16 +++++----
.../secure/AbstractSecureRegistryTest.java | 8 ++++-
.../registry/secure/TestSecureLogins.java | 25 ++++++++++----
.../registry/secure/TestSecureRegistry.java | 34 ++++++++++++++++++++
6 files changed, 81 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c5321a6..284842c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -609,6 +609,11 @@ Release 2.6.0 - UNRELEASED
YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
YARN-2668 yarn-registry JAR won't link against ZK 3.4.5. (stevel)
+
+ YARN-2689 TestSecureRMRegistryOperations failing on windows:
+ secure ZK won't start (stevel)
+
+ ---
YARN-2598 GHS should show N/A instead of null for the inaccessible information
(Zhijie Shen via mayank)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index a0e6365..0b68b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -249,9 +249,6 @@ public class CuratorService extends CompositeService
synchronized (CuratorService.class) {
// set the security options
- //log them
- securityConnectionDiagnostics = buildSecurityDiagnostics();
-
// build up the curator itself
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
builder.ensembleProvider(ensembleProvider)
@@ -264,7 +261,8 @@ public class CuratorService extends CompositeService
// set up the builder AND any JVM context
registrySecurity.applySecurityEnvironment(builder);
-
+ //log them
+ securityConnectionDiagnostics = buildSecurityDiagnostics();
framework = builder.build();
framework.start();
}
@@ -275,7 +273,7 @@ public class CuratorService extends CompositeService
@Override
public String toString() {
return super.toString()
- + bindingDiagnosticDetails();
+ + " " + bindingDiagnosticDetails();
}
/**
@@ -386,7 +384,9 @@ public class CuratorService extends CompositeService
ioe = new PathIsNotEmptyDirectoryException(path);
} else if (exception instanceof KeeperException.AuthFailedException) {
ioe = new AuthenticationFailedException(path,
- "Authentication Failed: " + exception, exception);
+ "Authentication Failed: " + exception
+ + "; " + securityConnectionDiagnostics,
+ exception);
} else if (exception instanceof KeeperException.NoChildrenForEphemeralsException) {
ioe = new NoChildrenForEphemeralsException(path,
"Cannot create a path under an ephemeral node: " + exception,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 6d5792e..5370880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -596,6 +596,7 @@ public class RegistrySecurity extends AbstractService {
+ " %s required\n"
// kerberos module
+ " keyTab=\"%s\"\n"
+ + " debug=true\n"
+ " principal=\"%s\"\n"
+ " useKeyTab=true\n"
+ " useTicketCache=false\n"
@@ -621,12 +622,15 @@ public class RegistrySecurity extends AbstractService {
"invalid context");
Preconditions.checkArgument(keytab != null && keytab.isFile(),
"Keytab null or missing: ");
+ String keytabpath = keytab.getAbsolutePath();
+ // fix up for windows; no-op on unix
+ keytabpath = keytabpath.replace('\\', '/');
return String.format(
Locale.ENGLISH,
JAAS_ENTRY,
context,
getKerberosAuthModuleForJVM(),
- keytab.getAbsolutePath(),
+ keytabpath,
principal);
}
@@ -846,11 +850,11 @@ public class RegistrySecurity extends AbstractService {
StringBuilder builder = new StringBuilder();
builder.append(secureRegistry ? "secure registry; "
: "insecure registry; ");
- builder.append("Access policy: ").append(access);
+ builder.append("Curator service access policy: ").append(access);
- builder.append(", System ACLs: ").append(aclsToString(systemACLs));
- builder.append(UgiInfo.fromCurrentUser());
- builder.append(" Kerberos Realm: ").append(kerberosRealm).append(" ; ");
+ builder.append("; System ACLs: ").append(aclsToString(systemACLs));
+ builder.append("User: ").append(UgiInfo.fromCurrentUser());
+ builder.append("; Kerberos Realm: ").append(kerberosRealm);
builder.append(describeProperty(Environment.JAAS_CONF_KEY));
String sasl =
System.getProperty(PROP_ZK_ENABLE_SASL_CLIENT,
@@ -859,7 +863,7 @@ public class RegistrySecurity extends AbstractService {
builder.append(describeProperty(PROP_ZK_ENABLE_SASL_CLIENT,
DEFAULT_ZK_ENABLE_SASL_CLIENT));
if (saslEnabled) {
- builder.append("JAAS Client Identity")
+ builder.append("; JAAS Client Identity")
.append("=")
.append(jaasClientIdentity)
.append("; ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
index ca3f9c9..7fdd261 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
@@ -46,6 +46,7 @@ import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.Principal;
import java.util.HashSet;
@@ -319,11 +320,16 @@ public class AbstractSecureRegistryTest extends RegistryTestHelper {
* @param keytab keytab
* @return the logged in context
* @throws LoginException failure to log in
+ * @throws FileNotFoundException no keytab
*/
protected LoginContext login(String principal,
- String context, File keytab) throws LoginException {
+ String context, File keytab) throws LoginException,
+ FileNotFoundException {
LOG.info("Logging in as {} in context {} with keytab {}",
principal, context, keytab);
+ if (!keytab.exists()) {
+ throw new FileNotFoundException(keytab.getAbsolutePath());
+ }
Set<Principal> principals = new HashSet<Principal>();
principals.add(new KerberosPrincipal(principal));
Subject subject = new Subject(false, principals, new HashSet<Object>(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
index ab9d490..9a90a45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
@@ -99,12 +99,25 @@ public class TestSecureLogins extends AbstractSecureRegistryTest {
ALICE_CLIENT_CONTEXT,
keytab_alice);
- logLoginDetails(ALICE_LOCALHOST, client);
- String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
- assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
- String config = FileUtils.readFileToString(new File(confFilename));
- LOG.info("{}=\n{}", confFilename, config);
- RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ try {
+ logLoginDetails(ALICE_LOCALHOST, client);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
+ String config = FileUtils.readFileToString(new File(confFilename));
+ LOG.info("{}=\n{}", confFilename, config);
+ RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ } finally {
+ client.logout();
+ }
+ }
+
+ @Test
+ public void testZKServerContextLogin() throws Throwable {
+ LoginContext client = login(ZOOKEEPER_LOCALHOST,
+ ZOOKEEPER_SERVER_CONTEXT,
+ keytab_zk);
+ logLoginDetails(ZOOKEEPER_LOCALHOST, client);
+
client.logout();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddbf52c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
index 2dad4bd..083f7f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
@@ -24,12 +24,16 @@ import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
import org.apache.hadoop.registry.client.impl.zk.CuratorService;
import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Login;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+import org.apache.zookeeper.server.auth.SaslServerCallbackHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
@@ -52,6 +56,36 @@ public class TestSecureRegistry extends AbstractSecureRegistryTest {
RegistrySecurity.clearZKSaslClientProperties();
}
+ /**
+ * this is a cut and paste of some of the ZK internal code that was
+ * failing on windows and swallowing its exceptions
+ */
+ @Test
+ public void testLowlevelZKSaslLogin() throws Throwable {
+ RegistrySecurity.bindZKToServerJAASContext(ZOOKEEPER_SERVER_CONTEXT);
+ String serverSection =
+ System.getProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
+ ZooKeeperSaslServer.DEFAULT_LOGIN_CONTEXT_NAME);
+ assertEquals(ZOOKEEPER_SERVER_CONTEXT, serverSection);
+
+ AppConfigurationEntry entries[];
+ entries = javax.security.auth.login.Configuration.getConfiguration()
+ .getAppConfigurationEntry(
+ serverSection);
+
+ assertNotNull("null entries", entries);
+
+ SaslServerCallbackHandler saslServerCallbackHandler =
+ new SaslServerCallbackHandler(
+ javax.security.auth.login.Configuration.getConfiguration());
+ Login login = new Login(serverSection, saslServerCallbackHandler);
+ try {
+ login.startThreadIfNeeded();
+ } finally {
+ login.shutdown();
+ }
+ }
+
@Test
public void testCreateSecureZK() throws Throwable {
startSecureZK();
[3/3] git commit: YARN-2689 TestSecureRMRegistryOperations failing on
windows: secure ZK won't start
Posted by st...@apache.org.
YARN-2689 TestSecureRMRegistryOperations failing on windows: secure ZK won't start
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f43491c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f43491c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f43491c
Branch: refs/heads/trunk
Commit: 6f43491c0343cfef36e9be5dfd06447cf2fee377
Parents: 2894433
Author: Steve Loughran <st...@apache.org>
Authored: Thu Oct 16 14:21:38 2014 -0700
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Oct 16 14:22:02 2014 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 5 +++
.../registry/client/impl/zk/CuratorService.java | 12 +++----
.../client/impl/zk/RegistrySecurity.java | 16 +++++----
.../secure/AbstractSecureRegistryTest.java | 8 ++++-
.../registry/secure/TestSecureLogins.java | 25 ++++++++++----
.../registry/secure/TestSecureRegistry.java | 34 ++++++++++++++++++++
6 files changed, 81 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b0e307c..f85735e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -639,6 +639,11 @@ Release 2.6.0 - UNRELEASED
YARN-2652 Add hadoop-yarn-registry package under hadoop-yarn. (stevel)
YARN-2668 yarn-registry JAR won't link against ZK 3.4.5. (stevel)
+
+ YARN-2689 TestSecureRMRegistryOperations failing on windows:
+ secure ZK won't start (stevel)
+
+ ---
YARN-2598 GHS should show N/A instead of null for the inaccessible information
(Zhijie Shen via mayank)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
index a0e6365..0b68b0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/CuratorService.java
@@ -249,9 +249,6 @@ public class CuratorService extends CompositeService
synchronized (CuratorService.class) {
// set the security options
- //log them
- securityConnectionDiagnostics = buildSecurityDiagnostics();
-
// build up the curator itself
CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
builder.ensembleProvider(ensembleProvider)
@@ -264,7 +261,8 @@ public class CuratorService extends CompositeService
// set up the builder AND any JVM context
registrySecurity.applySecurityEnvironment(builder);
-
+ //log them
+ securityConnectionDiagnostics = buildSecurityDiagnostics();
framework = builder.build();
framework.start();
}
@@ -275,7 +273,7 @@ public class CuratorService extends CompositeService
@Override
public String toString() {
return super.toString()
- + bindingDiagnosticDetails();
+ + " " + bindingDiagnosticDetails();
}
/**
@@ -386,7 +384,9 @@ public class CuratorService extends CompositeService
ioe = new PathIsNotEmptyDirectoryException(path);
} else if (exception instanceof KeeperException.AuthFailedException) {
ioe = new AuthenticationFailedException(path,
- "Authentication Failed: " + exception, exception);
+ "Authentication Failed: " + exception
+ + "; " + securityConnectionDiagnostics,
+ exception);
} else if (exception instanceof KeeperException.NoChildrenForEphemeralsException) {
ioe = new NoChildrenForEphemeralsException(path,
"Cannot create a path under an ephemeral node: " + exception,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
index 6d5792e..5370880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/impl/zk/RegistrySecurity.java
@@ -596,6 +596,7 @@ public class RegistrySecurity extends AbstractService {
+ " %s required\n"
// kerberos module
+ " keyTab=\"%s\"\n"
+ + " debug=true\n"
+ " principal=\"%s\"\n"
+ " useKeyTab=true\n"
+ " useTicketCache=false\n"
@@ -621,12 +622,15 @@ public class RegistrySecurity extends AbstractService {
"invalid context");
Preconditions.checkArgument(keytab != null && keytab.isFile(),
"Keytab null or missing: ");
+ String keytabpath = keytab.getAbsolutePath();
+ // fix up for windows; no-op on unix
+ keytabpath = keytabpath.replace('\\', '/');
return String.format(
Locale.ENGLISH,
JAAS_ENTRY,
context,
getKerberosAuthModuleForJVM(),
- keytab.getAbsolutePath(),
+ keytabpath,
principal);
}
@@ -846,11 +850,11 @@ public class RegistrySecurity extends AbstractService {
StringBuilder builder = new StringBuilder();
builder.append(secureRegistry ? "secure registry; "
: "insecure registry; ");
- builder.append("Access policy: ").append(access);
+ builder.append("Curator service access policy: ").append(access);
- builder.append(", System ACLs: ").append(aclsToString(systemACLs));
- builder.append(UgiInfo.fromCurrentUser());
- builder.append(" Kerberos Realm: ").append(kerberosRealm).append(" ; ");
+ builder.append("; System ACLs: ").append(aclsToString(systemACLs));
+ builder.append("User: ").append(UgiInfo.fromCurrentUser());
+ builder.append("; Kerberos Realm: ").append(kerberosRealm);
builder.append(describeProperty(Environment.JAAS_CONF_KEY));
String sasl =
System.getProperty(PROP_ZK_ENABLE_SASL_CLIENT,
@@ -859,7 +863,7 @@ public class RegistrySecurity extends AbstractService {
builder.append(describeProperty(PROP_ZK_ENABLE_SASL_CLIENT,
DEFAULT_ZK_ENABLE_SASL_CLIENT));
if (saslEnabled) {
- builder.append("JAAS Client Identity")
+ builder.append("; JAAS Client Identity")
.append("=")
.append(jaasClientIdentity)
.append("; ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
index ca3f9c9..7fdd261 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/AbstractSecureRegistryTest.java
@@ -46,6 +46,7 @@ import javax.security.auth.kerberos.KerberosPrincipal;
import javax.security.auth.login.LoginContext;
import javax.security.auth.login.LoginException;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.Principal;
import java.util.HashSet;
@@ -319,11 +320,16 @@ public class AbstractSecureRegistryTest extends RegistryTestHelper {
* @param keytab keytab
* @return the logged in context
* @throws LoginException failure to log in
+ * @throws FileNotFoundException no keytab
*/
protected LoginContext login(String principal,
- String context, File keytab) throws LoginException {
+ String context, File keytab) throws LoginException,
+ FileNotFoundException {
LOG.info("Logging in as {} in context {} with keytab {}",
principal, context, keytab);
+ if (!keytab.exists()) {
+ throw new FileNotFoundException(keytab.getAbsolutePath());
+ }
Set<Principal> principals = new HashSet<Principal>();
principals.add(new KerberosPrincipal(principal));
Subject subject = new Subject(false, principals, new HashSet<Object>(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
index ab9d490..9a90a45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureLogins.java
@@ -99,12 +99,25 @@ public class TestSecureLogins extends AbstractSecureRegistryTest {
ALICE_CLIENT_CONTEXT,
keytab_alice);
- logLoginDetails(ALICE_LOCALHOST, client);
- String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
- assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
- String config = FileUtils.readFileToString(new File(confFilename));
- LOG.info("{}=\n{}", confFilename, config);
- RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ try {
+ logLoginDetails(ALICE_LOCALHOST, client);
+ String confFilename = System.getProperty(Environment.JAAS_CONF_KEY);
+ assertNotNull("Unset: "+ Environment.JAAS_CONF_KEY, confFilename);
+ String config = FileUtils.readFileToString(new File(confFilename));
+ LOG.info("{}=\n{}", confFilename, config);
+ RegistrySecurity.setZKSaslClientProperties(ALICE, ALICE_CLIENT_CONTEXT);
+ } finally {
+ client.logout();
+ }
+ }
+
+ @Test
+ public void testZKServerContextLogin() throws Throwable {
+ LoginContext client = login(ZOOKEEPER_LOCALHOST,
+ ZOOKEEPER_SERVER_CONTEXT,
+ keytab_zk);
+ logLoginDetails(ZOOKEEPER_LOCALHOST, client);
+
client.logout();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f43491c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
index 2dad4bd..083f7f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/secure/TestSecureRegistry.java
@@ -24,12 +24,16 @@ import org.apache.hadoop.registry.client.impl.zk.ZKPathDumper;
import org.apache.hadoop.registry.client.impl.zk.CuratorService;
import org.apache.hadoop.registry.client.impl.zk.RegistrySecurity;
import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Login;
+import org.apache.zookeeper.server.ZooKeeperSaslServer;
+import org.apache.zookeeper.server.auth.SaslServerCallbackHandler;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
import static org.apache.hadoop.registry.client.api.RegistryConstants.*;
@@ -52,6 +56,36 @@ public class TestSecureRegistry extends AbstractSecureRegistryTest {
RegistrySecurity.clearZKSaslClientProperties();
}
+ /**
+ * this is a cut and paste of some of the ZK internal code that was
+ * failing on windows and swallowing its exceptions
+ */
+ @Test
+ public void testLowlevelZKSaslLogin() throws Throwable {
+ RegistrySecurity.bindZKToServerJAASContext(ZOOKEEPER_SERVER_CONTEXT);
+ String serverSection =
+ System.getProperty(ZooKeeperSaslServer.LOGIN_CONTEXT_NAME_KEY,
+ ZooKeeperSaslServer.DEFAULT_LOGIN_CONTEXT_NAME);
+ assertEquals(ZOOKEEPER_SERVER_CONTEXT, serverSection);
+
+ AppConfigurationEntry entries[];
+ entries = javax.security.auth.login.Configuration.getConfiguration()
+ .getAppConfigurationEntry(
+ serverSection);
+
+ assertNotNull("null entries", entries);
+
+ SaslServerCallbackHandler saslServerCallbackHandler =
+ new SaslServerCallbackHandler(
+ javax.security.auth.login.Configuration.getConfiguration());
+ Login login = new Login(serverSection, saslServerCallbackHandler);
+ try {
+ login.startThreadIfNeeded();
+ } finally {
+ login.shutdown();
+ }
+ }
+
@Test
public void testCreateSecureZK() throws Throwable {
startSecureZK();