You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/05/15 23:58:46 UTC
[39/50] [abbrv] hadoop git commit: YARN-8236. Invalid kerberos
principal file name cause NPE in native service. Contributed by Gour Saha.
YARN-8236. Invalid kerberos principal file name cause NPE in native service. Contributed by Gour Saha.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58b97c79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58b97c79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58b97c79
Branch: refs/heads/HDDS-4
Commit: 58b97c79e34901938d59acc84ed48c1f9344996a
Parents: ffb9210
Author: Sunil G <su...@apache.org>
Authored: Tue May 15 12:17:35 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue May 15 12:17:35 2018 +0530
----------------------------------------------------------------------
.../yarn/service/client/ServiceClient.java | 46 +++++++++++---------
.../exceptions/RestApiErrorMessages.java | 2 +
.../yarn/service/utils/ServiceApiUtil.java | 38 ++++++++++------
.../hadoop/yarn/service/TestServiceApiUtil.java | 41 +++++++++++++++++
4 files changed, 93 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b97c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 67306d2..364a94c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -1065,7 +1065,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
LOG.warn("No Kerberos principal name specified for " + service.getName());
return;
}
- if(StringUtils.isEmpty(service.getKerberosPrincipal().getKeytab())) {
+ if (StringUtils.isEmpty(service.getKerberosPrincipal().getKeytab())) {
LOG.warn("No Kerberos keytab specified for " + service.getName());
return;
}
@@ -1077,27 +1077,31 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
throw new YarnException(e);
}
- switch (keytabURI.getScheme()) {
- case "hdfs":
- Path keytabOnhdfs = new Path(keytabURI);
- if (!fileSystem.getFileSystem().exists(keytabOnhdfs)) {
- LOG.warn(service.getName() + "'s keytab (principalName = " +
- principalName + ") doesn't exist at: " + keytabOnhdfs);
- return;
+ if (keytabURI.getScheme() != null) {
+ switch (keytabURI.getScheme()) {
+ case "hdfs":
+ Path keytabOnhdfs = new Path(keytabURI);
+ if (!fileSystem.getFileSystem().exists(keytabOnhdfs)) {
+ LOG.warn(service.getName() + "'s keytab (principalName = "
+ + principalName + ") doesn't exist at: " + keytabOnhdfs);
+ return;
+ }
+ LocalResource keytabRes = fileSystem.createAmResource(keytabOnhdfs,
+ LocalResourceType.FILE);
+ localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION,
+ service.getName()), keytabRes);
+ LOG.info("Adding " + service.getName() + "'s keytab for "
+ + "localization, uri = " + keytabOnhdfs);
+ break;
+ case "file":
+ LOG.info("Using a keytab from localhost: " + keytabURI);
+ break;
+ default:
+ LOG.warn("Unsupported keytab URI scheme " + keytabURI);
+ break;
}
- LocalResource keytabRes =
- fileSystem.createAmResource(keytabOnhdfs, LocalResourceType.FILE);
- localResource.put(String.format(YarnServiceConstants.KEYTAB_LOCATION,
- service.getName()), keytabRes);
- LOG.debug("Adding " + service.getName() + "'s keytab for " +
- "localization, uri = " + keytabOnhdfs);
- break;
- case "file":
- LOG.debug("Using a keytab from localhost: " + keytabURI);
- break;
- default:
- LOG.warn("Unsupported URI scheme " + keytabURI);
- break;
+ } else {
+ LOG.warn("Unsupported keytab URI scheme " + keytabURI);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b97c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
index 6b2b8af..0e42533 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/exceptions/RestApiErrorMessages.java
@@ -103,4 +103,6 @@ public interface RestApiErrorMessages {
+ "expression element name %s specified in placement policy of component "
+ "%s. Expression element names should be a valid constraint name or an "
+ "expression name defined for this component only.";
+ String ERROR_KEYTAB_URI_SCHEME_INVALID = "Unsupported keytab URI scheme: %s";
+ String ERROR_KEYTAB_URI_INVALID = "Invalid keytab URI: %s";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b97c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index a4e5c0d..6e62c56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -29,13 +29,14 @@ import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.service.api.records.Container;
-import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Configuration;
+import org.apache.hadoop.yarn.service.api.records.Container;
+import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal;
import org.apache.hadoop.yarn.service.api.records.PlacementConstraint;
import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.conf.RestApiConstants;
import org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages;
@@ -111,14 +112,7 @@ public class ServiceApiUtil {
}
if (UserGroupInformation.isSecurityEnabled()) {
- if (!StringUtils.isEmpty(service.getKerberosPrincipal().getKeytab())) {
- try {
- // validate URI format
- new URI(service.getKerberosPrincipal().getKeytab());
- } catch (URISyntaxException e) {
- throw new IllegalArgumentException(e);
- }
- }
+ validateKerberosPrincipal(service.getKerberosPrincipal());
}
// Validate the Docker client config.
@@ -145,9 +139,8 @@ public class ServiceApiUtil {
throw new IllegalArgumentException("Component name collision: " +
comp.getName());
}
- // If artifact is of type SERVICE (which cannot be filled from
- // global), read external service and add its components to this
- // service
+ // If artifact is of type SERVICE (which cannot be filled from global),
+ // read external service and add its components to this service
if (comp.getArtifact() != null && comp.getArtifact().getType() ==
Artifact.TypeEnum.SERVICE) {
if (StringUtils.isEmpty(comp.getArtifact().getId())) {
@@ -226,6 +219,25 @@ public class ServiceApiUtil {
}
}
+ public static void validateKerberosPrincipal(
+ KerberosPrincipal kerberosPrincipal) throws IOException {
+ if (!StringUtils.isEmpty(kerberosPrincipal.getKeytab())) {
+ try {
+ // validate URI format
+ URI keytabURI = new URI(kerberosPrincipal.getKeytab());
+ if (keytabURI.getScheme() == null) {
+ throw new IllegalArgumentException(String.format(
+ RestApiErrorMessages.ERROR_KEYTAB_URI_SCHEME_INVALID,
+ kerberosPrincipal.getKeytab()));
+ }
+ } catch (URISyntaxException e) {
+ throw new IllegalArgumentException(
+ String.format(RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID,
+ e.getLocalizedMessage()));
+ }
+ }
+ }
+
private static void validateDockerClientConfiguration(Service service,
org.apache.hadoop.conf.Configuration conf) throws IOException {
String dockerClientConfig = service.getDockerClientConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58b97c79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
index d195b2c..b209bbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceApiUtil.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.KerberosPrincipal;
import org.apache.hadoop.yarn.service.api.records.PlacementConstraint;
import org.apache.hadoop.yarn.service.api.records.PlacementPolicy;
import org.apache.hadoop.yarn.service.api.records.Resource;
@@ -45,6 +46,7 @@ import static org.apache.hadoop.yarn.service.conf.RestApiConstants.DEFAULT_UNLIM
import static org.apache.hadoop.yarn.service.exceptions.RestApiErrorMessages.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
/**
* Test for ServiceApiUtil helper methods.
@@ -525,4 +527,43 @@ public class TestServiceApiUtil {
Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
}
}
+
+ @Test
+ public void testKerberosPrincipal() throws IOException {
+ SliderFileSystem sfs = ServiceTestUtils.initMockFs();
+ Service app = createValidApplication("comp-a");
+ KerberosPrincipal kp = new KerberosPrincipal();
+ kp.setKeytab("/some/path");
+ kp.setPrincipalName("user/_HOST@domain.com");
+ app.setKerberosPrincipal(kp);
+
+ try {
+ ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal());
+ Assert.fail(EXCEPTION_PREFIX + "service with invalid keytab URI scheme");
+ } catch (IllegalArgumentException e) {
+ assertEquals(
+ String.format(RestApiErrorMessages.ERROR_KEYTAB_URI_SCHEME_INVALID,
+ kp.getKeytab()),
+ e.getMessage());
+ }
+
+ kp.setKeytab("/ blank / in / paths");
+ try {
+ ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal());
+ Assert.fail(EXCEPTION_PREFIX + "service with invalid keytab");
+ } catch (IllegalArgumentException e) {
+ // strip out the %s at the end of the RestApiErrorMessages string constant
+ assertTrue(e.getMessage().contains(
+ RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID.substring(0,
+ RestApiErrorMessages.ERROR_KEYTAB_URI_INVALID.length() - 2)));
+ }
+
+ kp.setKeytab("file:///tmp/a.keytab");
+ // now it should succeed
+ try {
+ ServiceApiUtil.validateKerberosPrincipal(app.getKerberosPrincipal());
+ } catch (IllegalArgumentException e) {
+ Assert.fail(NO_EXCEPTION_PREFIX + e.getMessage());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org