You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2018/01/24 22:20:59 UTC
hadoop git commit: YARN-7777. Fix user name format in YARN Registry
DNS name. Contributed by Jian He
Repository: hadoop
Updated Branches:
refs/heads/trunk fa8cf4d1b -> 0c559b278
YARN-7777. Fix user name format in YARN Registry DNS name. Contributed by Jian He
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c559b27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c559b27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c559b27
Branch: refs/heads/trunk
Commit: 0c559b27820d12ebe5c315c6e2d1685c6de6bd33
Parents: fa8cf4d
Author: Billie Rinaldi <bi...@apache.org>
Authored: Wed Jan 24 13:50:47 2018 -0800
Committer: Billie Rinaldi <bi...@apache.org>
Committed: Wed Jan 24 13:50:47 2018 -0800
----------------------------------------------------------------------
.../hadoop/yarn/service/ServiceScheduler.java | 11 +++-------
.../yarn/service/utils/ServiceApiUtil.java | 15 +++++++++----
.../client/binding/RegistryPathUtils.java | 20 ++++++++++++++++++
.../registry/client/binding/RegistryUtils.java | 5 ++++-
.../server/dns/BaseServiceRecordProcessor.java | 22 ++------------------
.../client/binding/TestRegistryPathUtils.java | 10 ++++++++-
.../markdown/yarn-service/Configurations.md | 2 +-
7 files changed, 50 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 6cf4e14..dfe9808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -392,14 +392,9 @@ public class ServiceScheduler extends CompositeService {
// ZK
globalTokens.put(ServiceApiConstants.CLUSTER_ZK_QUORUM, getConfig()
.getTrimmed(KEY_REGISTRY_ZK_QUORUM, DEFAULT_REGISTRY_ZK_QUORUM));
- String user = null;
- try {
- user = UserGroupInformation.getCurrentUser().getShortUserName();
- } catch (IOException e) {
- LOG.error("Failed to get user.", e);
- }
- globalTokens
- .put(SERVICE_ZK_PATH, ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
+ String user = RegistryUtils.currentUser();
+ globalTokens.put(SERVICE_ZK_PATH,
+ ServiceRegistryUtils.mkServiceHomePath(user, app.getName()));
globalTokens.put(ServiceApiConstants.USER, user);
String dnsDomain = getConfig().getTrimmed(KEY_DNS_DOMAIN);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 7f85c95..a5716bd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -61,6 +61,9 @@ public class ServiceApiUtil {
private static final PatternValidator namePattern
= new PatternValidator("[a-z][a-z0-9-]*");
+ private static final PatternValidator userNamePattern
+ = new PatternValidator("[a-z][a-z0-9-.]*");
+
@VisibleForTesting
public static void setJsonSerDeser(JsonSerDeser jsd) {
jsonSerDeser = jsd;
@@ -72,11 +75,15 @@ public class ServiceApiUtil {
IOException {
boolean dnsEnabled = conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED,
RegistryConstants.DEFAULT_DNS_ENABLED);
- if (dnsEnabled && RegistryUtils.currentUser().length() > RegistryConstants
- .MAX_FQDN_LABEL_LENGTH) {
- throw new IllegalArgumentException(RestApiErrorMessages
- .ERROR_USER_NAME_INVALID);
+ if (dnsEnabled) {
+ if (RegistryUtils.currentUser().length()
+ > RegistryConstants.MAX_FQDN_LABEL_LENGTH) {
+ throw new IllegalArgumentException(
+ RestApiErrorMessages.ERROR_USER_NAME_INVALID);
+ }
+ userNamePattern.validate(RegistryUtils.currentUser());
}
+
if (StringUtils.isEmpty(service.getName())) {
throw new IllegalArgumentException(
RestApiErrorMessages.ERROR_APPLICATION_NAME_INVALID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
index 5fa45f9..b8e9ba1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryPathUtils.java
@@ -24,11 +24,13 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.registry.client.exceptions.InvalidPathnameException;
import org.apache.hadoop.registry.client.impl.zk.RegistryInternalConstants;
+import org.apache.hadoop.registry.server.dns.BaseServiceRecordProcessor;
import org.apache.zookeeper.common.PathUtils;
import java.net.IDN;
import java.util.ArrayList;
import java.util.List;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
@@ -45,6 +47,9 @@ public class RegistryPathUtils {
private static final Pattern PATH_ENTRY_VALIDATION_PATTERN =
Pattern.compile(RegistryInternalConstants.VALID_PATH_ENTRY_PATTERN);
+ private static final Pattern USER_NAME =
+ Pattern.compile("/users/([a-z][a-z0-9-.]*)");
+
/**
* Validate ZK path with the path itself included in
* the exception text
@@ -215,4 +220,19 @@ public class RegistryPathUtils {
public static String encodeYarnID(String yarnId) {
return yarnId.replace("container", "ctr").replace("_", "-");
}
+
+ /**
+ * Return the username found in the ZK path.
+ *
+ * @param recPath the ZK recPath.
+ * @return the user name.
+ */
+ public static String getUsername(String recPath) {
+ String user = "anonymous";
+ Matcher matcher = USER_NAME.matcher(recPath);
+ if (matcher.find()) {
+ user = matcher.group(1);
+ }
+ return user;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
index 858b6b1..4ef7b8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/client/binding/RegistryUtils.java
@@ -296,7 +296,10 @@ public class RegistryUtils {
*/
public static String currentUser() {
String shortUserName = currentUsernameUnencoded();
- return encodeForRegistry(shortUserName);
+ String encodedName = encodeForRegistry(shortUserName);
+ // DNS name doesn't allow "_", replace it with "-"
+ encodedName = RegistryUtils.convertUsername(encodedName);
+ return encodedName.replace("_", "-");
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
index fd5c74f..51ae99a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/server/dns/BaseServiceRecordProcessor.java
@@ -36,8 +36,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
/**
* Provides common service record processing logic.
@@ -51,7 +49,6 @@ public abstract class BaseServiceRecordProcessor
private String path;
private String domain;
- private static final Pattern USER_NAME = Pattern.compile("/users/(\\w*)/?");
private static final String YARN_SERVICE_API_PREFIX =
"classpath:org.apache.hadoop.yarn.service.";
private static final String HTTP_API_TYPE = "http://";
@@ -76,21 +73,6 @@ public abstract class BaseServiceRecordProcessor
}
/**
- * Return the username found in the ZK path.
- *
- * @param recPath the ZK recPath.
- * @return the user name.
- */
- protected String getUsername(String recPath) {
- String user = "anonymous";
- Matcher matcher = USER_NAME.matcher(recPath);
- if (matcher.find()) {
- user = matcher.group(1);
- }
- return user;
- }
-
- /**
* Return the IPv6 mapped address for the provided IPv4 address. Utilized
* to create corresponding AAAA records.
*
@@ -300,7 +282,7 @@ public abstract class BaseServiceRecordProcessor
String service = RegistryPathUtils.lastPathEntry(
RegistryPathUtils.parentOf(RegistryPathUtils.parentOf(getPath())));
String description = getRecord().description.toLowerCase();
- String user = getUsername(getPath());
+ String user = RegistryPathUtils.getUsername(getPath());
return Name.fromString(MessageFormat.format("{0}.{1}.{2}.{3}",
description,
service,
@@ -352,7 +334,7 @@ public abstract class BaseServiceRecordProcessor
* @throws TextParseException
*/
protected Name getServiceName() throws TextParseException {
- String user = getUsername(getPath());
+ String user = RegistryPathUtils.getUsername(getPath());
String service =
String.format("%s.%s.%s",
RegistryPathUtils.lastPathEntry(getPath()),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
index 9a24f1c..4346c9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/client/binding/TestRegistryPathUtils.java
@@ -80,7 +80,15 @@ public class TestRegistryPathUtils extends Assert {
assertCreatedPathEquals("/alice", "/alice", "/");
}
-
+ @Test
+ public void testGetUserFromPath() throws Exception {
+ assertEquals("bob", RegistryPathUtils
+ .getUsername("/registry/users/bob/services/yarn-service/test1/"));
+ assertEquals("bob-dev", RegistryPathUtils
+ .getUsername("/registry/users/bob-dev/services/yarn-service/test1"));
+ assertEquals("bob.dev", RegistryPathUtils
+ .getUsername("/registry/users/bob.dev/services/yarn-service/test1"));
+ }
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c559b27/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index a6fd998..c2e6d26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -155,7 +155,7 @@ where `regionserver-0` is the actual component instance name assigned by the sys
| Name | Description |
| ------------ | ------------- |
| SERVICE_NAME | name of the service defined by the user
-| USER | user who submits the service |
+| USER | user who submits the service. Note that user name which has "\_" will be converted to use "-", to conform with DNS hostname RFC format which doesn't allow "\_", and all characters will be lowercased E.g. "Bob_dev" will be converted to "bob-dev" |
| DOMAIN | the domain name for the cluster |
| COMPONENT_NAME | the name for a given component |
| COMPONENT_INSTANCE_NAME | the name for a given component instance (i.e. container) |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org