You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by br...@apache.org on 2021/03/30 04:17:44 UTC
[hadoop] branch branch-3.3 updated: YARN-10439. Yarn Service AM
listens on all IP's on the machine. Contributed by D M Murali Krishna Reddy
This is an automated email from the ASF dual-hosted git repository.
brahma pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.3 by this push:
new 5181b20 YARN-10439. Yarn Service AM listens on all IP's on the machine. Contributed by D M Murali Krishna Reddy
5181b20 is described below
commit 5181b2004b7b46cfc2b3e4d4c1ce9803b2714198
Author: Brahma Reddy Battula <br...@apache.org>
AuthorDate: Tue Mar 30 09:46:12 2021 +0530
YARN-10439. Yarn Service AM listens on all IP's on the machine. Contributed by D M Murali Krishna Reddy
(cherry picked from commit d0dcfc405c624f73ed1af9527bbf456a10337a6d)
---
.../apache/hadoop/yarn/service/ClientAMService.java | 19 ++++++++++++++-----
.../org/apache/hadoop/yarn/service/ServiceMaster.java | 7 ++++++-
.../hadoop/yarn/service/conf/YarnServiceConf.java | 2 ++
.../org/apache/hadoop/yarn/service/MockServiceAM.java | 10 ++++++++++
.../src/site/markdown/yarn-service/Configurations.md | 1 +
5 files changed, 33 insertions(+), 6 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index 72ac550..342d8d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.Server;
@@ -53,8 +54,10 @@ import org.apache.hadoop.yarn.service.api.records.ComponentContainers;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEvent;
import org.apache.hadoop.yarn.service.component.instance.ComponentInstanceEventType;
+import org.apache.hadoop.yarn.service.exceptions.BadClusterStateException;
import org.apache.hadoop.yarn.service.utils.FilterUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.ServiceUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -64,6 +67,7 @@ import java.util.List;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.DECOMMISSION_INSTANCE;
import static org.apache.hadoop.yarn.service.component.ComponentEventType.FLEX;
+import static org.apache.hadoop.yarn.service.conf.YarnServiceConf.YARN_SERVICE_AM_CLIENT_PORT_RANGE;
public class ClientAMService extends AbstractService
implements ClientAMProtocol {
@@ -84,9 +88,11 @@ public class ClientAMService extends AbstractService
@Override protected void serviceStart() throws Exception {
Configuration conf = getConfig();
YarnRPC rpc = YarnRPC.create(conf);
- InetSocketAddress address = new InetSocketAddress(0);
+ String nodeHostString = getNMHostName();
+
+ InetSocketAddress address = new InetSocketAddress(nodeHostString, 0);
server = rpc.getServer(ClientAMProtocol.class, this, address, conf,
- context.secretManager, 1);
+ context.secretManager, 1, YARN_SERVICE_AM_CLIENT_PORT_RANGE);
// Enable service authorization?
if (conf.getBoolean(
@@ -97,9 +103,6 @@ public class ClientAMService extends AbstractService
server.start();
- String nodeHostString =
- System.getenv(ApplicationConstants.Environment.NM_HOST.name());
-
bindAddress = NetUtils.createSocketAddrForHost(nodeHostString,
server.getListenerAddress().getPort());
@@ -107,6 +110,12 @@ public class ClientAMService extends AbstractService
super.serviceStart();
}
+ @VisibleForTesting
+ String getNMHostName() throws BadClusterStateException {
+ return ServiceUtils.mandatoryEnvVariable(
+ ApplicationConstants.Environment.NM_HOST.name());
+ }
+
@Override protected void serviceStop() throws Exception {
if (server != null) {
server.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
index 670fc21..3120fad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceMaster.java
@@ -129,7 +129,7 @@ public class ServiceMaster extends CompositeService {
DefaultMetricsSystem.initialize("ServiceAppMaster");
context.secretManager = new ClientToAMTokenSecretManager(attemptId, null);
- ClientAMService clientAMService = new ClientAMService(context);
+ ClientAMService clientAMService = createClientAMService();
context.clientAMService = clientAMService;
addService(clientAMService);
@@ -143,6 +143,11 @@ public class ServiceMaster extends CompositeService {
super.serviceInit(conf);
}
+ @VisibleForTesting
+ protected ClientAMService createClientAMService() {
+ return new ClientAMService(context);
+ }
+
// Record the tokens and use them for launching containers.
// e.g. localization requires the hdfs delegation tokens
@VisibleForTesting
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
index 86c4de2..d3716b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConf.java
@@ -49,6 +49,8 @@ public class YarnServiceConf {
public static final long DEFAULT_AM_FAILURES_VALIDITY_INTERVAL = -1;
public static final String AM_RESOURCE_MEM = "yarn.service.am-resource.memory";
public static final long DEFAULT_KEY_AM_RESOURCE_MEM = 1024;
+ public static final String YARN_SERVICE_AM_CLIENT_PORT_RANGE =
+ YARN_SERVICE_PREFIX + "am.client.port-range";
public static final String YARN_QUEUE = "yarn.service.queue";
public static final String DEFAULT_YARN_QUEUE = "default";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
index cf2b1f2..de1a4b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/MockServiceAM.java
@@ -130,6 +130,16 @@ public class MockServiceAM extends ServiceMaster {
}
@Override
+ protected ClientAMService createClientAMService() {
+ return new ClientAMService(context) {
+ @Override
+ String getNMHostName() {
+ return "0.0.0.0";
+ }
+ };
+ }
+
+ @Override
protected ServiceScheduler createServiceScheduler(ServiceContext context)
throws IOException, YarnException {
return new ServiceScheduler(context) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index 53ffa07..e811703 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -132,6 +132,7 @@ The above config allows the service AM to be retried a maximum of 10 times.
|yarn.service.rolling-log.include-pattern | Regex expression for including log files by name when aggregating the logs while app is running.|
|yarn.service.rolling-log.exclude-pattern | Regex expression for excluding log files by name when aggregating the logs while app is running. If the log file name matches both include and exclude pattern, this file will be excluded.|
|yarn.service.classpath | Comma separated extra class path parameters for yarn services AM. These path elements will be appended to the end of the YARN service AM classpath. |
+|yarn.service.am.client.port-range | Range of ports that the Yarn Service AM can use when binding. Leave blank if you want all possible ports. For example 50000-50050,50100-50200. |
### Component-level configuration properties
Component-level service AM configuration properties can be specified either in the cluster `yarn-site.xml` at the global level (effectively overriding the default values system-wide), specified per service in the `properties` field of the `Configuration` object, or specified per component in the `properties` field of the component's `Configuration` object.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org