You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/04 22:37:06 UTC
[3/5] incubator-slider git commit: SLIDER-82 trying to set up
YarnClient,
having config propagation issues in minicluster (RM client addr is 0.0.0.0)
SLIDER-82 trying to set up YarnClient, having config propagation issues in minicluster (RM client addr is 0.0.0.0)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dd607335
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dd607335
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dd607335
Branch: refs/heads/feature/SLIDER-82-pass-3
Commit: dd6073352d7c7b8690098fee19ecef5b9d4b321f
Parents: c6231fe
Author: Steve Loughran <st...@apache.org>
Authored: Wed Nov 4 21:12:53 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Nov 4 21:12:53 2015 +0000
----------------------------------------------------------------------
.../slider/client/SliderYarnClientImpl.java | 8 +++---
.../slider/core/launch/ContainerLauncher.java | 12 +++------
.../server/appmaster/SliderAppMaster.java | 27 ++++++++++----------
3 files changed, 23 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
index 803ccd6..867603b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderYarnClientImpl.java
@@ -45,6 +45,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.BindException;
+import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
@@ -69,10 +70,11 @@ public class SliderYarnClientImpl extends YarnClientImpl {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- String addr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (addr.startsWith("0.0.0.0")) {
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(conf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
// address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
+ throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS
+ + " value:" + conf.get(YarnConfiguration.RM_ADDRESS)
+ " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
}
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
index f8ea4ee..69b937d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/ContainerLauncher.java
@@ -55,17 +55,13 @@ public class ContainerLauncher extends AbstractLauncher {
public UserGroupInformation setupUGI() {
UserGroupInformation user =
UserGroupInformation.createRemoteUser(container.getId().toString());
- String cmIpPortStr =
- container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
- final InetSocketAddress cmAddress =
- NetUtils.createSocketAddr(cmIpPortStr);
+ String cmIpPortStr = container.getNodeId().getHost() + ":" + container.getNodeId().getPort();
+ final InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
- org.apache.hadoop.yarn.api.records.Token containerToken =
- container.getContainerToken();
+ org.apache.hadoop.yarn.api.records.Token containerToken = container.getContainerToken();
if (containerToken != null) {
Token<ContainerTokenIdentifier> token =
- ConverterUtils.convertFromYarn(containerToken,
- cmAddress);
+ ConverterUtils.convertFromYarn(containerToken, cmAddress);
user.addToken(token);
}
return user;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dd607335/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index e6a5bd5..1a127cf 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -502,15 +502,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
addService(executorService);
addService(actionQueues);
- // set up the YARN client. This may require patching in the RM client-API address if it
- // is (somehow) unset server-side.
- String clientRMaddr = conf.get(YarnConfiguration.RM_ADDRESS);
- if (clientRMaddr.startsWith("0.0.0.0")) {
- // address isn't known; fail fast
- throw new BindException("Invalid " + YarnConfiguration.RM_ADDRESS + " value:" + addr
- + " - see https://wiki.apache.org/hadoop/UnsetHostnameOrPort");
- }
- addService(yarnClient = new SliderYarnClientImpl());
//init all child services
super.serviceInit(conf);
@@ -542,8 +533,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @param args argument list
*/
@Override // RunService
- public Configuration bindArgs(Configuration config, String... args) throws
- Exception {
+ public Configuration bindArgs(Configuration config, String... args) throws Exception {
// let the superclass process it
Configuration superConf = super.bindArgs(config, args);
// add the slider XML config
@@ -667,10 +657,21 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
sliderAMProvider = new SliderAMProviderService();
initAndAddService(sliderAMProvider);
- InetSocketAddress address = SliderUtils.getRmSchedulerAddress(serviceConf);
- log.info("RM is at {}", address);
+ InetSocketAddress rmSchedulerAddress = SliderUtils.getRmSchedulerAddress(serviceConf);
+ log.info("RM is at {}", rmSchedulerAddress);
yarnRPC = YarnRPC.create(serviceConf);
+ // set up the YARN client. This may require patching in the RM client-API address if it
+ // is (somehow) unset server-side. String clientRMaddr = serviceConf.get(YarnConfiguration.RM_ADDRESS);
+ InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
+ if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
+ // client addr is being unset. We can lift it from the other RM APIs
+ serviceConf.set(YarnConfiguration.RM_ADDRESS,
+ String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
+ }
+ initAndAddService(yarnClient = new SliderYarnClientImpl());
+ yarnClient.start();
+
/*
* Extract the container ID. This is then
* turned into an (incompete) container