You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/11/07 00:29:48 UTC
[06/22] incubator-slider git commit: SLIDER-947 build node map from yarn update reports; serve via REST/IPC —this is done with YarnClient and building the initial map at launch time. Tests do not yet do this.
SLIDER-947 build node map from yarn update reports; serve via REST/IPC —this is done with YarnClient and building the initial map at launch time. Tests do not yet do this.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5887dde8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5887dde8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5887dde8
Branch: refs/heads/feature/SLIDER-82-pass-3.1
Commit: 5887dde8e68b5d650d692d6e1211a7382465c962
Parents: f1bad85
Author: Steve Loughran <st...@apache.org>
Authored: Thu Nov 5 14:15:54 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Nov 5 14:15:54 2015 +0000
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 21 +++++++++++---------
.../core/launch/JavaCommandLineBuilder.java | 17 ++++++++++++++--
.../server/appmaster/SliderAppMaster.java | 1 +
.../slider/server/appmaster/state/AppState.java | 5 ++++-
.../appmaster/state/AppStateBindingInfo.java | 1 +
.../server/appmaster/state/RoleHistory.java | 7 ++++---
.../slider/agent/rest/TestStandaloneREST.groovy | 7 +++----
7 files changed, 40 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 07c915c..ea6810b 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -2103,9 +2103,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
- /**
- * pass the registry binding
- */
+ // pass the registry binding
commandLine.addConfOptionToCLI(config, RegistryConstants.KEY_REGISTRY_ZK_ROOT,
RegistryConstants.DEFAULT_ZK_REGISTRY_ROOT);
commandLine.addMandatoryConfOption(config, RegistryConstants.KEY_REGISTRY_ZK_QUORUM);
@@ -2115,6 +2113,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// the relevant security settings go over
commandLine.addConfOption(config, DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY);
}
+
+ // copy over any/all YARN RM client values, in case the server-side XML conf file
+ // has the 0.0.0.0 address
+ commandLine.addConfOptions(config,
+ YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.RM_HOSTNAME,
+ YarnConfiguration.RM_PRINCIPAL);
+
// write out the path output
commandLine.addOutAndErrFiles(STDOUT_AM, STDERR_AM);
@@ -2129,12 +2136,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// Set the priority for the application master
-
- int amPriority = config.getInt(KEY_YARN_QUEUE_PRIORITY,
- DEFAULT_YARN_QUEUE_PRIORITY);
-
-
- amLauncher.setPriority(amPriority);
+ amLauncher.setPriority(config.getInt(KEY_YARN_QUEUE_PRIORITY,
+ DEFAULT_YARN_QUEUE_PRIORITY));
// Set the queue to which this application is to be submitted in the RM
// Queue for App master
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
index 9197e5d..ccb610a 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/JavaCommandLineBuilder.java
@@ -25,6 +25,8 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
+import java.util.Map;
+
/**
* Command line builder purely for the Java CLI.
* Some of the <code>define</code> methods are designed to work with Hadoop tool and
@@ -86,8 +88,18 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
}
public boolean addConfOption(Configuration conf, String key) {
- String val = conf.get(key);
- return defineIfSet(key, val);
+ return defineIfSet(key, conf.get(key));
+ }
+
+ /**
+ * Add a varargs list of configuration parameters —if they are present
+ * @param conf configuration source
+ * @param keys keys
+ */
+ public void addConfOptions(Configuration conf, String...keys) {
+ for (String key : keys) {
+ addConfOption(conf, key);
+ }
}
public String addConfOptionToCLI(Configuration conf,
@@ -137,4 +149,5 @@ public class JavaCommandLineBuilder extends CommandLineBuilder {
throw new BadConfigException("Missing configuration option: " + key);
}
}
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 1a127cf..d74688b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -666,6 +666,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
InetSocketAddress clientRpcAddress = SliderUtils.getRmAddress(serviceConf);
if (!SliderUtils.isAddressDefined(clientRpcAddress)) {
// client addr is being unset. We can lift it from the other RM APIs
+ log.warn("Yarn RM address was unbound; attempting to fix up");
serviceConf.set(YarnConfiguration.RM_ADDRESS,
String.format("%s:%d", rmSchedulerAddress.getHostString(), clientRpcAddress.getPort() ));
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 1325148..9e29af2 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -541,7 +541,7 @@ public class AppState {
//set the livespan
MapOperations globalResOpts = instanceDefinition.getResourceOperations().getGlobalOptions();
-
+
startTimeThreshold = globalResOpts.getOptionInt(
InternalKeys.INTERNAL_CONTAINER_FAILURE_SHORTLIFE,
InternalKeys.DEFAULT_INTERNAL_CONTAINER_FAILURE_SHORTLIFE);
@@ -559,6 +559,9 @@ public class AppState {
roleHistory = new RoleHistory(roleList);
roleHistory.register(metricsAndMonitoring);
roleHistory.onStart(binding.fs, binding.historyPath);
+ // trigger first node update
+ roleHistory.onNodesUpdated(binding.nodeReports);
+
//rebuild any live containers
rebuildModelFromRestart(binding.liveContainers);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
index a2a0b60..a4a9b7e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppStateBindingInfo.java
@@ -58,5 +58,6 @@ public class AppStateBindingInfo {
Preconditions.checkArgument(roles != null, "null providerRoles");
Preconditions.checkArgument(fs != null, "null fs");
Preconditions.checkArgument(historyPath != null, "null historyDir");
+ Preconditions.checkArgument(nodeReports != null, "null nodeReports");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index d9a6b34..53c2689 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -76,7 +76,8 @@ public class RoleHistory {
/** Time when saved */
private final Timestamp saveTime = new Timestamp(0);
- /** If the history was loaded, the time at which the history was saved */
+ /** If the history was loaded, the time at which the history was saved.
+ * That is: the time the data was valid */
private final Timestamp thawedDataTime = new Timestamp(0);
private NodeMap nodemap;
@@ -817,12 +818,12 @@ public class RoleHistory {
if (hostname == null || nodeState == null) {
continue;
}
+ log.debug("host {} is in state {}", hostname, nodeState);
// update the node; this also creates an instance if needed
boolean updated = nodemap.updateNode(hostname, updatedNode);
if (updated) {
- log.debug("Updated host {} to state {}", hostname, nodeState);
if (nodeState.isUnusable()) {
- log.info("Failed node {}", hostname);
+ log.info("Failed node {} state {}", hostname, nodeState);
failedNodes.add(hostname);
} else {
failedNodes.remove(hostname);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5887dde8/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
index 29fa51a..7cb1837 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/rest/TestStandaloneREST.groovy
@@ -71,7 +71,7 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
execOperation(WEB_STARTUP_TIME) {
GET(directAM)
}
-
+
execOperation(WEB_STARTUP_TIME) {
def metrics = GET(directAM, SYSTEM_METRICS_JSON)
log.info prettyPrintJson(metrics)
@@ -84,14 +84,13 @@ class TestStandaloneREST extends AgentMiniClusterTestBase {
log.info GET(proxyAM, SYSTEM_HEALTHCHECK)
log.info GET(proxyAM, SYSTEM_METRICS_JSON)
- // using the metrics, await the first node status update
- /* SLIDER--82: disabled
+ // using the metrics, await the first node status update.
+ // this should be from AM launch itself
awaitGaugeValue(
appendToURL(proxyAM, SYSTEM_METRICS_JSON),
"org.apache.slider.server.appmaster.state.RoleHistory.nodes-updated.flag",
1,
WEB_STARTUP_TIME * 2, 500)
- */
// Is the back door required? If so, don't test complex verbs via the proxy
def proxyComplexVerbs = !SliderXmlConfKeys.X_DEV_INSECURE_REQUIRED