You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by el...@apache.org on 2016/06/18 23:15:51 UTC
[01/31] incubator-slider git commit: SLIDER-906 Support for Docker
based application packaging with first class YARN support - Phase 2
[contributed by Thomas (Yu) Liu, Billie Rinaldi and Gour Saha]
Repository: incubator-slider
Updated Branches:
refs/heads/branches/branch-0.91 4b4e24f6c -> fff0e2805
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 0f31d73..cf5c57b 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -359,8 +359,11 @@ public class TestAgentProviderService {
ConfTreeOperations treeOps = aggConf.getAppConfOperations();
treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/");
+ treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf);
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
replay(access, ctx, container, sliderFileSystem, mockFs);
try {
@@ -1279,8 +1282,11 @@ public class TestAgentProviderService {
treeOps.getOrAddComponent("HBASE_MASTER").put(AgentKeys.WAIT_HEARTBEAT, "0");
treeOps.getOrAddComponent("HBASE_REGIONSERVER").put(AgentKeys.WAIT_HEARTBEAT, "0");
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
+ treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/");
+ treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database");
expect(access.getInstanceDefinitionSnapshot()).andReturn(aggConf).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
+ expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
replay(access, ctx, container, sliderFileSystem, mockFs);
@@ -1673,7 +1679,7 @@ public class TestAgentProviderService {
configurations.add("global");
List<String> sysConfigurations = new ArrayList<String>();
configurations.add("core-site");
- doReturn(configurations).when(mockAps).getApplicationConfigurationTypes();
+ doReturn(configurations).when(mockAps).getApplicationConfigurationTypes(anyString());
doReturn(sysConfigurations).when(mockAps).getSystemConfigurationsRequested(any(ConfTreeOperations.class));
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
@@ -1761,7 +1767,7 @@ public class TestAgentProviderService {
configurations.add("global");
List<String> sysConfigurations = new ArrayList<String>();
configurations.add("core-site");
- doReturn(configurations).when(mockAps).getApplicationConfigurationTypes();
+ doReturn(configurations).when(mockAps).getApplicationConfigurationTypes(anyString());
doReturn(sysConfigurations).when(mockAps).getSystemConfigurationsRequested(any(ConfTreeOperations.class));
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-funtest/src/test/groovy/org/apache/slider/funtest/dockeronyarn/DockerAppLaunchedOnYarnIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/dockeronyarn/DockerAppLaunchedOnYarnIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/dockeronyarn/DockerAppLaunchedOnYarnIT.groovy
new file mode 100644
index 0000000..eb0f6ae
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/dockeronyarn/DockerAppLaunchedOnYarnIT.groovy
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.funtest.dockeronyarn
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+
+import org.apache.hadoop.security.UserGroupInformation
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.ClusterNode
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.common.tools.SliderUtils
+import org.apache.slider.api.StateValues
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.apache.slider.funtest.framework.FileUploader
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class DockerAppLaunchedOnYarnIT extends AgentCommandTestBase{
+
+ static String CLUSTER = "test-docker-on-yarn"
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ private boolean testable(){
+ //currently the test cases below are designed for ycloud
+ //and should be disabled for general fun test
+ return false;
+ }
+
+ @Test
+ public void testBasicDockerApp() throws Throwable {
+ if(!testable()){
+ return;
+ }
+ String BASIC_APP_RESOURCE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json"
+ String BASIC_APP_META = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json"
+ String BASIC_APP_TEMPLATE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json"
+
+ launchDockerAppOnYarn(BASIC_APP_TEMPLATE, BASIC_APP_META, BASIC_APP_RESOURCE)
+ ClusterDescription cd = execStatus(CLUSTER)
+ ensureNumberOfContainersAlive("YCLOUD", 1, cd);
+ ensureIpAndHostnamePresent("YCLOUD", 1, cd);
+ }
+
+ @Test
+ public void testMultiCompDockerApp() throws Throwable {
+ if(!testable()){
+ return;
+ }
+ String BASIC_APP_RESOURCE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json"
+ String BASIC_APP_META = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json"
+ String BASIC_APP_TEMPLATE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json"
+
+ launchDockerAppOnYarn(BASIC_APP_TEMPLATE, BASIC_APP_META, BASIC_APP_RESOURCE)
+ ClusterDescription cd = execStatus(CLUSTER)
+ ensureNumberOfContainersAlive("YCLOUD1", 1, cd);
+ ensureIpAndHostnamePresent("YCLOUD1", 1, cd);
+ ensureNumberOfContainersAlive("YCLOUD2", 1, cd);
+ ensureIpAndHostnamePresent("YCLOUD2", 1, cd);
+ }
+
+ @Test
+ public void testMultiCompMultiContainersDockerApp() throws Throwable {
+ if(!testable()){
+ return;
+ }
+ String BASIC_APP_RESOURCE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json"
+ String BASIC_APP_META = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json"
+ String BASIC_APP_TEMPLATE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json"
+
+ launchDockerAppOnYarn(BASIC_APP_TEMPLATE, BASIC_APP_META, BASIC_APP_RESOURCE)
+ //need more time for containers to come up
+ sleep(10000)
+ ClusterDescription cd = execStatus(CLUSTER)
+ ensureNumberOfContainersAlive("YCLOUD1", 2, cd);
+ ensureIpAndHostnamePresent("YCLOUD1", 2, cd);
+ ensureNumberOfContainersAlive("YCLOUD2", 3, cd);
+ ensureIpAndHostnamePresent("YCLOUD2", 3, cd);
+ }
+
+ @Test
+ public void testOneCompFailedDockerApp() throws Throwable {
+ if(!testable()){
+ return;
+ }
+ String BASIC_APP_RESOURCE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json"
+ String BASIC_APP_META = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json"
+ String BASIC_APP_TEMPLATE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json"
+
+ launchDockerAppOnYarn(BASIC_APP_TEMPLATE, BASIC_APP_META, BASIC_APP_RESOURCE)
+ //need more time for containers to come up
+ sleep(30000)
+ ClusterDescription cd = execStatus(CLUSTER)
+ print cd.toString()
+ if(cd.status.get("live") != null){
+ Map live = (Map)cd.status.get("live")
+ if (live.get("YCLOUD1") != null){
+ Map compMap = (Map)live.get("YCLOUD1")
+ String[] keys = compMap.keySet().toArray()
+ for(int i = 0; i < 2; i++){
+ ClusterNode node = (ClusterNode)compMap.get(keys[i])
+ assert node.state == StateValues.STATE_LIVE
+ assert node.ip == null
+ assert node.hostname == null
+ }
+ } else {
+ fail("ycloud1 not present")
+ }
+ } else {
+ fail("live is not in the returned state")
+ }
+ ensureNumberOfContainersAlive("YCLOUD2", 2, cd);
+ ensureIpAndHostnamePresent("YCLOUD2", 2, cd);
+ }
+
+ @Test
+ public void testAllCompFailedDockerApp() throws Throwable {
+ if(!testable()){
+ return;
+ }
+ String BASIC_APP_RESOURCE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json"
+ String BASIC_APP_META = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json"
+ String BASIC_APP_TEMPLATE = "../slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json"
+
+ launchDockerAppOnYarn(BASIC_APP_TEMPLATE, BASIC_APP_META, BASIC_APP_RESOURCE)
+ //need more time for containers to come up
+ sleep(30000)
+ ClusterDescription cd = execStatus(CLUSTER)
+ print cd.toString()
+ if(cd.status.get("live") != null){
+ Map live = (Map)cd.status.get("live")
+ if (live.get("YCLOUD1") != null){
+ Map compMap = (Map)live.get("YCLOUD1")
+ String[] keys = compMap.keySet().toArray()
+ for(int i = 0; i < 2; i++){
+ ClusterNode node = (ClusterNode)compMap.get(keys[i])
+ assert node.state == StateValues.STATE_LIVE
+ assert node.ip == null
+ assert node.hostname == null
+ }
+ } else {
+ fail("ycloud1 not present")
+ }
+ if (live.get("YCLOUD2") != null){
+ Map compMap = (Map)live.get("YCLOUD2")
+ String[] keys = compMap.keySet().toArray()
+ for(int i = 0; i < 2; i++){
+ ClusterNode node = (ClusterNode)compMap.get(keys[i])
+ assert node.state == StateValues.STATE_LIVE
+ assert node.ip == null
+ assert node.hostname == null
+ }
+ } else {
+ fail("ycloud2 not present")
+ }
+ } else {
+ fail("live is not in the returned state")
+ }
+ }
+
+ private void launchDockerAppOnYarn(String appConfig, String metainfo, String resources) throws Throwable {
+ describe("Create a cluster using metainfo, resources, and appConfig that deploy docker based application on yarn")
+ assumeNotWindows()
+ def clusterpath = buildClusterPath(CLUSTER)
+ File launchReportFile = createTempJsonFile();
+
+ SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
+ metainfo,
+ resources,
+ appConfig,
+ [],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+ //need some time for containers to come up
+ sleep(40000)
+ }
+
+ //get cluster description from yarn. make sure all containers and the app master are good
+ private void ensureNumberOfContainersAlive(String compName, int numOfContainers, ClusterDescription cd){
+ print cd.toString()
+ if(cd.status.get("live") != null){
+ Map live = (Map)cd.status.get("live")
+ if (live.get(compName) != null){
+ Map compMap = (Map)live.get(compName)
+ String[] keys = compMap.keySet().toArray()
+ assert numOfContainers == keys.length
+ for(int i = 0; i < numOfContainers; i++){
+ ClusterNode node = (ClusterNode)compMap.get(keys[i])
+ assert node.state == StateValues.STATE_LIVE
+ }
+ } else {
+ fail(compName + " not present")
+ }
+ } else {
+ fail("live is not in the returned state")
+ }
+ }
+
+ private void ensureIpAndHostnamePresent(String compName, int numOfContainers, ClusterDescription cd){
+ print cd.toString()
+ if(cd.status.get("live") != null){
+ Map live = (Map)cd.status.get("live")
+ if (live.get(compName) != null){
+ Map compMap = (Map)live.get(compName)
+ String[] keys = compMap.keySet().toArray()
+ assert numOfContainers == keys.length
+ for(int i = 0; i < numOfContainers; i++){
+ ClusterNode node = (ClusterNode)compMap.get(keys[i])
+ assert node.ip != null
+ assert node.hostname != null
+ }
+ } else {
+ fail(compName + " not present")
+ }
+ } else {
+ fail("live is not in the returned state")
+ }
+ }
+}
[12/31] incubator-slider git commit: SLIDER-1128 Update accumulo test
resources
Posted by el...@apache.org.
SLIDER-1128 Update accumulo test resources
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/cfb516bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/cfb516bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/cfb516bd
Branch: refs/heads/branches/branch-0.91
Commit: cfb516bd9a06cee6ad0458467b8485478f049332
Parents: 0c1f70a
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Fri May 20 08:42:05 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Fri May 20 08:42:05 2016 -0700
----------------------------------------------------------------------
app-packages/accumulo/src/test/resources/resources.json | 4 ++--
.../accumulo/src/test/resources/resources_with_proxy.json | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cfb516bd/app-packages/accumulo/src/test/resources/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/resources.json b/app-packages/accumulo/src/test/resources/resources.json
index 6afbc74..e64de3f 100644
--- a/app-packages/accumulo/src/test/resources/resources.json
+++ b/app-packages/accumulo/src/test/resources/resources.json
@@ -10,10 +10,10 @@
"ACCUMULO_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "64"
+ "yarn.memory": "256"
},
"slider-appmaster": {
- "yarn.memory": "128"
+ "yarn.memory": "384"
},
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/cfb516bd/app-packages/accumulo/src/test/resources/resources_with_proxy.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/resources/resources_with_proxy.json b/app-packages/accumulo/src/test/resources/resources_with_proxy.json
index b424f09..43707a6 100644
--- a/app-packages/accumulo/src/test/resources/resources_with_proxy.json
+++ b/app-packages/accumulo/src/test/resources/resources_with_proxy.json
@@ -10,10 +10,10 @@
"ACCUMULO_MASTER": {
"yarn.role.priority": "1",
"yarn.component.instances": "1",
- "yarn.memory": "64"
+ "yarn.memory": "256"
},
"slider-appmaster": {
- "yarn.memory": "128"
+ "yarn.memory": "384"
},
"ACCUMULO_TSERVER": {
"yarn.role.priority": "2",
[21/31] incubator-slider git commit: SLIDER-1115 stdout/stderr files
have errors
Posted by el...@apache.org.
SLIDER-1115 stdout/stderr files have errors
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/66257d77
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/66257d77
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/66257d77
Branch: refs/heads/branches/branch-0.91
Commit: 66257d771055a96ba0a4d403f41dac4b3da994cf
Parents: e6999d0
Author: Yu Liu <yu...@apache.org>
Authored: Wed Jun 1 11:03:17 2016 -0700
Committer: Yu Liu <yu...@apache.org>
Committed: Wed Jun 1 11:03:17 2016 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/agent/ActionQueue.py | 4 ++--
slider-agent/src/main/python/agent/Constants.py | 4 ++--
.../src/main/python/agent/DockerManager.py | 16 +++++++---------
.../src/main/python/agent/YarnDockerManager.py | 17 ++++++++---------
.../core/providers/windows/system.py | 14 ++++++--------
.../main/python/resource_management/core/shell.py | 17 +++++++----------
6 files changed, 32 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 107d6c3..be6af53 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -72,8 +72,8 @@ class ActionQueue(threading.Thread):
self.customServiceOrchestrator = CustomServiceOrchestrator(config,
controller,
self.queueOutAgentToggleLogger)
- self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
- self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
+ self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
+ self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), config.getLogPath(), self.customServiceOrchestrator)
def stop(self):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/Constants.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Constants.py b/slider-agent/src/main/python/agent/Constants.py
index 6ede66e..524db6a 100644
--- a/slider-agent/src/main/python/agent/Constants.py
+++ b/slider-agent/src/main/python/agent/Constants.py
@@ -34,7 +34,7 @@ ZK_QUORUM="zk_quorum"
ZK_REG_PATH="zk_reg_path"
AUTO_GENERATED="auto_generated"
MAX_AM_CONNECT_RETRIES = 10
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = "application"
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = ".out"
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application_"
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = "application"
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = ".err"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/DockerManager.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/DockerManager.py b/slider-agent/src/main/python/agent/DockerManager.py
index 023a163..48085fe 100644
--- a/slider-agent/src/main/python/agent/DockerManager.py
+++ b/slider-agent/src/main/python/agent/DockerManager.py
@@ -31,9 +31,10 @@ class DockerManager():
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -128,14 +129,11 @@ class DockerManager():
docker_command = self.add_additional_param_to_command(docker_command, additional_param)
#adding redirecting stdout stderr to file
logger.info("docker run command: " + str(docker_command))
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
return self.execute_command_on_linux(docker_command, stdoutFile, stderrFile)
def add_docker_run_options_to_command(self, docker_command, options):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/agent/YarnDockerManager.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py
index abdb2fb..4a4d6de 100644
--- a/slider-agent/src/main/python/agent/YarnDockerManager.py
+++ b/slider-agent/src/main/python/agent/YarnDockerManager.py
@@ -24,6 +24,7 @@ import subprocess
import Constants
import time
import traceback
+from AgentConfig import AgentConfig
from resource_management import *
logger = logging.getLogger()
@@ -33,9 +34,10 @@ class YarnDockerManager(Script):
stored_command = ''
container_id = ''
- def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ def __init__(self, tmpdir, workroot, logDir, customServiceOrchestrator):
self.tmpdir = tmpdir
self.workroot = workroot
+ self.logDir = logDir
self.customServiceOrchestrator = customServiceOrchestrator
def execute_command(self, command, store_command=False):
@@ -168,14 +170,11 @@ class YarnDockerManager(Script):
#extracting param needed by docker run from the command passed from AM
startCommand = self.extract_config_from_command(command, 'docker.startCommand')
#adding redirecting stdout stderr to file
- outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = self.logDir + '/' + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = self.logDir + '/' + Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
returncode,out,err = self.execute_command_on_linux(startCommand, False,
stdoutFile, stderrFile)
return returncode,out,err
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index b85c020..a3d9126 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -30,9 +30,9 @@ import subprocess
import shutil
from resource_management.libraries.script import Script
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def _merge_env(env1, env2, merge_keys=['PYTHONPATH']):
@@ -74,14 +74,12 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=
# TODO implement user
Logger.info("Executing %s" % (command))
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False)
code = None
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/66257d77/slider-agent/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index f21dbbf..4a383c7 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -31,9 +31,9 @@ from exceptions import ExecuteTimeoutException
from resource_management.core.logger import Logger
import time
-APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE = '.log'
-APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application-'
+APPLICATION_STD_ERROR_LOG_FILE_PREFIX = 'application'
APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE = '.err'
def checked_call(command, logoutput=False,
@@ -69,14 +69,11 @@ def _call(command, logoutput=False, throw_on_failure=True,
"""
command = ["/bin/bash","--login","-c", command]
#adding redirecting stdout stderr to file
- outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
-
- errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
- str(pid_file_name) + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
-
- stdoutFile = open(outfilename, 'w')
- stderrFile = open(errfilename, 'w')
+ outfilename = APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+ errfilename = APPLICATION_STD_ERROR_LOG_FILE_PREFIX + APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w+')
+ stderrFile = open(errfilename, 'w+')
proc = subprocess.Popen(command, stdout = stdoutFile, stderr = stderrFile, universal_newlines = True,
cwd=cwd, env=env, shell=False,
[07/31] incubator-slider git commit: SLIDER-1063 duplicated port
allocation when slider.allowed.ports is set (kyungwan nam via gourksaha)
Posted by el...@apache.org.
SLIDER-1063 duplicated port allocation when slider.allowed.ports is set (kyungwan nam via gourksaha)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/4b7d5c83
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/4b7d5c83
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/4b7d5c83
Branch: refs/heads/branches/branch-0.91
Commit: 4b7d5c832ddfbaf00fe538281b61b0aea174a7e0
Parents: 1b833e5
Author: Gour Saha <go...@apache.org>
Authored: Sat May 7 08:09:29 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Sat May 7 08:09:29 2016 -0700
----------------------------------------------------------------------
.../python/agent/CustomServiceOrchestrator.py | 10 ++++--
.../agent/TestCustomServiceOrchestrator.py | 32 ++++++++++++++++++++
2 files changed, 40 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b7d5c83/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index b40a8b5..13c978b 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -63,6 +63,8 @@ class CustomServiceOrchestrator():
self.stored_command = {}
self.allocated_ports = {}
self.log_folders = {}
+
+ self.allocated_ports_set = set()
# Clean up old status command files if any
try:
os.unlink(self.status_commands_stdout)
@@ -300,7 +302,6 @@ class CustomServiceOrchestrator():
allocated_for_any = ".ALLOCATED_PORT}"
port_allocation_req = allocated_for_this_component_format.format(component)
- allowed_ports = self.get_allowed_ports(command)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -311,7 +312,7 @@ class CustomServiceOrchestrator():
value = value.replace("${AGENT_LOG_ROOT}",
self.config.getLogPath())
if port_allocation_req in value:
- value = self.allocate_ports(value, port_allocation_req, allowed_ports)
+ value = self.allocate_ports(value, port_allocation_req, self.get_allowed_ports(command))
allocated_ports[key + "." + k] = value
elif allocated_for_any in value:
## All unallocated ports should be set to 0
@@ -413,6 +414,10 @@ class CustomServiceOrchestrator():
value = value.replace(replaced_pattern, str(port), 1)
logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
index = value.find(port_req_pattern)
+
+ if allowed_ports != None:
+ allowed_ports.remove(port)
+ self.allocated_ports_set.add(port)
pass
return value
pass
@@ -488,6 +493,7 @@ class CustomServiceOrchestrator():
except:
# not an int and not a range...
invalid.add(i)
+ selection = selection - self.allocated_ports_set
selection = random.sample(selection, min (len(selection), num_values))
# Report invalid tokens before returning valid selection
logger.info("Allowed port values: " + str(selection))
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/4b7d5c83/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index aaef3f9..a6fa054 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -674,6 +674,38 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertTrue(set(allowed_ports).issubset(port_range_full_list))
+ def test_finalize_command_when_set_allowed_ports(self):
+ dummy_controller = MagicMock()
+ tempdir = tempfile.gettempdir()
+ tempWorkDir = tempdir + "W"
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempWorkDir
+ config.getLogPath.return_value = tempdir
+
+ allowed_ports = "6700-6701"
+ allowed_ports_full_list = [6700, 6701]
+
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
+ command = {}
+ command['componentName'] = "HBASE_MASTER"
+ command['configurations'] = {}
+ command['configurations']['global'] = {}
+ command['configurations']['global']['slider.allowed.ports'] = allowed_ports
+ command['configurations']['hbase-site'] = {}
+ command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
+ command['configurations']['hbase-site']['a_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+ command['configurations']['hbase-site']['b_port'] = "${HBASE_MASTER.ALLOCATED_PORT}"
+
+ orchestrator.finalize_command(command, False, {})
+ a_port = int(command['configurations']['hbase-site']['a_port'])
+ b_port = int(command['configurations']['hbase-site']['b_port'])
+
+ self.assertTrue((a_port in allowed_ports_full_list) and (b_port in allowed_ports_full_list))
+ self.assertTrue(a_port != b_port)
+
+
def tearDown(self):
# enable stdout
sys.stdout = sys.__stdout__
[16/31] incubator-slider git commit: SLIDER-1130 Hadoop 2.8 YARN RPC
changes have broken slider mock classes
Posted by el...@apache.org.
SLIDER-1130 Hadoop 2.8 YARN RPC changes have broken slider mock classes
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9937bbca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9937bbca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9937bbca
Branch: refs/heads/branches/branch-0.91
Commit: 9937bbcae3ac1575a82b0140558264f2507962ff
Parents: 1154470
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 22:40:49 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 22:40:49 2016 +0100
----------------------------------------------------------------------
.../appstate/TestMockAppStateAAPlacement.groovy | 4 +-
.../appstate/TestMockLabelledAAPlacement.groovy | 11 +--
.../model/history/TestRoleHistoryAA.groovy | 12 ++--
.../model/mock/MockContainerStatus.groovy | 34 ---------
.../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++-
.../appmaster/model/mock/MockNodeReport.groovy | 75 --------------------
.../model/mock/MockRecordFactory.groovy | 37 ++++++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 9 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 5 +-
9 files changed, 114 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index e43d894..85c09ff 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -24,12 +24,10 @@ import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.providers.PlacementPolicy
import org.apache.slider.server.appmaster.model.mock.MockAppState
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -283,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index f0fed95..b2e2a74 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -22,7 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
+import org.apache.slider.server.appmaster.model.mock.MockFactory
import org.apache.slider.server.appmaster.model.mock.MockRoles
import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
@@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
super.setup()
// node 1 is GPU
- updateNodes(new MockNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(new MockNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
@@ -108,7 +108,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
assert 0 == appState.reviewRequestAndReleaseNodes().size()
// switch node 2 into being labelled
- def outcome = updateNodes(new MockNodeReport("00000002", NodeState.RUNNING, "gpu"))
+ def outcome = updateNodes(MockFactory.instance.
+ newNodeReport("00000002", NodeState.RUNNING, "gpu"))
assert cloneNodemap().size() == NODES
assert outcome.clusterChanged
@@ -118,7 +119,7 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(new MockNodeReport("00000004", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("00000004", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
index db84b0b..9c60ea5 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryAA.groovy
@@ -21,12 +21,9 @@ package org.apache.slider.server.appmaster.model.history
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.slider.api.proto.Messages
import org.apache.slider.api.proto.RestTypeMarshalling
-import org.apache.slider.api.types.NodeInformation
import org.apache.slider.api.types.NodeInformationList
import org.apache.slider.server.appmaster.model.mock.MockFactory
-import org.apache.slider.server.appmaster.model.mock.MockNodeReport
import org.apache.slider.server.appmaster.model.mock.MockRoleHistory
import org.apache.slider.server.appmaster.state.NodeEntry
import org.apache.slider.server.appmaster.state.NodeInstance
@@ -72,7 +69,7 @@ class TestRoleHistoryAA extends SliderTestBase {
}
protected boolean setNodeState(NodeInstance node, NodeState state) {
- node.updateNode(new MockNodeReport(node.hostname, state))
+ node.updateNode(MockFactory.instance.newNodeReport(node.hostname, state))
}
@Test
@@ -84,7 +81,8 @@ class TestRoleHistoryAA extends SliderTestBase {
@Test
public void testFindSomeNodesSomeLabel() throws Throwable {
// all three will surface at first
- update(nodeMap, [new MockNodeReport("1", NodeState.RUNNING, "GPU")])
+ update(nodeMap,
+ [MockFactory.instance.newNodeReport("1", NodeState.RUNNING, "GPU")])
def gpuNodes = nodeMap.findAllNodesForRole(1, "GPU")
verifyResultSize(1, gpuNodes)
def instance = gpuNodes[0]
@@ -159,7 +157,7 @@ class TestRoleHistoryAA extends SliderTestBase {
def role1 = node1.getOrCreate(1)
def role2 = node1.getOrCreate(2)
nodeMap.values().each {
- it.updateNode(new MockNodeReport("0", NodeState.UNHEALTHY))
+ it.updateNode(MockFactory.instance.newNodeReport("0", NodeState.UNHEALTHY))
}
assertNoAvailableNodes(1)
assertNoAvailableNodes(2)
@@ -249,6 +247,6 @@ class TestRoleHistoryAA extends SliderTestBase {
def NodeMap createNodeMap(List<String> hosts, NodeState state,
String label = "") {
- createNodeMap(MockNodeReport.createInstances(hosts, state, label))
+ createNodeMap(MockFactory.instance.createNodeReports(hosts, state, label))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
deleted file mode 100644
index 4237f1e..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockContainerStatus.groovy
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import org.apache.hadoop.yarn.api.records.ContainerId
-import org.apache.hadoop.yarn.api.records.ContainerState
-import org.apache.hadoop.yarn.api.records.ContainerStatus
-import org.apache.hadoop.yarn.api.records.Resource
-
-class MockContainerStatus extends ContainerStatus {
-
- ContainerId containerId
- ContainerState state
- String diagnostics
- int exitStatus
-
- Resource capability;
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
index d873390..3ba6e31 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockFactory.groovy
@@ -23,8 +23,13 @@ import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
import org.apache.hadoop.yarn.api.records.Priority
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ResourceKeys
@@ -40,6 +45,8 @@ import org.apache.slider.providers.ProviderRole
@Slf4j
class MockFactory implements MockRoles {
+ public static MockFactory instance = new MockFactory();
+
/*
Ignore any IDE hints about needless references to the ROLE values; groovyc fails without them.
*/
@@ -239,7 +246,56 @@ class MockFactory implements MockRoles {
return new MockResource(memory, vcores)
}
- MockContainerStatus newContainerStatus() {
- return new MockContainerStatus()
+ ContainerStatus newContainerStatus() {
+ return newContainerStatus(null, null, "", 0)
+ }
+
+ ContainerStatus newContainerStatus(ContainerId containerId,
+ ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerStatus.newInstance(containerId, containerState, diagnostics, exitStatus)
+ }
+
+ /**
+ * Create a single instance
+ * @param hostname
+ * @param nodeState
+ * @param label
+ */
+ NodeReport newNodeReport(String hostname, NodeState nodeState, String label = "") {
+ def nodeId = NodeId.newInstance(hostname, 80)
+ Integer.valueOf(hostname, 16)
+ newNodeReport(hostname, nodeId, nodeState, label)
}
+
+ NodeReport newNodeReport(
+ String hostname,
+ NodeId nodeId,
+ NodeState nodeState,
+ String label) {
+ def report = new NodeReportPBImpl();
+ def nodeLabels = new HashSet<>()
+ nodeLabels.add(label)
+ report.nodeId = nodeId
+ report.nodeLabels = nodeLabels
+ report.nodeState = nodeState
+ report.httpAddress = "http$hostname:80"
+ report
+ }
+
+ /**
+ * Create a list of instances -one for each hostname
+ * @param hostnames hosts
+ * @param nodeState state of all of them
+ * @param label label for all of them
+ * @return
+ */
+ List<NodeReport> createNodeReports(
+ List<String> hostnames,
+ NodeState nodeState = NodeState.RUNNING,
+ String label = "") {
+ hostnames.collect { String name ->
+ newNodeReport(name, nodeState, label)
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
deleted file mode 100644
index 8c3b712..0000000
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockNodeReport.groovy
+++ /dev/null
@@ -1,75 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.server.appmaster.model.mock
-
-import groovy.transform.CompileStatic
-import org.apache.hadoop.yarn.api.records.NodeId
-import org.apache.hadoop.yarn.api.records.NodeReport
-import org.apache.hadoop.yarn.api.records.NodeState
-import org.apache.hadoop.yarn.api.records.Resource
-
-/**
- * Node report for testing
- */
-@CompileStatic
-class MockNodeReport extends NodeReport {
- NodeId nodeId;
- NodeState nodeState;
- String httpAddress;
- String rackName;
- Resource used;
- Resource capability;
- int numContainers;
- String healthReport;
- long lastHealthReportTime;
- Set<String> nodeLabels;
-
- MockNodeReport() {
- }
-
- /**
- * Create a single instance
- * @param hostname
- * @param nodeState
- * @param label
- */
- MockNodeReport(String hostname, NodeState nodeState, String label ="") {
- nodeId = NodeId.newInstance(hostname, 80)
- Integer.valueOf(hostname, 16)
- this.nodeState = nodeState
- this.httpAddress = "http$hostname:80"
- this.nodeLabels = new HashSet<>()
- nodeLabels.add(label)
- }
-
- /**
- * Create a list of instances -one for each hostname
- * @param hostnames hosts
- * @param nodeState state of all of them
- * @param label label for all of them
- * @return
- */
- static List<MockNodeReport> createInstances(
- List<String> hostnames,
- NodeState nodeState = NodeState.RUNNING,
- String label = "") {
- hostnames.collect { String name ->
- new MockNodeReport(name, nodeState, label)}
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
new file mode 100644
index 0000000..a608e5a
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockRecordFactory.groovy
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.mock
+
+import groovy.transform.CompileStatic
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.ContainerState
+import org.apache.hadoop.yarn.api.records.ContainerStatus
+import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
+import org.apache.hadoop.yarn.api.records.NodeState
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl
+
+/**
+ * Node report for testing
+ */
+@CompileStatic
+class MockRecordFactory {
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index 265a796..e1dfa25 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.hadoop.yarn.api.records.NodeId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.api.records.NodeState
/**
@@ -149,7 +150,7 @@ public class MockYarnCluster {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
nodes.collect { MockYarnClusterNode n -> n.nodeReport }
}
@@ -167,7 +168,7 @@ public class MockYarnCluster {
public final MockNodeId nodeId;
public final MockYarnClusterContainer[] containers;
private boolean offline;
- public MockNodeReport nodeReport
+ public NodeReport nodeReport
public MockYarnClusterNode(int index, int size) {
nodeIndex = index;
@@ -181,9 +182,7 @@ public class MockYarnCluster {
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = new MockNodeReport()
- nodeReport.nodeId = nodeId
- nodeReport.nodeState = NodeState.RUNNING
+ nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9937bbca/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ab97fa..d946c6b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId
import org.apache.hadoop.yarn.api.records.ApplicationId
import org.apache.hadoop.yarn.api.records.Container
import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.api.records.NodeReport
import org.apache.hadoop.yarn.client.api.AMRMClient
import org.apache.slider.server.appmaster.operations.AbstractRMOperation
import org.apache.slider.server.appmaster.operations.CancelSingleRequest
@@ -166,7 +167,7 @@ class MockYarnEngine {
* Get the list of node reports. These are not cloned; updates will persist in the nodemap
* @return current node report list
*/
- List<MockNodeReport> getNodeReports() {
+ List<NodeReport> getNodeReports() {
cluster.nodeReports
}
-}
\ No newline at end of file
+}
[03/31] incubator-slider git commit: SLIDER-1100 Support unique
component names
Posted by el...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index a11ce56..ed87b89 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -77,6 +77,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -85,6 +86,7 @@ import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicInteger;
import static org.apache.slider.api.ResourceKeys.*;
@@ -166,13 +168,13 @@ public class AppState {
private ClusterDescription clusterStatusTemplate = new ClusterDescription();
private final Map<Integer, RoleStatus> roleStatusMap =
- new ConcurrentHashMap<>();
+ new ConcurrentSkipListMap<>();
private final Map<String, ProviderRole> roles =
new ConcurrentHashMap<>();
- private final Map<Integer, ProviderRole> rolePriorityMap =
- new ConcurrentHashMap<>();
+ private final ConcurrentSkipListMap<Integer, ProviderRole> rolePriorityMap =
+ new ConcurrentSkipListMap<>();
/**
* The master node.
@@ -533,14 +535,19 @@ public class AppState {
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
- if (!roles.containsKey(name)) {
- // this is a new value
- log.info("Adding role {}", name);
- MapOperations resComponent = resources.getComponent(name);
- ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
- buildRole(dynamicRole);
- roleList.add(dynamicRole);
+ if (roles.containsKey(name)) {
+ continue;
+ }
+ if (hasUniqueNames(resources, name)) {
+ log.info("Skipping group {}", name);
+ continue;
}
+ // this is a new value
+ log.info("Adding role {}", name);
+ MapOperations resComponent = resources.getComponent(name);
+ ProviderRole dynamicRole = createDynamicProviderRole(name, resComponent);
+ buildRole(dynamicRole);
+ roleList.add(dynamicRole);
}
//then pick up the requirements
buildRoleRequirementsFromResources();
@@ -615,6 +622,18 @@ public class AppState {
*/
public ProviderRole createDynamicProviderRole(String name, MapOperations component)
throws BadConfigException {
+ return createDynamicProviderRole(name, name, component);
+ }
+
+ /**
+ * Build a dynamic provider role
+ * @param name name of role
+ * @param group group of role
+ * @return a new provider role
+ * @throws BadConfigException bad configuration
+ */
+ public ProviderRole createDynamicProviderRole(String name, String group, MapOperations component)
+ throws BadConfigException {
String priOpt = component.getMandatoryOption(COMPONENT_PRIORITY);
int priority = SliderUtils.parseAndValidate(
"value of " + name + " " + COMPONENT_PRIORITY, priOpt, 0, 1, -1);
@@ -629,9 +648,10 @@ public class AppState {
DEFAULT_PLACEMENT_ESCALATE_DELAY_SECONDS);
ProviderRole newRole = new ProviderRole(name,
+ group,
priority,
placement,
- getNodeFailureThresholdForRole(name),
+ getNodeFailureThresholdForRole(group),
placementTimeout,
component.getOption(YARN_LABEL_EXPRESSION, DEF_YARN_LABEL_EXPRESSION));
log.info("New {} ", newRole);
@@ -659,6 +679,10 @@ public class AppState {
//note the time
snapshotTime = now();
+ for (String component : instanceDefinition.getResourceOperations().getComponentNames()) {
+ instanceDefinition.getAppConfOperations().getOrAddComponent(component);
+ }
+
// resolve references if not already done
instanceDefinition.resolve();
@@ -709,7 +733,7 @@ public class AppState {
// and then driving application size
instanceDefinition.setResources(serDeser.fromInstance(resources));
onInstanceDefinitionUpdated();
-
+
// propagate the role table
Map<String, Map<String, String>> updated = resources.components;
getClusterStatus().roles = SliderUtils.deepClone(updated);
@@ -732,6 +756,7 @@ public class AppState {
instanceDefinition.getResourceOperations();
// Add all the existing roles
+ Map<String, Integer> groupCounts = new HashMap<>();
for (RoleStatus roleStatus : getRoleStatusMap().values()) {
if (roleStatus.isExcludeFromFlexing()) {
// skip inflexible roles, e.g AM itself
@@ -739,14 +764,33 @@ public class AppState {
}
long currentDesired = roleStatus.getDesired();
String role = roleStatus.getName();
- int desiredInstanceCount = getDesiredInstanceCount(resources, role);
- if (desiredInstanceCount == 0) {
+ String roleGroup = roleStatus.getGroup();
+ int desiredInstanceCount = getDesiredInstanceCount(resources, roleGroup);
+
+ int newDesired = desiredInstanceCount;
+ if (hasUniqueNames(resources, roleGroup)) {
+ Integer groupCount = 0;
+ if (groupCounts.containsKey(roleGroup)) {
+ groupCount = groupCounts.get(roleGroup);
+ }
+
+ newDesired = desiredInstanceCount - groupCount;
+
+ if (newDesired > 0) {
+ newDesired = 1;
+ groupCounts.put(roleGroup, groupCount + newDesired);
+ } else {
+ newDesired = 0;
+ }
+ }
+
+ if (newDesired == 0) {
log.info("Role {} has 0 instances specified", role);
}
- if (currentDesired != desiredInstanceCount) {
+ if (currentDesired != newDesired) {
log.info("Role {} flexed from {} to {}", role, currentDesired,
- desiredInstanceCount);
- roleStatus.setDesired(desiredInstanceCount);
+ newDesired);
+ roleStatus.setDesired(newDesired);
}
}
@@ -754,7 +798,34 @@ public class AppState {
// add any role status entries not in the role status
Set<String> roleNames = resources.getComponentNames();
for (String name : roleNames) {
- if (!roles.containsKey(name)) {
+ if (roles.containsKey(name)) {
+ continue;
+ }
+ if (hasUniqueNames(resources, name)) {
+ int desiredInstanceCount = getDesiredInstanceCount(resources, name);
+ Integer groupCount = 0;
+ if (groupCounts.containsKey(name)) {
+ groupCount = groupCounts.get(name);
+ }
+ for (int i = groupCount + 1; i <= desiredInstanceCount; i++) {
+ int priority = resources.getComponentOptInt(name, COMPONENT_PRIORITY, i);
+ // this is a new instance of an existing group
+ String newName = String.format("%s%d", name, i);
+ int newPriority = getNewPriority(priority + i - 1);
+ log.info("Adding new role {}", newName);
+ MapOperations component = resources.getComponent(name,
+ Collections.singletonMap(COMPONENT_PRIORITY,
+ Integer.toString(newPriority)));
+ ProviderRole dynamicRole = createDynamicProviderRole(newName, name, component);
+ RoleStatus roleStatus = buildRole(dynamicRole);
+ roleStatus.setDesired(1);
+ log.info("New role {}", roleStatus);
+ if (roleHistory != null) {
+ roleHistory.addNewRole(roleStatus);
+ }
+ newRoles.add(dynamicRole);
+ }
+ } else {
// this is a new value
log.info("Adding new role {}", name);
MapOperations component = resources.getComponent(name);
@@ -772,6 +843,13 @@ public class AppState {
return newRoles;
}
+ private int getNewPriority(int start) {
+ if (!rolePriorityMap.containsKey(start)) {
+ return start;
+ }
+ return rolePriorityMap.lastKey() + 1;
+ }
+
/**
* Get the desired instance count of a role, rejecting negative values
* @param resources resource map
@@ -794,6 +872,15 @@ public class AppState {
return desiredInstanceCount;
}
+ private Boolean hasUniqueNames(ConfTreeOperations resources, String group) {
+ MapOperations component = resources.getComponent(group);
+ if (component == null) {
+ log.info("Component was null for {} when checking unique names", group);
+ return Boolean.FALSE;
+ }
+ return component.getOptionBool(UNIQUE_NAMES, Boolean.FALSE);
+ }
+
/**
* Add knowledge of a role.
* This is a build-time operation that is not synchronized, and
@@ -851,6 +938,7 @@ public class AppState {
container.setNodeHttpAddress(nodeHttpAddress);
RoleInstance am = new RoleInstance(container);
am.role = SliderKeys.COMPONENT_AM;
+ am.group = SliderKeys.COMPONENT_AM;
am.roleId = SliderKeys.ROLE_AM_PRIORITY_INDEX;
am.createTime =now();
am.startTime = am.createTime;
@@ -1863,7 +1951,7 @@ public class AppState {
private int getFailureThresholdForRole(RoleStatus roleStatus) {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- return resources.getComponentOptInt(roleStatus.getName(),
+ return resources.getComponentOptInt(roleStatus.getGroup(),
CONTAINER_FAILURE_THRESHOLD,
failureThreshold);
}
@@ -2335,6 +2423,7 @@ public class AppState {
RoleInstance instance = new RoleInstance(container);
instance.command = roleName;
instance.role = roleName;
+ instance.group = role.getGroup();
instance.roleId = roleId;
instance.environment = new String[0];
instance.container = container;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 345c67e..30cfec9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -56,6 +56,7 @@ public final class RoleInstance implements Cloneable {
* Name of the role
*/
public String role;
+ public String group;
/**
* Version of the app
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index ce0d60e..0a3a3c9 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -45,6 +45,7 @@ import java.util.Map;
public final class RoleStatus implements Cloneable, MetricSet {
private final String name;
+ private final String group;
/**
* Role priority
@@ -81,6 +82,7 @@ public final class RoleStatus implements Cloneable, MetricSet {
public RoleStatus(ProviderRole providerRole) {
this.providerRole = providerRole;
this.name = providerRole.name;
+ this.group = providerRole.group;
this.key = providerRole.id;
}
@@ -118,6 +120,10 @@ public final class RoleStatus implements Cloneable, MetricSet {
return name;
}
+ public String getGroup() {
+ return group;
+ }
+
public int getKey() {
return key;
}
@@ -412,6 +418,7 @@ public final class RoleStatus implements Cloneable, MetricSet {
public String toString() {
final StringBuilder sb = new StringBuilder("RoleStatus{");
sb.append("name='").append(name).append('\'');
+ sb.append(", group=").append(group);
sb.append(", key=").append(key);
sb.append(", desired=").append(desired);
sb.append(", actual=").append(actual);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index eb97a55..d3864b8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -259,10 +259,10 @@ public class ExecutionCommand {
return builder.toString();
}
- public void addContainerDetails(String componentName, Metainfo metaInfo) {
- Component component = metaInfo.getApplicationComponent(componentName);
+ public void addContainerDetails(String componentGroup, Metainfo metaInfo) {
+ Component component = metaInfo.getApplicationComponent(componentGroup);
this.setComponentType(component.getType());
- log.info("Adding container details for {}", componentName, " from ",
+ log.info("Adding container details for {}", componentGroup, " from ",
metaInfo.toString());
for (DockerContainer metaContainer : component.getDockerContainers()) {
DockerContainer container = new DockerContainer();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
new file mode 100644
index 0000000..5256163
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.fs.Path
+import org.apache.slider.api.ResourceKeys
+import org.apache.slider.core.conf.AggregateConf
+import org.apache.slider.core.conf.ConfTreeOperations
+import org.apache.slider.core.exceptions.BadConfigException
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockAppState
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.model.mock.MockYarnEngine
+import org.apache.slider.server.appmaster.state.AppStateBindingInfo
+import org.apache.slider.server.appmaster.state.MostRecentContainerReleaseSelector
+import org.apache.slider.server.avro.RoleHistoryWriter
+import org.junit.Test
+
+/**
+ * Test that if you have more than one role, the right roles are chosen for release.
+ */
+@CompileStatic
+@Slf4j
+class TestMockAppStateUniqueNames extends BaseMockAppStateTest
+ implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockAppStateUniqueNames"
+ }
+
+ /**
+ * Small cluster with multiple containers per node,
+ * to guarantee many container allocations on each node
+ * @return
+ */
+ @Override
+ MockYarnEngine createYarnEngine() {
+ return new MockYarnEngine(4, 4)
+ }
+
+ @Override
+ AppStateBindingInfo buildBindingInfo() {
+ def bindingInfo = super.buildBindingInfo()
+ bindingInfo.releaseSelector = new MostRecentContainerReleaseSelector()
+ bindingInfo
+ }
+
+ @Override
+ AggregateConf buildInstanceDefinition() {
+ def instance = factory.newInstanceDefinition(0, 0, 0)
+
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "1",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.UNIQUE_NAMES) : "true",
+ ]
+
+ instance.resourceOperations.components["group1"] = opts
+ instance
+ }
+
+ private ConfTreeOperations init() {
+ createAndStartNodes();
+ def resources = appState.instanceDefinition.resources;
+ return new ConfTreeOperations(resources)
+ }
+
+ @Test
+ public void testDynamicFlexAddRole() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "2",
+ (ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.UNIQUE_NAMES) : "true",
+ ]
+
+ cd.components["group2"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ assert 1 == appState.lookupRoleStatus("group11").desired
+ assert 1 == appState.lookupRoleStatus("group21").desired
+ assert 1 == appState.lookupRoleStatus("group22").desired
+ assert 6 == appState.lookupRoleStatus("group11").priority
+ assert 7 == appState.lookupRoleStatus("group21").priority
+ assert 8 == appState.lookupRoleStatus("group22").priority
+ }
+
+ @Test
+ public void testDynamicFlexDown() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "0",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.UNIQUE_NAMES) : "true",
+ ]
+
+ cd.components["group1"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ appState.lookupRoleStatus(6)
+ assert 0 == appState.lookupRoleStatus("group11").desired
+ assert 6 == appState.lookupRoleStatus("group11").priority
+ }
+
+ @Test
+ public void testDynamicFlexUp() throws Throwable {
+ def cd = init()
+ def opts = [
+ (ResourceKeys.COMPONENT_INSTANCES): "3",
+ (ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.UNIQUE_NAMES) : "true",
+ ]
+
+ cd.components["group1"] = opts
+ appState.updateResourceDefinitions(cd.confTree);
+ createAndStartNodes();
+ dumpClusterDescription("updated CD", appState.getClusterStatus())
+ appState.lookupRoleStatus(6)
+ appState.lookupRoleStatus(7)
+ appState.lookupRoleStatus(8)
+ assert 1 == appState.lookupRoleStatus("group11").desired
+ assert 1 == appState.lookupRoleStatus("group12").desired
+ assert 1 == appState.lookupRoleStatus("group13").desired
+ assert 6 == appState.lookupRoleStatus("group11").priority
+ assert 7 == appState.lookupRoleStatus("group12").priority
+ assert 8 == appState.lookupRoleStatus("group13").priority
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index c8914b0..659af0e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -196,7 +196,7 @@ class MockProviderService implements ProviderService {
ContainerLauncher containerLauncher,
AggregateConf instanceDefinition,
Container container,
- String role,
+ ProviderRole role,
SliderFileSystem sliderFileSystem,
Path generatedConfPath,
MapOperations resourceComponent,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index cf5c57b..9fbb3d0 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -282,7 +282,7 @@ public class TestAgentProviderService {
instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
Container container = createNiceMock(Container.class);
- String role = "HBASE_MASTER";
+ ProviderRole role = new ProviderRole("HBASE_MASTER", 1);
SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
Path generatedConfPath = new Path(".", "test");
@@ -323,6 +323,7 @@ public class TestAgentProviderService {
doReturn(true).when(mockAps).isMaster(anyString());
doNothing().when(mockAps).addInstallCommand(
eq("HBASE_MASTER"),
+ eq("HBASE_MASTER"),
eq("mockcontainer_1"),
any(HeartBeatResponse.class),
eq("scripts/hbase_master.py"),
@@ -337,6 +338,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
anyString(),
+ anyString(),
anyMap()
);
@@ -400,6 +402,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
anyString(),
+ anyString(),
anyMap()
);
@@ -474,7 +477,7 @@ public class TestAgentProviderService {
try {
doReturn(true).when(mockAps).isMaster(anyString());
- doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"),
+ doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"), eq("HBASE_MASTER"),
eq("mockcontainer_1"), any(HeartBeatResponse.class),
eq("scripts/hbase_master.py"), eq((ComponentCommand)null),
eq(600L), anyString());
@@ -483,7 +486,8 @@ public class TestAgentProviderService {
}
doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(),
- anyString(), anyMap());
+ anyString(), anyString(),
+ anyMap());
expect(access.isApplicationLive()).andReturn(true).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -535,7 +539,7 @@ public class TestAgentProviderService {
instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
Container container = createNiceMock(Container.class);
- String role_hm = "HBASE_MASTER";
+ ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
Path generatedConfPath = new Path(".", "test");
@@ -565,6 +569,7 @@ public class TestAgentProviderService {
doNothing().when(mockAps).addInstallCommand(
anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -635,6 +640,7 @@ public class TestAgentProviderService {
Assert.assertTrue(hbr.isTerminateAgent());
Mockito.verify(mockAps, Mockito.times(3)).addInstallCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -683,6 +689,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
anyString(),
+ anyString(),
anyMap()
);
@@ -751,6 +758,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
anyString(),
+ anyString(),
anyMap()
);
@@ -896,6 +904,7 @@ public class TestAgentProviderService {
mockAps.processAndPublishComponentSpecificExports(ports,
"mockcontainer_1",
"host1",
+ "HBASE_REGIONSERVER",
"HBASE_REGIONSERVER");
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
@@ -946,10 +955,12 @@ public class TestAgentProviderService {
mockAps.processAndPublishComponentSpecificExports(ports,
"mockcontainer_1",
"host1",
+ "HBASE_REGIONSERVER",
"HBASE_REGIONSERVER");
mockAps.processAndPublishComponentSpecificExports(ports,
"mockcontainer_2",
"host1",
+ "HBASE_REGIONSERVER",
"HBASE_REGIONSERVER");
pubExports = pubExpSet.get("QuickLinks".toLowerCase());
Assert.assertEquals(1, pubExports.entries.size());
@@ -1209,8 +1220,8 @@ public class TestAgentProviderService {
instanceDefinition.getAppConfOperations().getGlobalOptions().put(AgentKeys.AGENT_VERSION, ".");
Container container = createNiceMock(Container.class);
- String role_hm = "HBASE_MASTER";
- String role_hrs = "HBASE_REGIONSERVER";
+ ProviderRole role_hm = new ProviderRole("HBASE_MASTER", 1);
+ ProviderRole role_hrs = new ProviderRole("HBASE_REGIONSERVER", 2);
SliderFileSystem sliderFileSystem = createNiceMock(SliderFileSystem.class);
ContainerLauncher launcher = createNiceMock(ContainerLauncher.class);
ContainerLauncher launcher2 = createNiceMock(ContainerLauncher.class);
@@ -1242,6 +1253,7 @@ public class TestAgentProviderService {
doNothing().when(mockAps).addInstallCommand(
anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1250,6 +1262,7 @@ public class TestAgentProviderService {
doNothing().when(mockAps).addStartCommand(
anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1259,6 +1272,7 @@ public class TestAgentProviderService {
doNothing().when(mockAps).addGetConfigCommand(
anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class));
doNothing().when(mockAps).publishFolderPaths(
anyMap(),
@@ -1335,6 +1349,7 @@ public class TestAgentProviderService {
Assert.assertEquals(2, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(1)).addInstallCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1348,6 +1363,7 @@ public class TestAgentProviderService {
Assert.assertEquals(2, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(2)).addInstallCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1369,6 +1385,7 @@ public class TestAgentProviderService {
Assert.assertEquals(3, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1383,6 +1400,7 @@ public class TestAgentProviderService {
Assert.assertEquals(4, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(0)).addStartCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1407,6 +1425,7 @@ public class TestAgentProviderService {
Assert.assertEquals(3, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1426,6 +1445,7 @@ public class TestAgentProviderService {
Assert.assertEquals(5, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(1)).addStartCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1444,6 +1464,7 @@ public class TestAgentProviderService {
mockAps.handleHeartBeat(hb);
Mockito.verify(mockAps, Mockito.times(1)).addGetConfigCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class));
// RS starts now
@@ -1454,6 +1475,7 @@ public class TestAgentProviderService {
Assert.assertEquals(6, hbr.getResponseId());
Mockito.verify(mockAps, Mockito.times(2)).addStartCommand(anyString(),
anyString(),
+ anyString(),
any(HeartBeatResponse.class),
anyString(),
eq((ComponentCommand)null),
@@ -1619,7 +1641,7 @@ public class TestAgentProviderService {
replay(access);
- mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "", null, 0, null);
+ mockAps.addInstallCommand("HBASE_MASTER", "HBASE_MASTER", "cid1", hbr, "", null, 0, null);
ExecutionCommand cmd = hbr.getExecutionCommands().get(0);
String pkgs = cmd.getHostLevelParams().get(AgentKeys.PACKAGE_LIST);
Assert.assertEquals("[{\"type\":\"tarball\",\"name\":\"files/hbase-0.96.1-hadoop2-bin.tar.gz\"}]", pkgs);
@@ -1701,7 +1723,7 @@ public class TestAgentProviderService {
ComponentCommand startCmd = ComponentCommand.getDefaultComponentCommand();
ComponentCommand stopCmd = ComponentCommand.getDefaultComponentCommand("STOP");
- mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", startCmd, stopCmd, 0, Boolean.FALSE);
+ mockAps.addStartCommand("HBASE_MASTER", "HBASE_MASTER", "cid1", hbr, "", startCmd, stopCmd, 0, Boolean.FALSE);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
@@ -1787,7 +1809,7 @@ public class TestAgentProviderService {
replay(access);
- mockAps.addStopCommand("HBASE_MASTER", "cid1", hbr, "/tmp/stop_cmd.sh", 10, false);
+ mockAps.addStopCommand("HBASE_MASTER", "HBASE_MASTER", "cid1", hbr, "/tmp/stop_cmd.sh", 10, false);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site"));
[18/31] incubator-slider git commit: Merge branch
'feature/SLIDER-1129-httpclient' into develop
Posted by el...@apache.org.
Merge branch 'feature/SLIDER-1129-httpclient' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/44f4a545
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/44f4a545
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/44f4a545
Branch: refs/heads/branches/branch-0.91
Commit: 44f4a5457a075a4315ae9a247daac5cadcb759cd
Parents: 5c391cc 80fe8c7
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 18:00:09 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 24 18:00:09 2016 +0100
----------------------------------------------------------------------
pom.xml | 32 ++-------
slider-core/pom.xml | 8 +--
.../appmaster/state/OutstandingRequest.java | 2 +
.../appstate/TestMockAppStateAAPlacement.groovy | 8 +--
.../appstate/TestMockLabelledAAPlacement.groovy | 11 +--
.../TestOutstandingRequestValidation.groovy | 26 -------
.../model/history/TestRoleHistoryAA.groovy | 12 ++--
...tRoleHistoryOutstandingRequestTracker.groovy | 5 +-
.../model/mock/MockContainerStatus.groovy | 34 ---------
.../appmaster/model/mock/MockFactory.groovy | 60 +++++++++++++++-
.../appmaster/model/mock/MockNodeReport.groovy | 75 --------------------
.../model/mock/MockRecordFactory.groovy | 37 ++++++++++
.../appmaster/model/mock/MockYarnCluster.groovy | 9 ++-
.../appmaster/model/mock/MockYarnEngine.groovy | 5 +-
14 files changed, 131 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
[04/31] incubator-slider git commit: SLIDER-1100 Support unique
component names
Posted by el...@apache.org.
SLIDER-1100 Support unique component names
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/9130f0ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/9130f0ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/9130f0ea
Branch: refs/heads/branches/branch-0.91
Commit: 9130f0eac8c404d27403867a5e9b59bf0c22068a
Parents: 1ab0cbb
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Apr 13 08:57:51 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Apr 13 08:57:51 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/api/ResourceKeys.java | 5 +
.../slider/core/conf/ConfTreeOperations.java | 18 +
.../conf/ResourcesInputPropertiesValidator.java | 3 +-
.../providers/AbstractClientProvider.java | 2 +-
.../apache/slider/providers/ProviderRole.java | 35 ++
.../slider/providers/ProviderService.java | 4 +-
.../providers/agent/AgentLaunchParameter.java | 10 +-
.../providers/agent/AgentProviderService.java | 422 +++++++++----------
.../agent/application/metadata/Metainfo.java | 4 +-
.../slideram/SliderAMProviderService.java | 2 +-
.../server/appmaster/RoleLaunchService.java | 27 +-
.../server/appmaster/SliderAppMaster.java | 9 +-
.../actions/RegisterComponentInstance.java | 5 +-
.../slider/server/appmaster/state/AppState.java | 127 +++++-
.../server/appmaster/state/RoleInstance.java | 1 +
.../server/appmaster/state/RoleStatus.java | 7 +
.../web/rest/agent/ExecutionCommand.java | 6 +-
.../appstate/TestMockAppStateUniqueNames.groovy | 150 +++++++
.../model/mock/MockProviderService.groovy | 2 +-
.../agent/TestAgentProviderService.java | 40 +-
20 files changed, 602 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index aba544b..92890be 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -50,6 +50,11 @@ public interface ResourceKeys {
String COMPONENT_INSTANCES = "yarn.component.instances";
/**
+ * Whether to use unique names for each instance of a component: {@value}
+ */
+ String UNIQUE_NAMES = "component.unique.names";
+
+ /**
* Amount of memory to ask YARN for in MB.
* <i>Important:</i> this may be a hard limit on the
* amount of RAM that the service can use
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
index 58896ee..9013edb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ConfTreeOperations.java
@@ -129,6 +129,24 @@ public class ConfTreeOperations {
}
/**
+ * look up a component and return its options with the specified replacements
+ * @param component component name
+ * @param replacementOptions replacement options
+ * @return component mapping or null
+ */
+ public MapOperations getComponent(String component, Map<String,String>
+ replacementOptions) {
+ Map<String, String> instance = confTree.components.get(component);
+ if (instance != null) {
+ Map<String, String> newInstance = new HashMap<>();
+ newInstance.putAll(instance);
+ newInstance.putAll(replacementOptions);
+ return new MapOperations(component, newInstance);
+ }
+ return null;
+ }
+
+ /**
* Get at the underlying component map
* @return a map of components. This is the raw ConfTree data structure
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java b/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
index 5a8c444..19f6f8d 100644
--- a/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
+++ b/slider-core/src/main/java/org/apache/slider/core/conf/ResourcesInputPropertiesValidator.java
@@ -16,6 +16,7 @@
*/
package org.apache.slider.core.conf;
+import org.apache.slider.api.ResourceKeys;
import org.apache.slider.core.exceptions.BadConfigException;
/**
@@ -25,7 +26,7 @@ public class ResourcesInputPropertiesValidator
extends AbstractInputPropertiesValidator {
void validatePropertyNamePrefix(String key) throws BadConfigException {
- if (!key.startsWith("yarn.")) {
+ if (!key.startsWith("yarn.") && !key.equals(ResourceKeys.UNIQUE_NAMES)) {
throw new BadConfigException(
"argument %s does not have 'yarn.' prefix", key);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index 1a1baf7..fcab65e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -76,7 +76,7 @@ public abstract class AbstractClientProvider extends Configured {
instanceDefinition.getResourceOperations();
for (ProviderRole role : roles) {
String name = role.name;
- MapOperations component = resources.getComponent(name);
+ MapOperations component = resources.getComponent(role.group);
if (component != null) {
String instances = component.get(COMPONENT_INSTANCES);
if (instances == null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
index 1b95b42..761ac0f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderRole.java
@@ -28,6 +28,7 @@ import org.apache.slider.api.ResourceKeys;
*/
public final class ProviderRole {
public final String name;
+ public final String group;
public final int id;
public int placementPolicy;
public int nodeFailureThreshold;
@@ -36,6 +37,7 @@ public final class ProviderRole {
public ProviderRole(String name, int id) {
this(name,
+ name,
id,
PlacementPolicy.DEFAULT,
ResourceKeys.DEFAULT_NODE_FAILURE_THRESHOLD,
@@ -59,7 +61,39 @@ public final class ProviderRole {
int nodeFailureThreshold,
long placementTimeoutSeconds,
String labelExpression) {
+ this(name,
+ name,
+ id,
+ policy,
+ nodeFailureThreshold,
+ placementTimeoutSeconds,
+ labelExpression);
+ }
+
+ /**
+ * Create a provider role with a role group
+ * @param name role/component name
+ * @param group role/component group
+ * @param id ID. This becomes the YARN priority
+ * @param policy placement policy
+ * @param nodeFailureThreshold threshold for node failures (within a reset interval)
+ * after which a node failure is considered an app failure
+ * @param placementTimeoutSeconds for lax placement, timeout in seconds before
+ * @param labelExpression label expression for requests; may be null
+ */
+ public ProviderRole(String name,
+ String group,
+ int id,
+ int policy,
+ int nodeFailureThreshold,
+ long placementTimeoutSeconds,
+ String labelExpression) {
this.name = name;
+ if (group == null) {
+ this.group = name;
+ } else {
+ this.group = group;
+ }
this.id = id;
this.placementPolicy = policy;
this.nodeFailureThreshold = nodeFailureThreshold;
@@ -89,6 +123,7 @@ public final class ProviderRole {
public String toString() {
final StringBuilder sb = new StringBuilder("ProviderRole{");
sb.append("name='").append(name).append('\'');
+ sb.append(", group=").append(group);
sb.append(", id=").append(id);
sb.append(", placementPolicy=").append(placementPolicy);
sb.append(", nodeFailureThreshold=").append(nodeFailureThreshold);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 4689ae8..f754eee 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -55,7 +55,7 @@ public interface ProviderService extends ProviderCore,
* @param containerLauncher
* @param instanceDefinition
* @param container
- * @param role
+ * @param providerRole
* @param sliderFileSystem
* @param generatedConfPath
* @param appComponent
@@ -64,7 +64,7 @@ public interface ProviderService extends ProviderCore,
void buildContainerLaunchContext(ContainerLauncher containerLauncher,
AggregateConf instanceDefinition,
Container container,
- String role,
+ ProviderRole providerRole,
SliderFileSystem sliderFileSystem,
Path generatedConfPath,
MapOperations resourceComponent,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
index c8b0e1d..18c6374 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentLaunchParameter.java
@@ -41,18 +41,18 @@ class AgentLaunchParameter {
/**
* Get command for the component type
*
- * @param componentName
+ * @param componentGroup
*
* @return
*/
- public String getNextLaunchParameter(String componentName) {
+ public String getNextLaunchParameter(String componentGroup) {
if (launchParameterTracker != null) {
- if (launchParameterTracker.containsKey(componentName)
+ if (launchParameterTracker.containsKey(componentGroup)
|| launchParameterTracker.containsKey(ANY_COMPONENT)) {
synchronized (this) {
CommandTracker indexTracker = null;
- if (launchParameterTracker.containsKey(componentName)) {
- indexTracker = launchParameterTracker.get(componentName);
+ if (launchParameterTracker.containsKey(componentGroup)) {
+ indexTracker = launchParameterTracker.get(componentGroup);
} else {
indexTracker = launchParameterTracker.get(ANY_COMPONENT);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 9d12f1d..d5fae19 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -333,7 +333,7 @@ public class AgentProviderService extends AbstractProviderService implements
public void buildContainerLaunchContext(ContainerLauncher launcher,
AggregateConf instanceDefinition,
Container container,
- String role,
+ ProviderRole providerRole,
SliderFileSystem fileSystem,
Path generatedConfPath,
MapOperations resourceComponent,
@@ -342,6 +342,8 @@ public class AgentProviderService extends AbstractProviderService implements
IOException,
SliderException {
+ String roleName = providerRole.name;
+ String roleGroup = providerRole.group;
String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
@@ -351,18 +353,18 @@ public class AgentProviderService extends AbstractProviderService implements
log.debug(instanceDefinition.toString());
//if we are launching docker based app on yarn, then we need to pass docker image
- if (isYarnDockerContainer(role)) {
+ if (isYarnDockerContainer(roleGroup)) {
launcher.setYarnDockerMode(true);
- launcher.setDockerImage(getConfigFromMetaInfo(role, "image"));
- launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(role, "runPriviledgedContainer"));
+ launcher.setDockerImage(getConfigFromMetaInfo(roleGroup, "image"));
+ launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(roleGroup, "runPriviledgedContainer"));
launcher
.setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
- role, "yarn.container.mount.points"));
+ roleGroup, "yarn.container.mount.points"));
}
// Set the environment
launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
- getStandardTokenMap(getAmState().getAppConfSnapshot(), role)));
+ getStandardTokenMap(getAmState().getAppConfSnapshot(), roleName, roleGroup)));
String workDir = ApplicationConstants.Environment.PWD.$();
launcher.setEnv("AGENT_WORK_ROOT", workDir);
@@ -417,8 +419,7 @@ public class AgentProviderService extends AbstractProviderService implements
} else {
String msg =
String.format("Required agent image slider-agent.tar.gz is unavailable at %s", agentImagePath.toString());
- MapOperations compOps = instanceDefinition.
- getAppConfOperations().getComponent(role);
+ MapOperations compOps = appComponent;
boolean relaxVerificationForTest = compOps != null ? Boolean.valueOf(compOps.
getOptionBool(AgentKeys.TEST_RELAX_VERIFICATION, false)) : false;
log.error(msg);
@@ -464,11 +465,10 @@ public class AgentProviderService extends AbstractProviderService implements
localizeContainerSSLResources(launcher, container, fileSystem);
}
- MapOperations compOps = instanceDefinition.
- getAppConfOperations().getComponent(role);
+ MapOperations compOps = appComponent;
if (areStoresRequested(compOps)) {
- localizeContainerSecurityStores(launcher, container, role, fileSystem,
- instanceDefinition);
+ localizeContainerSecurityStores(launcher, container, roleName, fileSystem,
+ instanceDefinition, compOps);
}
//add the configuration resources
@@ -476,7 +476,7 @@ public class AgentProviderService extends AbstractProviderService implements
generatedConfPath,
SliderKeys.PROPAGATED_CONF_DIR_NAME));
- String label = getContainerLabel(container, role);
+ String label = getContainerLabel(container, roleName, roleGroup);
CommandLineBuilder operation = new CommandLineBuilder();
String pythonExec = instanceDefinition.getAppConfOperations()
@@ -492,7 +492,7 @@ public class AgentProviderService extends AbstractProviderService implements
operation.add(ARG_ZOOKEEPER_REGISTRY_PATH);
operation.add(getZkRegistryPath());
- String debugCmd = agentLaunchParameter.getNextLaunchParameter(role);
+ String debugCmd = agentLaunchParameter.getNextLaunchParameter(roleGroup);
if (SliderUtils.isSet(debugCmd)) {
operation.add(ARG_DEBUG);
operation.add(debugCmd);
@@ -550,21 +550,21 @@ public class AgentProviderService extends AbstractProviderService implements
// component 'role'
for (ComponentsInAddonPackage comp : appPkg.getApplicationPackage()
.getComponents()) {
- log.debug("Current component: {} component in metainfo: {}", role,
+ log.debug("Current component: {} component in metainfo: {}", roleName,
comp.getName());
- if (comp.getName().equals(role)
+ if (comp.getName().equals(roleGroup)
|| comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) {
pkgStatuses.put(appPkg.getApplicationPackage().getName(), State.INIT);
}
}
}
- log.debug("For component: {} pkg status map: {}", role,
+ log.debug("For component: {} pkg status map: {}", roleName,
pkgStatuses.toString());
// initialize the component instance state
getComponentStatuses().put(label,
new ComponentInstanceState(
- role,
+ roleName,
container.getId(),
getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME),
pkgStatuses));
@@ -574,10 +574,9 @@ public class AgentProviderService extends AbstractProviderService implements
Container container,
String role,
SliderFileSystem fileSystem,
- AggregateConf instanceDefinition)
+ AggregateConf instanceDefinition,
+ MapOperations compOps)
throws SliderException, IOException {
- MapOperations compOps = instanceDefinition.getAppConfOperations()
- .getComponent(role);
// generate and localize security stores
SecurityStore[] stores = generateSecurityStores(container, role,
instanceDefinition, compOps);
@@ -741,7 +740,7 @@ public class AgentProviderService extends AbstractProviderService implements
.extractRole(container));
if (role != null) {
String roleName = role.name;
- String label = getContainerLabel(container, roleName);
+ String label = getContainerLabel(container, roleName, role.group);
log.info("Rebuilding in-memory: container {} in role {} in cluster {}",
container.getId(), roleName, applicationId);
getComponentStatuses().put(label,
@@ -784,6 +783,7 @@ public class AgentProviderService extends AbstractProviderService implements
updateComponentStatusWithAgentState(componentStatus, agentState);
String roleName = getRoleName(label);
+ String roleGroup = getRoleGroup(label);
String containerId = getContainerId(label);
if (SliderUtils.isSet(registration.getTags())) {
@@ -795,7 +795,7 @@ public class AgentProviderService extends AbstractProviderService implements
String hostFqdn = registration.getPublicHostname();
Map<String, String> ports = registration.getAllocatedPorts();
if (ports != null && !ports.isEmpty()) {
- processAllocatedPorts(hostFqdn, roleName, containerId, ports);
+ processAllocatedPorts(hostFqdn, roleName, roleGroup, containerId, ports);
}
Map<String, String> folders = registration.getLogFolders();
@@ -856,6 +856,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.debug("package received: " + pkg);
String roleName = getRoleName(label);
+ String roleGroup = getRoleGroup(label);
String containerId = getContainerId(label);
boolean doUpgrade = false;
if (isInUpgradeMode && upgradeContainers.contains(containerId)) {
@@ -863,10 +864,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
StateAccessForProviders accessor = getAmState();
- CommandScript cmdScript = getScriptPathForMasterPackage(roleName);
- List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands();
+ CommandScript cmdScript = getScriptPathForMasterPackage(roleGroup);
+ List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleGroup).getCommands();
- if (!isDockerContainer(roleName) && !isYarnDockerContainer(roleName)
+ if (!isDockerContainer(roleGroup) && !isYarnDockerContainer(roleGroup)
&& (cmdScript == null || cmdScript.getScript() == null)
&& commands.size() == 0) {
log.error(
@@ -940,7 +941,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- Boolean isMaster = isMaster(roleName);
+ Boolean isMaster = isMaster(roleGroup);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
componentStatus.heartbeat(System.currentTimeMillis());
if (doUpgrade) {
@@ -967,14 +968,14 @@ public class AgentProviderService extends AbstractProviderService implements
componentStatus.setStopInitiated(true);
}
- publishConfigAndExportGroups(heartBeat, componentStatus, roleName);
+ publishConfigAndExportGroups(heartBeat, componentStatus, roleGroup);
CommandResult result = null;
List<CommandReport> reports = heartBeat.getReports();
if (SliderUtils.isNotEmpty(reports)) {
CommandReport report = reports.get(0);
Map<String, String> ports = report.getAllocatedPorts();
if (SliderUtils.isNotEmpty(ports)) {
- processAllocatedPorts(heartBeat.getFqdn(), roleName, containerId, ports);
+ processAllocatedPorts(heartBeat.getFqdn(), roleName, roleGroup, containerId, ports);
}
result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -989,7 +990,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
int waitForCount = accessor.getInstanceDefinitionSnapshot().
- getAppConfOperations().getComponentOptInt(roleName, AgentKeys.WAIT_HEARTBEAT, 0);
+ getAppConfOperations().getComponentOptInt(roleGroup, AgentKeys.WAIT_HEARTBEAT, 0);
if (id < waitForCount) {
log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
@@ -1004,11 +1005,12 @@ public class AgentProviderService extends AbstractProviderService implements
componentStatus.getNextPkgToInstall(), command.toString());
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
- addInstallDockerCommand(roleName, containerId, response, null, timeout);
+ if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
+ addInstallDockerCommand(roleName, roleGroup, containerId,
+ response, null, timeout);
} else if (scriptPath != null) {
- addInstallCommand(roleName, containerId, response, scriptPath,
- null, timeout, null);
+ addInstallCommand(roleName, roleGroup, containerId, response,
+ scriptPath, null, timeout, null);
} else {
// commands
ComponentCommand installCmd = null;
@@ -1017,7 +1019,7 @@ public class AgentProviderService extends AbstractProviderService implements
installCmd = compCmd;
}
}
- addInstallCommand(roleName, containerId, response, null,
+ addInstallCommand(roleName, roleGroup, containerId, response, null,
installCmd, timeout, null);
}
componentStatus.commandIssued(command);
@@ -1030,12 +1032,12 @@ public class AgentProviderService extends AbstractProviderService implements
// should only execute once per heartbeat
log.debug("Addon component: {} pkg: {} script: {}", comp.getName(),
nextPkgToInstall, comp.getCommandScript().getScript());
- if (comp.getName().equals(roleName)
+ if (comp.getName().equals(roleGroup)
|| comp.getName().equals(AgentKeys.ADDON_FOR_ALL_COMPONENTS)) {
scriptPath = comp.getCommandScript().getScript();
if (scriptPath != null) {
- addInstallCommand(roleName, containerId, response, scriptPath,
- null, timeout, nextPkgToInstall);
+ addInstallCommand(roleName, roleGroup, containerId, response,
+ scriptPath, null, timeout, nextPkgToInstall);
} else {
ComponentCommand installCmd = null;
for (ComponentCommand compCmd : comp.getCommands()) {
@@ -1043,28 +1045,30 @@ public class AgentProviderService extends AbstractProviderService implements
installCmd = compCmd;
}
}
- addInstallCommand(roleName, containerId, response, null,
- installCmd, timeout, nextPkgToInstall);
+ addInstallCommand(roleName, roleGroup, containerId, response,
+ null, installCmd, timeout, nextPkgToInstall);
}
}
}
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
- boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
+ boolean canExecute = commandOrder.canExecute(roleGroup, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
- addStartDockerCommand(roleName, containerId, response, null, timeout, false);
+ if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
+ addStartDockerCommand(roleName, roleGroup, containerId,
+ response, null, timeout, false);
} else if (scriptPath != null) {
addStartCommand(roleName,
+ roleGroup,
containerId,
response,
scriptPath,
null,
null,
timeout,
- isMarkedAutoRestart(roleName));
+ isMarkedAutoRestart(roleGroup));
} else {
ComponentCommand startCmd = null;
for (ComponentCommand compCmd : commands) {
@@ -1078,21 +1082,22 @@ public class AgentProviderService extends AbstractProviderService implements
stopCmd = compCmd;
}
}
- addStartCommand(roleName, containerId, response, null, startCmd, stopCmd, timeout, false);
+ addStartCommand(roleName, roleGroup, containerId, response, null,
+ startCmd, stopCmd, timeout, false);
}
componentStatus.commandIssued(command);
} else {
log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
}
} else if (command == Command.UPGRADE) {
- addUpgradeCommand(roleName, containerId, response, scriptPath,
- timeout);
+ addUpgradeCommand(roleName, roleGroup, containerId, response,
+ scriptPath, timeout);
componentStatus.commandIssued(command, true);
} else if (command == Command.STOP) {
log.info("Stop command being sent to container with id {}",
containerId);
- addStopCommand(roleName, containerId, response, scriptPath, timeout,
- doUpgrade);
+ addStopCommand(roleName, roleGroup, containerId, response, scriptPath,
+ timeout, doUpgrade);
componentStatus.commandIssued(command);
} else if (command == Command.TERMINATE) {
log.info("A formal terminate command is being sent to container {}"
@@ -1106,10 +1111,10 @@ public class AgentProviderService extends AbstractProviderService implements
&& command == Command.NOP) {
if (!componentStatus.getConfigReported()) {
log.info("Requesting applied config for {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
- addGetConfigDockerCommand(roleName, containerId, response);
+ if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)){
+ addGetConfigDockerCommand(roleName, roleGroup, containerId, response);
} else {
- addGetConfigCommand(roleName, containerId, response);
+ addGetConfigCommand(roleName, roleGroup, containerId, response);
}
}
}
@@ -1117,7 +1122,7 @@ public class AgentProviderService extends AbstractProviderService implements
// if restart is required then signal
response.setRestartEnabled(false);
if (componentStatus.getState() == State.STARTED
- && command == Command.NOP && isMarkedAutoRestart(roleName)) {
+ && command == Command.NOP && isMarkedAutoRestart(roleGroup)) {
response.setRestartEnabled(true);
}
@@ -1137,16 +1142,16 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
- private boolean isDockerContainer(String roleName) {
- String type = getMetaInfo().getApplicationComponent(roleName).getType();
+ private boolean isDockerContainer(String roleGroup) {
+ String type = getMetaInfo().getApplicationComponent(roleGroup).getType();
if (SliderUtils.isSet(type)) {
return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
}
return false;
}
- private boolean isYarnDockerContainer(String roleName) {
- String type = getMetaInfo().getApplicationComponent(roleName).getType();
+ private boolean isYarnDockerContainer(String roleGroup) {
+ String type = getMetaInfo().getApplicationComponent(roleGroup).getType();
if (SliderUtils.isSet(type)) {
return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
}
@@ -1155,6 +1160,7 @@ public class AgentProviderService extends AbstractProviderService implements
protected void processAllocatedPorts(String fqdn,
String roleName,
+ String roleGroup,
String containerId,
Map<String, String> ports) {
RoleInstance instance;
@@ -1186,13 +1192,13 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
- processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName);
+ processAndPublishComponentSpecificData(ports, containerId, fqdn, roleGroup);
+ processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName, roleGroup);
// and update registration entries
if (instance != null) {
queueAccess.put(new RegisterComponentInstance(instance.getId(),
- roleName, 0, TimeUnit.MILLISECONDS));
+ roleName, roleGroup, 0, TimeUnit.MILLISECONDS));
}
}
@@ -1450,8 +1456,13 @@ public class AgentProviderService extends AbstractProviderService implements
return amState.getRoleClusterNodeMapping();
}
- private String getContainerLabel(Container container, String role) {
- return container.getId().toString() + LABEL_MAKER + role;
+ private String getContainerLabel(Container container, String role, String group) {
+ if (role.equals(group)) {
+ return container.getId().toString() + LABEL_MAKER + role;
+ } else {
+ return container.getId().toString() + LABEL_MAKER + role + LABEL_MAKER +
+ group;
+ }
}
protected String getClusterInfoPropertyValue(String name) {
@@ -1563,8 +1574,8 @@ public class AgentProviderService extends AbstractProviderService implements
* @param heartBeat
* @param componentStatus
*/
- protected void publishConfigAndExportGroups(
- HeartBeat heartBeat, ComponentInstanceState componentStatus, String componentName) {
+ protected void publishConfigAndExportGroups(HeartBeat heartBeat,
+ ComponentInstanceState componentStatus, String componentGroup) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
log.info("Processing {} status reports.", statuses.size());
@@ -1574,7 +1585,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (status.getConfigs() != null) {
Application application = getMetaInfo().getApplication();
- if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) {
+ if (canAnyMasterPublishConfig() == false || canPublishConfig(componentGroup)) {
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -1602,7 +1613,7 @@ public class AgentProviderService extends AbstractProviderService implements
boolean hasExportGroups = SliderUtils.isNotEmpty(appExportGroups);
Set<String> appExports = new HashSet();
- String appExportsStr = getApplicationComponent(componentName).getAppExports();
+ String appExportsStr = getApplicationComponent(componentGroup).getAppExports();
if (SliderUtils.isSet(appExportsStr)) {
for (String appExport : appExportsStr.split(",")) {
if (!appExport.trim().isEmpty()) {
@@ -1711,14 +1722,14 @@ public class AgentProviderService extends AbstractProviderService implements
protected void processAndPublishComponentSpecificData(Map<String, String> ports,
String containerId,
String hostFqdn,
- String componentName) {
+ String componentGroup) {
String portVarFormat = "${site.%s}";
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<String, String>();
Application application = getMetaInfo().getApplication();
for (Component component : application.getComponents()) {
- if (component.getName().equals(componentName)) {
+ if (component.getName().equals(componentGroup)) {
if (component.getComponentExports().size() > 0) {
for (ComponentExport export : component.getComponentExports()) {
@@ -1762,12 +1773,13 @@ public class AgentProviderService extends AbstractProviderService implements
protected void processAndPublishComponentSpecificExports(Map<String, String> ports,
String containerId,
String hostFqdn,
- String compName) {
+ String compName,
+ String compGroup) {
String portVarFormat = "${site.%s}";
- String hostNamePattern = "${" + compName + "_HOST}";
+ String hostNamePattern = "${" + compGroup + "_HOST}";
List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups();
- Component component = getMetaInfo().getApplicationComponent(compName);
+ Component component = getMetaInfo().getApplicationComponent(compGroup);
if (component != null && SliderUtils.isSet(component.getCompExports())
&& SliderUtils.isNotEmpty(appExportGroups)) {
@@ -1862,24 +1874,24 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
- * Return Component based on name
+ * Return Component based on group
*
- * @param roleName component name
+ * @param roleGroup component group
*
* @return the component entry or null for no match
*/
- protected Component getApplicationComponent(String roleName) {
- return getMetaInfo().getApplicationComponent(roleName);
+ protected Component getApplicationComponent(String roleGroup) {
+ return getMetaInfo().getApplicationComponent(roleGroup);
}
/**
* Extract script path from the application metainfo
*
- * @param roleName component name
+ * @param roleGroup component group
* @return the script path or null for no match
*/
- protected CommandScript getScriptPathForMasterPackage(String roleName) {
- Component component = getApplicationComponent(roleName);
+ protected CommandScript getScriptPathForMasterPackage(String roleGroup) {
+ Component component = getApplicationComponent(roleGroup);
if (component != null) {
return component.getCommandScript();
}
@@ -1889,12 +1901,12 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Is the role of type MASTER
*
- * @param roleName component name
+ * @param roleGroup component group
*
* @return true if the role category is MASTER
*/
- protected boolean isMaster(String roleName) {
- Component component = getApplicationComponent(roleName);
+ protected boolean isMaster(String roleGroup) {
+ Component component = getApplicationComponent(roleGroup);
if (component != null) {
if (component.getCategory().equals("MASTER")) {
return true;
@@ -1906,12 +1918,12 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Can the role publish configuration
*
- * @param roleName component name
+ * @param roleGroup component group
*
* @return true if it can be pubished
*/
- protected boolean canPublishConfig(String roleName) {
- Component component = getApplicationComponent(roleName);
+ protected boolean canPublishConfig(String roleGroup) {
+ Component component = getApplicationComponent(roleGroup);
if (component != null) {
return Boolean.TRUE.toString().equals(component.getPublishConfig());
}
@@ -1921,12 +1933,12 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Checks if the role is marked auto-restart
*
- * @param roleName component name
+ * @param roleGroup component group
*
* @return true if it is auto-restart
*/
- protected boolean isMarkedAutoRestart(String roleName) {
- Component component = getApplicationComponent(roleName);
+ protected boolean isMarkedAutoRestart(String roleGroup) {
+ Component component = getApplicationComponent(roleGroup);
if (component != null) {
return component.getAutoStartOnFailureBoolean();
}
@@ -1960,7 +1972,17 @@ public class AgentProviderService extends AbstractProviderService implements
}
private String getRoleName(String label) {
- return label.substring(label.indexOf(LABEL_MAKER) + LABEL_MAKER.length());
+ int index1 = label.indexOf(LABEL_MAKER);
+ int index2 = label.lastIndexOf(LABEL_MAKER);
+ if (index1 == index2) {
+ return label.substring(index1 + LABEL_MAKER.length());
+ } else {
+ return label.substring(index1 + LABEL_MAKER.length(), index2);
+ }
+ }
+
+ private String getRoleGroup(String label) {
+ return label.substring(label.lastIndexOf(LABEL_MAKER) + LABEL_MAKER.length());
}
private String getContainerId(String label) {
@@ -1970,7 +1992,8 @@ public class AgentProviderService extends AbstractProviderService implements
/**
* Add install command to the heartbeat response
*
- * @param componentName
+ * @param roleName
+ * @param roleGroup
* @param containerId
* @param response
* @param scriptPath
@@ -1981,7 +2004,8 @@ public class AgentProviderService extends AbstractProviderService implements
* @throws SliderException
*/
@VisibleForTesting
- protected void addInstallCommand(String componentName,
+ protected void addInstallCommand(String roleName,
+ String roleGroup,
String containerId,
HeartBeatResponse response,
String scriptPath,
@@ -1998,8 +2022,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.INSTALL.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(componentName);
- cmd.setRole(componentName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
cmd.setPkg(pkg);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
@@ -2007,7 +2031,8 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
+ Map<String, Map<String, String>> configurations =
+ buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
cmd.setConfigurations(configurations);
Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
cmd.setComponentConfigurations(componentConfigurations);
@@ -2032,7 +2057,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addInstallDockerCommand(String componentName,
+ protected void addInstallDockerCommand(String roleName,
+ String roleGroup,
String containerId,
HeartBeatResponse response,
ComponentCommand compCmd,
@@ -2047,15 +2073,15 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.INSTALL.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(componentName);
- cmd.setRole(componentName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
cmd.setConfigurations(configurations);
Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
cmd.setComponentConfigurations(componentConfigurations);
@@ -2070,19 +2096,19 @@ public class AgentProviderService extends AbstractProviderService implements
configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
- cmd.addContainerDetails(componentName, getMetaInfo());
+ cmd.addContainerDetails(roleGroup, getMetaInfo());
Map<String, String> dockerConfig = new HashMap<String, String>();
- if(isYarnDockerContainer(componentName)){
+ if(isYarnDockerContainer(roleGroup)){
//put nothing
cmd.setYarnDockerMode(true);
} else {
dockerConfig.put(
"docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"commandPath"));
dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(componentName, "image"));
+ getConfigFromMetaInfo(roleGroup, "image"));
}
configurations.put("docker", dockerConfig);
@@ -2187,7 +2213,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addStatusCommand(String componentName,
+ protected void addStatusCommand(String roleName,
+ String roleGroup,
String containerId,
HeartBeatResponse response,
String scriptPath,
@@ -2195,9 +2222,9 @@ public class AgentProviderService extends AbstractProviderService implements
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- if (isDockerContainer(componentName) || isYarnDockerContainer(componentName)) {
- addStatusDockerCommand(componentName, containerId, response, scriptPath,
- timeout);
+ if (isDockerContainer(roleGroup) || isYarnDockerContainer(roleGroup)) {
+ addStatusDockerCommand(roleName, roleGroup, containerId, response,
+ scriptPath, timeout);
return;
}
@@ -2205,7 +2232,7 @@ public class AgentProviderService extends AbstractProviderService implements
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(componentName);
+ cmd.setComponentName(roleName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
@@ -2217,7 +2244,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
cmd.setConfigurations(configurations);
@@ -2225,7 +2252,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addStatusDockerCommand(String componentName,
+ protected void addStatusDockerCommand(String roleName,
+ String roleGroup,
String containerId,
HeartBeatResponse response,
String scriptPath,
@@ -2238,7 +2266,7 @@ public class AgentProviderService extends AbstractProviderService implements
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(componentName);
+ cmd.setComponentName(roleName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
@@ -2251,11 +2279,11 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
Map<String, String> dockerConfig = new HashMap<String, String>();
- String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand");
+ String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
if (statusCommand == null) {
- if(isYarnDockerContainer(componentName)){
+ if(isYarnDockerContainer(roleGroup)){
//should complain the required field is null
cmd.setYarnDockerMode(true);
} else {
@@ -2272,7 +2300,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addGetConfigDockerCommand(String componentName,
+ protected void addGetConfigDockerCommand(String roleName, String roleGroup,
String containerId, HeartBeatResponse response) throws SliderException {
assert getAmState().isApplicationLive();
@@ -2280,7 +2308,7 @@ public class AgentProviderService extends AbstractProviderService implements
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(componentName);
+ cmd.setComponentName(roleName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
@@ -2292,11 +2320,11 @@ public class AgentProviderService extends AbstractProviderService implements
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
Map<String, String> dockerConfig = new HashMap<String, String>();
- String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand");
+ String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "statusCommand");
if (statusCommand == null) {
- if(isYarnDockerContainer(componentName)){
+ if(isYarnDockerContainer(roleGroup)){
//should complain the required field is null
cmd.setYarnDockerMode(true);
} else {
@@ -2314,9 +2342,10 @@ public class AgentProviderService extends AbstractProviderService implements
response.addStatusCommand(cmd);
}
- private String getConfigFromMetaInfoWithAppConfigOverriding(String componentName, String configName){
+ private String getConfigFromMetaInfoWithAppConfigOverriding(String roleGroup,
+ String configName){
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- String containerName = getMetaInfo().getApplicationComponent(componentName)
+ String containerName = getMetaInfo().getApplicationComponent(roleGroup)
.getDockerContainers().get(0).getName();
String composedConfigName = null;
String appConfigValue = null;
@@ -2334,23 +2363,23 @@ public class AgentProviderService extends AbstractProviderService implements
} else {
composedConfigName = containerName + "." + configName;
}
- appConfigValue = appConf.getComponentOpt(componentName, composedConfigName,
+ appConfigValue = appConf.getComponentOpt(roleGroup, composedConfigName,
null);
log.debug(
"Docker- value from appconfig component: {} configName: {} value: {}",
- componentName, composedConfigName, appConfigValue);
+ roleGroup, composedConfigName, appConfigValue);
if (appConfigValue == null) {
- appConfigValue = getConfigFromMetaInfo(componentName, configName);
+ appConfigValue = getConfigFromMetaInfo(roleGroup, configName);
log.debug(
"Docker- value from metainfo component: {} configName: {} value: {}",
- componentName, configName, appConfigValue);
+ roleGroup, configName, appConfigValue);
}
return appConfigValue;
}
@VisibleForTesting
- protected void addStartDockerCommand(String componentName,
+ protected void addStartDockerCommand(String roleName, String roleGroup,
String containerId, HeartBeatResponse response,
ComponentCommand startCommand, long timeout, boolean isMarkedAutoRestart)
throws
@@ -2367,8 +2396,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.START.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(componentName);
- cmd.setRole(componentName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<>();
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -2383,32 +2412,28 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(startCommand, timeout, true));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
cmd.setComponentConfigurations(componentConfigurations);
- log.debug("before resolution: " + appConf.toString());
- resolveVariablesForComponentAppConfigs(appConf, componentName, containerId);
- log.debug("after resolution: " + appConf.toString());
-
Map<String, String> dockerConfig = new HashMap<String, String>();
- if (isYarnDockerContainer(componentName)) {
+ if (isYarnDockerContainer(roleGroup)) {
dockerConfig.put(
"docker.startCommand",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"start_command"));
cmd.setYarnDockerMode(true);
} else {
dockerConfig.put(
"docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"commandPath"));
dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(componentName, "image"));
+ getConfigFromMetaInfo(roleGroup, "image"));
// options should always have -d
String options = getConfigFromMetaInfoWithAppConfigOverriding(
- componentName, "options");
+ roleGroup, "options");
if(options != null && !options.isEmpty()){
options = options + " -d";
} else {
@@ -2418,39 +2443,39 @@ public class AgentProviderService extends AbstractProviderService implements
// options should always have -d
dockerConfig.put(
"docker.containerPort",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"containerPort"));
dockerConfig
.put(
"docker.hostPort",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"hostPort"));
dockerConfig.put(
"docker.mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"containerMount"));
dockerConfig
.put(
"docker.host_mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup,
"hostMount"));
dockerConfig.put("docker.additional_param",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName, "additionalParam"));
+ getConfigFromMetaInfoWithAppConfigOverriding(roleGroup, "additionalParam"));
dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo(
- componentName, "containerPath"));
+ roleGroup, "containerPath"));
}
String lifetime = getConfigFromMetaInfoWithAppConfigOverriding(
- componentName, "lifetime");
+ roleGroup, "lifetime");
dockerConfig.put("docker.lifetime", lifetime);
configurations.put("docker", dockerConfig);
String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(
- componentName, "statusCommand");
+ roleGroup, "statusCommand");
if (statusCommand == null) {
- if(isYarnDockerContainer(componentName)){
+ if(isYarnDockerContainer(roleGroup)){
//should complain the required field is null
} else {
statusCommand = "docker top "
@@ -2461,54 +2486,18 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setConfigurations(configurations);
// configurations.get("global").put("exec_cmd", startCommand.getExec());
- cmd.addContainerDetails(componentName, getMetaInfo());
+ cmd.addContainerDetails(roleGroup, getMetaInfo());
log.info("Docker- command: {}", cmd.toString());
response.addExecutionCommand(cmd);
}
- private void resolveVariablesForComponentAppConfigs(
- ConfTreeOperations appConf, String componentName, String containerId)
- throws SliderException {
- Map<String, String> tokens = getStandardTokenMap(appConf, componentName);
- addRoleRelatedTokens(tokens);
- log.debug("docker- tokens: {}", tokens);
-
- MapOperations compConf = appConf.getComponent(componentName);
- if (compConf == null){
- return;
- }
- for(Entry<String, String> element: compConf.entrySet()){
-
- log.debug("docker- key: {} value: {}", element.getKey(), element.getValue());
-
- Object value = element.getValue();
- if (value instanceof String){
- String valueStr = (String)value;
-
- //resolving host names
- for (Map.Entry<String,String> token : tokens.entrySet()) {
- valueStr = valueStr.replaceAll(Pattern.quote(token.getKey()),
- token.getValue());
- compConf.put(element.getKey(), valueStr);
- }
-
- // resolving container ids
- if (valueStr.contains("${CONTAINER_ID}")) {
- valueStr = valueStr.replace("${CONTAINER_ID}",
- containerId);
- compConf.put(element.getKey(), valueStr);
- }
- }
- }
- }
-
- private String getConfigFromMetaInfo(String componentName, String configName) {
+ private String getConfigFromMetaInfo(String roleGroup, String configName) {
String result = null;
List<DockerContainer> containers = getMetaInfo().getApplicationComponent(
- componentName).getDockerContainers();// to support multi container per
+ roleGroup).getDockerContainers();// to support multi container per
// component later
log.debug("Docker- containers metainfo: {}", containers.toString());
if (containers.size() > 0) {
@@ -2579,20 +2568,20 @@ public class AgentProviderService extends AbstractProviderService implements
break;
}
}
- log.debug("Docker- component: {} configName: {} value: {}", componentName, configName, result);
+ log.debug("Docker- component: {} configName: {} value: {}", roleGroup, configName, result);
return result;
}
@VisibleForTesting
- protected void addGetConfigCommand(String componentName, String containerId, HeartBeatResponse response)
- throws SliderException {
+ protected void addGetConfigCommand(String roleName, String roleGroup,
+ String containerId, HeartBeatResponse response) throws SliderException {
assert getAmState().isApplicationLive();
StatusCommand cmd = new StatusCommand();
String clusterName = getClusterName();
cmd.setCommandType(AgentCommandType.STATUS_COMMAND);
- cmd.setComponentName(componentName);
+ cmd.setComponentName(roleName);
cmd.setServiceName(clusterName);
cmd.setClusterName(clusterName);
cmd.setRoleCommand(StatusCommand.GET_CONFIG_COMMAND);
@@ -2606,7 +2595,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected void addStartCommand(String componentName, String containerId, HeartBeatResponse response,
+ protected void addStartCommand(String roleName, String roleGroup, String containerId,
+ HeartBeatResponse response,
String scriptPath, ComponentCommand startCommand,
ComponentCommand stopCommand,
long timeout, boolean isMarkedAutoRestart)
@@ -2624,8 +2614,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.START.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(componentName);
- cmd.setRole(componentName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(CONTAINER_ID, containerId);
@@ -2635,7 +2625,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, roleName, roleGroup);
cmd.setConfigurations(configurations);
Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
cmd.setComponentConfigurations(componentConfigurations);
@@ -2644,7 +2634,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
} else {
if (startCommand == null) {
- throw new SliderException("Expected START command not found for component " + componentName);
+ throw new SliderException("Expected START command not found for component " + roleName);
}
cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
configurations.get("global").put("exec_cmd", startCommand.getExec());
@@ -2664,8 +2654,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmdStop.setClusterName(clusterName);
cmdStop.setRoleCommand(Command.STOP.toString());
cmdStop.setServiceName(clusterName);
- cmdStop.setComponentName(componentName);
- cmdStop.setRole(componentName);
+ cmdStop.setComponentName(roleName);
+ cmdStop.setRole(roleName);
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
.getOption(JAVA_HOME, ""));
@@ -2689,13 +2679,13 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
cmdStop.setConfigurations(configurationsStop);
response.addExecutionCommand(cmdStop);
}
@VisibleForTesting
- protected void addUpgradeCommand(String componentName, String containerId,
+ protected void addUpgradeCommand(String roleName, String roleGroup, String containerId,
HeartBeatResponse response, String scriptPath, long timeout)
throws SliderException {
assert getAmState().isApplicationLive();
@@ -2711,8 +2701,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setClusterName(clusterName);
cmd.setRoleCommand(Command.UPGRADE.toString());
cmd.setServiceName(clusterName);
- cmd.setComponentName(componentName);
- cmd.setRole(componentName);
+ cmd.setComponentName(roleName);
+ cmd.setRole(roleName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions()
.getMandatoryOption(JAVA_HOME));
@@ -2721,12 +2711,12 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
cmd.setConfigurations(configurations);
response.addExecutionCommand(cmd);
}
- protected void addStopCommand(String componentName, String containerId,
+ protected void addStopCommand(String roleName, String roleGroup, String containerId,
HeartBeatResponse response, String scriptPath, long timeout,
boolean isInUpgradeMode) throws SliderException {
assert getAmState().isApplicationLive();
@@ -2745,8 +2735,8 @@ public class AgentProviderService extends AbstractProviderService implements
// UPGRADE_STOP
cmdStop.setRoleCommand(Command.transform(Command.STOP, isInUpgradeMode));
cmdStop.setServiceName(clusterName);
- cmdStop.setComponentName(componentName);
- cmdStop.setRole(componentName);
+ cmdStop.setComponentName(roleName);
+ cmdStop.setRole(roleName);
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
.getMandatoryOption(JAVA_HOME));
@@ -2755,7 +2745,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
- appConf, containerId, componentName);
+ appConf, containerId, roleName, roleGroup);
cmdStop.setConfigurations(configurationsStop);
response.addExecutionCommand(cmdStop);
}
@@ -2794,23 +2784,24 @@ public class AgentProviderService extends AbstractProviderService implements
}
private Map<String, Map<String, String>> buildCommandConfigurations(
- ConfTreeOperations appConf, String containerId, String componentName)
+ ConfTreeOperations appConf, String containerId, String roleName, String roleGroup)
throws SliderException {
Map<String, Map<String, String>> configurations =
new TreeMap<String, Map<String, String>>();
- Map<String, String> tokens = getStandardTokenMap(appConf, componentName);
+ Map<String, String> tokens = getStandardTokenMap(appConf, roleName, roleGroup);
+ tokens.put("${CONTAINER_ID}", containerId);
Set<String> configs = new HashSet<String>();
- configs.addAll(getApplicationConfigurationTypes(componentName));
+ configs.addAll(getApplicationConfigurationTypes(roleGroup));
configs.addAll(getSystemConfigurationsRequested(appConf));
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens, containerId, componentName);
- if (appConf.getComponent(componentName) != null) {
- addNamedConfiguration(configType, appConf.getComponent(componentName).options,
- configurations, tokens, containerId, componentName);
+ configurations, tokens, containerId, roleName);
+ if (appConf.getComponent(roleGroup) != null) {
+ addNamedConfiguration(configType, appConf.getComponent(roleGroup).options,
+ configurations, tokens, containerId, roleName);
}
}
@@ -2846,7 +2837,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf,
- String componentName) throws SliderException {
+ String componentName, String componentGroup) throws SliderException {
Map<String, String> tokens = new HashMap<String, String>();
String nnuri = appConf.get("site.fs.defaultFS");
tokens.put("${NN_URI}", nnuri);
@@ -2859,6 +2850,9 @@ public class AgentProviderService extends AbstractProviderService implements
.getMandatoryOption(InternalKeys.INTERNAL_DATA_DIR_PATH));
tokens.put("${JAVA_HOME}", appConf.get(AgentKeys.JAVA_HOME));
tokens.put("${COMPONENT_NAME}", componentName);
+ if (!componentName.equals(componentGroup) && componentName.startsWith(componentGroup)) {
+ tokens.put("${COMPONENT_ID}", componentName.substring(componentGroup.length()));
+ }
return tokens;
}
@@ -2879,7 +2873,7 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
- protected List<String> getApplicationConfigurationTypes(String componentName) {
+ protected List<String> getApplicationConfigurationTypes(String roleGroup) {
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
@@ -2889,7 +2883,7 @@ public class AgentProviderService extends AbstractProviderService implements
configList.add(configFile.getDictionaryName());
}
for (Component component : getMetaInfo().getApplication().getComponents()) {
- if (!component.getName().equals(componentName)) {
+ if (!component.getName().equals(roleGroup)) {
continue;
}
if (component.getDockerContainers() == null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
index f89a8a6..036d98e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -58,12 +58,12 @@ public class Metainfo {
this.application = application;
}
- public Component getApplicationComponent(String roleName) {
+ public Component getApplicationComponent(String roleGroup) {
if (application == null) {
log.error("Malformed app definition: Expect application as the top level element for metainfo");
} else {
for (Component component : application.getComponents()) {
- if (component.getName().equals(roleName)) {
+ if (component.getName().equals(roleGroup)) {
return component;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
index f0dab8c..67d3647 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/slideram/SliderAMProviderService.java
@@ -82,7 +82,7 @@ public class SliderAMProviderService extends AbstractProviderService implements
public void buildContainerLaunchContext(ContainerLauncher containerLauncher,
AggregateConf instanceDefinition,
Container container,
- String role,
+ ProviderRole role,
SliderFileSystem sliderFileSystem,
Path generatedConfPath,
MapOperations resourceComponent,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
index 7515c1a..3cfe167 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/RoleLaunchService.java
@@ -88,8 +88,7 @@ public class RoleLaunchService
/**
* Construct an instance of the launcher
- * @param startOperation the callback to start the opreation
- * @param actionQueue
+ * @param queueAccess
* @param provider the provider
* @param fs filesystem
* @param generatedConfDirPath path in the FS for the generated dir
@@ -120,8 +119,7 @@ public class RoleLaunchService
/**
* Start an asychronous launch operation
- * @param container container target
- * @param role role
+ * @param assignment container assignment
* @param clusterSpec cluster spec to use for template
* @param credentials credentials to use
*/
@@ -130,13 +128,14 @@ public class RoleLaunchService
Credentials credentials) {
RoleStatus role = assignment.role;
String roleName = role.getName();
+ String roleGroup = role.getGroup();
// prelaunch safety check
Preconditions.checkArgument(provider.isSupportedRole(roleName));
RoleLaunchService.RoleLauncher launcher =
new RoleLaunchService.RoleLauncher(assignment,
clusterSpec,
- clusterSpec.getResourceOperations().getOrAddComponent(roleName),
- clusterSpec.getAppConfOperations().getOrAddComponent(roleName),
+ clusterSpec.getResourceOperations().getOrAddComponent(roleGroup),
+ clusterSpec.getAppConfOperations().getOrAddComponent(roleGroup),
credentials);
execute(launcher);
}
@@ -149,7 +148,6 @@ public class RoleLaunchService
private final ContainerAssignment assignment;
// Allocated container
public final Container container;
- public final String containerRole;
private final MapOperations resourceComponent;
private final MapOperations appComponent;
private final AggregateConf instanceDefinition;
@@ -171,7 +169,6 @@ public class RoleLaunchService
assert appComponent != null;
ProviderRole providerRole = roleStatus.getProviderRole();
assert providerRole != null;
- this.containerRole = providerRole.name;
this.role = providerRole;
this.resourceComponent = resourceComponent;
this.appComponent = appComponent;
@@ -186,7 +183,8 @@ public class RoleLaunchService
public String toString() {
return "RoleLauncher{" +
"container=" + container.getId() +
- ", containerRole='" + containerRole + '\'' +
+ ", containerRole='" + role.name + '\'' +
+ ", containerGroup='" + role.group + '\'' +
'}';
}
@@ -200,7 +198,7 @@ public class RoleLaunchService
log.debug("Launching container {} into role {}",
container.getId(),
- containerRole);
+ role.name);
//now build up the configuration data
Path containerTmpDirPath =
@@ -208,7 +206,7 @@ public class RoleLaunchService
provider.buildContainerLaunchContext(containerLauncher,
instanceDefinition,
container,
- containerRole,
+ role,
fs,
generatedConfDirPath,
resourceComponent,
@@ -223,7 +221,8 @@ public class RoleLaunchService
commandsAsString);
instance.command = commandsAsString;
- instance.role = containerRole;
+ instance.role = role.name;
+ instance.group = role.group;
instance.roleId = role.id;
instance.appVersion = instanceDefinition.getAppConfOperations()
.getGlobalOptions().get(SliderKeys.APP_VERSION);
@@ -241,7 +240,7 @@ public class RoleLaunchService
}
log.info("Container launch delay for {} set to {} seconds",
role.name, delay);
- actionQueue.schedule(new ActionStartContainer("starting " + containerRole,
+ actionQueue.schedule(new ActionStartContainer("starting " + role.name,
container,
containerLauncher.completeContainerLaunch(),
instance,
@@ -249,7 +248,7 @@ public class RoleLaunchService
TimeUnit.SECONDS));
} catch (Exception e) {
log.error("Exception thrown while trying to start {}: {}",
- containerRole, e, e);
+ role.name, e, e);
raisedException = e;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 73f24c7..8d30da7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1382,10 +1382,11 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Register/re-register an ephemeral container that is already in the app state
* @param id the component
* @param description component description
+ * @param type component type
* @return true if the component is registered
*/
- public boolean registerComponent(ContainerId id, String description) throws
- IOException {
+ public boolean registerComponent(ContainerId id, String description,
+ String type) throws IOException {
RoleInstance instance = appState.getOwnedContainer(id);
if (instance == null) {
return false;
@@ -1399,7 +1400,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
container.set(YarnRegistryAttributes.YARN_PERSISTENCE,
PersistencePolicies.CONTAINER);
MapOperations compOps = getInstanceDefinition().getAppConfOperations().
- getComponent(description);
+ getComponent(type);
setProvidedServiceRecordAttributes(compOps, container);
try {
yarnRegistryOperations.putComponent(cid, container);
@@ -2270,7 +2271,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
nmClientAsync.getContainerStatusAsync(containerId,
cinfo.container.getNodeId());
// push out a registration
- queue(new RegisterComponentInstance(containerId, cinfo.role,
+ queue(new RegisterComponentInstance(containerId, cinfo.role, cinfo.group,
0, TimeUnit.MILLISECONDS));
} else {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/9130f0ea/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
index 3145ecb..4cf4981 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/actions/RegisterComponentInstance.java
@@ -34,14 +34,17 @@ public class RegisterComponentInstance extends AsyncAction {
public final ContainerId containerId;
public final String description;
+ public final String type;
public RegisterComponentInstance(ContainerId containerId,
String description,
+ String type,
long delay,
TimeUnit timeUnit) {
super("RegisterComponentInstance :" + containerId,
delay, timeUnit);
this.description = description;
+ this.type = type;
Preconditions.checkArgument(containerId != null);
this.containerId = containerId;
}
@@ -51,6 +54,6 @@ public class RegisterComponentInstance extends AsyncAction {
QueueAccess queueService,
AppState appState) throws Exception {
- appMaster.registerComponent(containerId, description);
+ appMaster.registerComponent(containerId, description, type);
}
}
[08/31] incubator-slider git commit: SLIDER-1121 fix slider AM port
allocation race condition
Posted by el...@apache.org.
SLIDER-1121 fix slider AM port allocation race condition
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/55dd69dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/55dd69dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/55dd69dd
Branch: refs/heads/branches/branch-0.91
Commit: 55dd69dd36e743e645a67a53429c185dc4612307
Parents: 4b7d5c8
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Tue May 17 06:57:29 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Tue May 17 06:57:29 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/common/SliderKeys.java | 5 --
.../apache/slider/common/tools/PortScanner.java | 16 +----
.../apache/slider/common/tools/SliderUtils.java | 16 +++++
.../server/appmaster/SliderAppMaster.java | 63 ++++++++------------
.../appmaster/web/rest/agent/AgentWebApp.java | 35 ++++++++++-
5 files changed, 79 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 1d2d5f8..05c7048 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -266,11 +266,6 @@ public interface SliderKeys extends SliderXmlConfKeys {
* {@value}
*/
String KEY_ALLOWED_PORT_RANGE = "site.global.slider.allowed.ports";
-
- /**
- * Allowed port range
- */
- String KEY_AM_ALLOWED_PORT_RANGE = "slider.am.allowed.port.range";
/**
* env var for custom JVM options.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index b5b21ce..5b80f9f 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -19,6 +19,7 @@ package org.apache.slider.common.tools;
import org.apache.slider.common.SliderExitCodes;
import org.apache.slider.core.exceptions.SliderException;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -39,8 +40,6 @@ public class PortScanner {
public PortScanner() {
}
- int nextPort = 1024;
-
public void setPortRange(String input) {
// first split based on commas
Set<Integer> inputPorts= new TreeSet<Integer>();
@@ -68,23 +67,14 @@ public class PortScanner {
return remainingPortsToCheck;
}
- public int getAvailablePort() throws SliderException {
+ public int getAvailablePort() throws SliderException, IOException {
if (remainingPortsToCheck != null) {
return getAvailablePortViaPortArray();
} else {
- return getAvailablePortViaCounter();
+ return SliderUtils.getOpenPort();
}
}
- private int getAvailablePortViaCounter() throws SliderException {
- int port;
- do {
- port = nextPort;
- nextPort++;
- } while (!SliderUtils.isPortAvailable(port));
- return port;
- }
-
private int getAvailablePortViaPortArray() throws SliderException {
boolean found = false;
int availablePort = -1;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index eae80f5..746e468 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -1114,6 +1114,22 @@ public final class SliderUtils {
}
/**
+ * Get a random open port
+ * @return true if the port was available for listening on
+ */
+ public static int getOpenPort() throws IOException {
+ ServerSocket socket = null;
+ try {
+ socket = new ServerSocket(0);
+ return socket.getLocalPort();
+ } finally {
+ if (socket != null) {
+ socket.close();
+ }
+ }
+ }
+
+ /**
* See if a port is available for listening on by trying to listen
* on it and seeing if that works or fails.
* @param port port to listen to
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 8d30da7..0776a6c 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -404,11 +404,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
private SecurityConfiguration securityConfiguration;
/**
- * The port for the web application
- */
- private int webAppPort;
-
- /**
* Is security enabled?
* Set early on in the {@link #createAndRunCluster(String)} operation.
*/
@@ -776,11 +771,23 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
uploadServerCertForLocalization(clustername, fs);
}
- webAppPort = getPortToRequest();
- if (webAppPort == 0) {
- // failure to find a port
- throw new BadConfigException("Failed to fix a web application port");
- }
+ // Web service endpoints: initialize
+ WebAppApiImpl webAppApi =
+ new WebAppApiImpl(
+ stateForProviders,
+ providerService,
+ certificateManager,
+ registryOperations,
+ metricsAndMonitoring,
+ actionQueues,
+ this,
+ contentCache);
+ initAMFilterOptions(serviceConf);
+
+ // start the agent web app
+ startAgentWebApp(appInformation, serviceConf, webAppApi);
+ int webAppPort = deployWebApplication(webAppApi);
+
String scheme = WebAppUtils.HTTP_PREFIX;
appMasterTrackingUrl = scheme + appMasterHostname + ":" + webAppPort;
@@ -926,7 +933,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
Path tmpDirPath = new Path(amTmpDir);
Path launcherTmpDirPath = new Path(tmpDirPath, rolesTmpSubdir);
fs.getFileSystem().mkdirs(launcherTmpDirPath);
-
+
//launcher service
launchService = new RoleLaunchService(actionQueues,
providerService,
@@ -972,25 +979,6 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
scheduleEscalation(instanceDefinition.getInternal());
try {
-
- // Web service endpoints: initialize
-
- WebAppApiImpl webAppApi =
- new WebAppApiImpl(
- stateForProviders,
- providerService,
- certificateManager,
- registryOperations,
- metricsAndMonitoring,
- actionQueues,
- this,
- contentCache);
- initAMFilterOptions(serviceConf);
-
- // start the agent web app
- startAgentWebApp(appInformation, serviceConf, webAppApi);
- deployWebApplication(webAppPort, webAppApi);
-
// schedule YARN Registry registration
queue(new ActionRegisterServiceInstance(clustername, appid));
@@ -1051,7 +1039,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
}
/**
- * List the node reports: uses {@link #yarnClient} as the login user
+ * List the node reports: uses {@link SliderYarnClientImpl} as the login user
* @param yarnClient client to the RM
* @return the node reports
* @throws IOException
@@ -1083,17 +1071,18 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* Creates and starts the web application, and adds a
* <code>WebAppService</code> service under the AM, to ensure
* a managed web application shutdown.
- * @param port port to deploy the web application on
* @param webAppApi web app API instance
+ * @return port the web application is deployed on
* @throws IOException general problems starting the webapp (network, etc)
* @throws WebAppException other issues
*/
- private void deployWebApplication(int port, WebAppApiImpl webAppApi)
- throws IOException {
+ private int deployWebApplication(WebAppApiImpl webAppApi)
+ throws IOException, SliderException {
try {
webApp = new SliderAMWebApp(webAppApi);
HttpConfig.Policy policy = HttpConfig.Policy.HTTP_ONLY;
+ int port = getPortToRequest();
log.info("Launching web application at port {} with policy {}", port, policy);
WebApps.$for(SliderAMWebApp.BASE_PATH,
@@ -1101,7 +1090,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
webAppApi,
RestPaths.WS_CONTEXT)
.withHttpPolicy(getConfig(), policy)
- .at(port)
+ .at("0.0.0.0", port, true)
.inDevMode()
.start(webApp);
@@ -1109,6 +1098,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
new WebAppService<>("slider", webApp);
deployChildService(webAppService);
+ return webApp.port();
} catch (WebAppException e) {
if (e.getCause() instanceof IOException) {
throw (IOException)e.getCause();
@@ -1167,8 +1157,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
* @return the port to request.
* @throws SliderException
*/
- private int getPortToRequest()
- throws SliderException {
+ private int getPortToRequest() throws SliderException, IOException {
return portScanner.getAvailablePort();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55dd69dd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
index 200fbc2..3a3b0c0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/AgentWebApp.java
@@ -25,7 +25,6 @@ import com.sun.jersey.spi.inject.SingletonTypeInjectableProvider;
import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.providers.agent.AgentKeys;
import org.apache.slider.server.appmaster.web.WebAppApi;
-import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.SecurityUtils;
import org.mortbay.jetty.Connector;
import org.mortbay.jetty.Server;
@@ -40,6 +39,7 @@ import javax.ws.rs.ext.Provider;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
+import java.net.BindException;
import java.util.Set;
/**
@@ -91,6 +91,7 @@ public class AgentWebApp implements Closeable {
new QueuedThreadPool(
configsMap.getOptionInt("agent.threadpool.size.max", 25)));
agentServer.setStopAtShutdown(true);
+ agentServer.setGracefulShutdown(1000);
SslSelectChannelConnector ssl1WayConnector = createSSLConnector(false, port);
SslSelectChannelConnector ssl2WayConnector =
@@ -115,6 +116,7 @@ public class AgentWebApp implements Closeable {
agentRoot.addServlet(agent, "/*");
try {
+ openListeners();
agentServer.start();
} catch (IOException e) {
LOG.error("Unable to start agent server", e);
@@ -131,6 +133,37 @@ public class AgentWebApp implements Closeable {
}
+ private void openListeners() throws Exception {
+ // from HttpServer2.openListeners()
+ for (Connector listener : agentServer.getConnectors()) {
+ if (listener.getLocalPort() != -1) {
+ // This listener is either started externally or has been bound
+ continue;
+ }
+ int port = listener.getPort();
+ while (true) {
+ // jetty has a bug where you can't reopen a listener that previously
+ // failed to open w/o issuing a close first, even if the port is changed
+ try {
+ listener.close();
+ listener.open();
+ LOG.info("Jetty bound to port " + listener.getLocalPort());
+ break;
+ } catch (BindException ex) {
+ if (port == 0) {
+ BindException be = new BindException("Port in use: "
+ + listener.getHost() + ":" + listener.getPort());
+ be.initCause(ex);
+ throw be;
+ }
+ }
+ // try the next port number
+ listener.setPort(++port);
+ Thread.sleep(100);
+ }
+ }
+ }
+
private SslSelectChannelConnector createSSLConnector(boolean needClientAuth, int port) {
SslSelectChannelConnector sslConnector = new
SslSelectChannelConnector();
[31/31] incubator-slider git commit: SLIDER-874 rat-check should
define exclusions using pluginManagement
Posted by el...@apache.org.
SLIDER-874 rat-check should define exclusions using pluginManagement
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/fff0e280
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/fff0e280
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/fff0e280
Branch: refs/heads/branches/branch-0.91
Commit: fff0e2805f628669963544e571bd9693a8304537
Parents: 55193be
Author: Josh Elser <el...@apache.org>
Authored: Thu Jun 16 23:23:21 2016 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jun 16 23:23:21 2016 -0400
----------------------------------------------------------------------
app-packages/accumulo/pom.xml | 20 +++++++++++
app-packages/pom.xml | 46 +++++++++++++------------
pom.xml | 70 ++++++++++++++++++++------------------
3 files changed, 80 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fff0e280/app-packages/accumulo/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/pom.xml b/app-packages/accumulo/pom.xml
index 193245d..d366c81 100644
--- a/app-packages/accumulo/pom.xml
+++ b/app-packages/accumulo/pom.xml
@@ -224,6 +224,26 @@
</plugins>
</build>
</profile>
+ <profile>
+ <id>rat</id>
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <exclude>src/license/THIRD-PARTY.properties</exclude>
+ <exclude>**/*.json</exclude>
+ <exclude>src/test/resources/test_password_file</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+ </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fff0e280/app-packages/pom.xml
----------------------------------------------------------------------
diff --git a/app-packages/pom.xml b/app-packages/pom.xml
index aa9632b..65fb5cb 100644
--- a/app-packages/pom.xml
+++ b/app-packages/pom.xml
@@ -96,28 +96,30 @@
<id>rat</id>
<!-- RAT profile -->
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/test_password_file</exclude>
- <exclude>command-logger/**</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/test_password_file</exclude>
+ <exclude>command-logger/**</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
</profile>
</profiles>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/fff0e280/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ddbfb82..0a6781f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1394,40 +1394,42 @@
<profile>
<id>rat</id>
<build>
- <plugins>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <version>${apache-rat-plugin.version}</version>
- <executions>
- <execution>
- <id>check-licenses</id>
- <goals>
- <goal>check</goal>
- </goals>
- </execution>
- </executions>
- <configuration>
- <excludes>
- <exclude>**/*.json</exclude>
- <exclude>**/*.tar</exclude>
- <exclude>**/THIRD-PARTY.properties</exclude>
- <exclude>**/build.properties</exclude>
- <exclude>**/regionservers</exclude>
- <exclude>**/slaves</exclude>
- <exclude>**/httpfs-signature.secret</exclude>
- <exclude>**/dfs.exclude</exclude>
- <exclude>**/*.iml</exclude>
- <exclude>**/rat.txt</exclude>
- <exclude>**/test_password_file</exclude>
- <exclude>DISCLAIMER</exclude>
- <exclude>app-packages/hbase/target/**</exclude>
- <exclude>target/*</exclude>
- <exclude>DEPENDENCIES</exclude>
- </excludes>
- </configuration>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <version>${apache-rat-plugin.version}</version>
+ <executions>
+ <execution>
+ <id>check-licenses</id>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ </execution>
+ </executions>
+ <configuration>
+ <excludes>
+ <exclude>**/*.json</exclude>
+ <exclude>**/*.tar</exclude>
+ <exclude>**/THIRD-PARTY.properties</exclude>
+ <exclude>**/build.properties</exclude>
+ <exclude>**/regionservers</exclude>
+ <exclude>**/slaves</exclude>
+ <exclude>**/httpfs-signature.secret</exclude>
+ <exclude>**/dfs.exclude</exclude>
+ <exclude>**/*.iml</exclude>
+ <exclude>**/rat.txt</exclude>
+ <exclude>**/test_password_file</exclude>
+ <exclude>DISCLAIMER</exclude>
+ <exclude>app-packages/hbase/target/**</exclude>
+ <exclude>target/*</exclude>
+ <exclude>DEPENDENCIES</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
</profile>
[15/31] incubator-slider git commit: SLIDER-1129 remove all
httpclient maven imports: use the one the Hadoop version gives you
Posted by el...@apache.org.
SLIDER-1129 remove all httpclient maven imports: use the one the Hadoop version gives you
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1154470d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1154470d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1154470d
Branch: refs/heads/branches/branch-0.91
Commit: 1154470d33e1eabde81e020119145996949e0df9
Parents: 34f559a
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 22:40:16 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 22:40:16 2016 +0100
----------------------------------------------------------------------
pom.xml | 31 +++++--------------------------
slider-core/pom.xml | 2 ++
2 files changed, 7 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1154470d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 36f16c0..b1f2000 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,8 +154,6 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
- <httpclient4.version>4.5.2</httpclient4.version>
- <httpcore.version>4.4.4</httpcore.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -522,12 +520,6 @@
</exclusions>
</dependency>
- <!-- hadoop-client includes the following jars, so they do not need to be
- included separately:
- hadoop-common, hadoop-hdfs (client?), hadoop-mapreduce-client-app,
- hadoop-yarn-api, hadoop-mapreduce-client-core,
- hadoop-mapreduce-client-jobclient, and hadoop-annotations
- -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
@@ -541,14 +533,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -569,14 +553,6 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -697,6 +673,8 @@
<version>${commons-lang.version}</version>
</dependency>
+<!--
+
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
@@ -708,7 +686,7 @@
</exclusion>
</exclusions>
</dependency>
-
+ T
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
@@ -720,7 +698,8 @@
<artifactId>httpcore</artifactId>
<version>${httpcore.version}</version>
</dependency>
-
+-->
+
<!-- ======================================================== -->
<!-- HBASE -->
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1154470d/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 6e942c0..8b921a5 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -326,6 +326,7 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+<!--
<dependency>
<groupId>commons-httpclient</groupId>
@@ -337,6 +338,7 @@
<artifactId>httpclient</artifactId>
<scope>test</scope>
</dependency>
+-->
<dependency>
<groupId>commons-lang</groupId>
[29/31] incubator-slider git commit: SLIDER-1142 increase AM memory
for some funtests and add checks for None in agent
Posted by el...@apache.org.
SLIDER-1142 increase AM memory for some funtests and add checks for None in agent
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2b41da2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2b41da2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2b41da2d
Branch: refs/heads/branches/branch-0.91
Commit: 2b41da2d56a6165eb54fa0fc9cab6900fad9d177
Parents: 454f2cf
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 15 07:21:15 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 15 08:08:37 2016 -0700
----------------------------------------------------------------------
slider-agent/src/main/python/agent/ActionQueue.py | 2 +-
slider-agent/src/main/python/agent/main.py | 2 +-
slider-core/src/test/app_packages/test_command_log/resources.json | 2 +-
.../test/app_packages/test_command_log/resources_add_on_pkg.json | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index be6af53..7514337 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -164,7 +164,7 @@ class ActionQueue(threading.Thread):
if ActionQueue.STORE_APPLIED_CONFIG in command['commandParams']:
store_config = 'true' == command['commandParams'][ActionQueue.STORE_APPLIED_CONFIG]
store_command = False
- if 'roleParams' in command and ActionQueue.AUTO_RESTART in command['roleParams']:
+ if 'roleParams' in command and command['roleParams'] is not None and ActionQueue.AUTO_RESTART in command['roleParams']:
store_command = 'true' == command['roleParams'][ActionQueue.AUTO_RESTART]
if store_command:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index 39de1ae..1932a37 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -60,7 +60,7 @@ def signal_handler(signum, frame):
if docker_mode:
tmpdir = controller.actionQueue.dockerManager.stop_container()
- if controller is not None and hasattr(controller, 'stopCommand'):
+ if controller is not None and hasattr(controller, 'stopCommand') and controller.stopCommand is not None:
controller.stopCommand = _increment_task_id(controller.stopCommand)
controller.appGracefulStopQueued = True
controller.actionQueue.execute_command(controller.stopCommand)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-core/src/test/app_packages/test_command_log/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources.json b/slider-core/src/test/app_packages/test_command_log/resources.json
index 18c505f..f6a401a 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources.json
@@ -11,7 +11,7 @@
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2b41da2d/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
index 3f71eb3..25e090e 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
@@ -11,7 +11,7 @@
"yarn.component.instances": "1"
},
"slider-appmaster": {
- "yarn.memory": "256"
+ "yarn.memory": "512"
}
}
}
[27/31] incubator-slider git commit: SLIDER-1079 converting the
metainfo cache to request level
Posted by el...@apache.org.
SLIDER-1079 converting the metainfo cache to request level
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/24a5062a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/24a5062a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/24a5062a
Branch: refs/heads/branches/branch-0.91
Commit: 24a5062ac965208ba11115d63a7c94413b755f71
Parents: f95ba34
Author: Gour Saha <go...@apache.org>
Authored: Tue Jun 14 16:44:19 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Tue Jun 14 16:48:27 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/providers/agent/AgentClientProvider.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/24a5062a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 12d581c..f3dcd1d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -86,7 +86,7 @@ public class AgentClientProvider extends AbstractClientProvider
public static final String E_COULD_NOT_READ_METAINFO
= "Not a valid app package. Could not read metainfo.";
- protected static Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
+ protected Map<String, Metainfo> metaInfoMap = new ConcurrentHashMap<String, Metainfo>();
protected AgentClientProvider(Configuration conf) {
super(conf);
[25/31] incubator-slider git commit: SLIDER-1129 cut out (currently
commented out) references to the httpclient and httpcore.
Posted by el...@apache.org.
SLIDER-1129 cut out (currently commented out) references to the httpclient and httpcore.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eb045245
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eb045245
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eb045245
Branch: refs/heads/branches/branch-0.91
Commit: eb0452450a77f5493e5038a9985da619a1d4fc85
Parents: c7a173c
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 18:26:11 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jun 9 11:59:40 2016 +0100
----------------------------------------------------------------------
pom.xml | 28 ----------------------------
1 file changed, 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eb045245/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b1f2000..ddbfb82 100644
--- a/pom.xml
+++ b/pom.xml
@@ -153,7 +153,6 @@
<guava.version>11.0.2</guava.version>
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
- <httpclient.version>3.1</httpclient.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -673,33 +672,6 @@
<version>${commons-lang.version}</version>
</dependency>
-<!--
-
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- <version>${httpclient.version}</version>
- <exclusions>
- <exclusion>
- <groupId>commons-codec</groupId>
- <artifactId>commons-codec</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
- T
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${httpclient4.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
- <version>${httpcore.version}</version>
- </dependency>
--->
-
<!-- ======================================================== -->
<!-- HBASE -->
<!-- ======================================================== -->
[09/31] incubator-slider git commit: SLIDER-1125 fix & typo
Posted by el...@apache.org.
SLIDER-1125 fix & typo
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/39e76023
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/39e76023
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/39e76023
Branch: refs/heads/branches/branch-0.91
Commit: 39e760232bebc4fa33a134a8d166045bad1421e9
Parents: 55dd69d
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu May 19 11:58:19 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu May 19 11:58:19 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/providers/agent/AgentProviderService.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/39e76023/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index d5fae19..f20757a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -906,7 +906,7 @@ public class AgentProviderService extends AbstractProviderService implements
role = amState.getOwnedContainer(containerId);
role.ip = status.getIp();
}
- if(status.getHostname() != null & !status.getHostname().isEmpty()){
+ if(status.getHostname() != null && !status.getHostname().isEmpty()){
role = amState.getOwnedContainer(containerId);
role.hostname = status.getHostname();
}
[10/31] incubator-slider git commit: SLIDER-1127 slider workflow
classes have some javadoc/codestyle issues
Posted by el...@apache.org.
SLIDER-1127 slider workflow classes have some javadoc/codestyle issues
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f3334cc3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f3334cc3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f3334cc3
Branch: refs/heads/branches/branch-0.91
Commit: f3334cc3c20974e67f1647aec35905179ad8d3d8
Parents: 39e7602
Author: Steve Loughran <st...@apache.org>
Authored: Fri May 20 10:08:15 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri May 20 10:08:15 2016 +0100
----------------------------------------------------------------------
.../slider/server/services/workflow/LongLivedProcess.java | 8 ++++----
.../services/workflow/ServiceTerminatingRunnable.java | 4 ++--
.../server/services/workflow/ServiceThreadFactory.java | 10 +++++-----
.../server/services/workflow/WorkflowCallbackService.java | 9 ++++++---
4 files changed, 17 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f3334cc3/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
index 9bd4d3d..9e9e7ac 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/LongLivedProcess.java
@@ -53,8 +53,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
* <li>Output is streamed to the output logger provided</li>.
* <li>the input stream is closed as soon as the process starts.</li>
* <li>The most recent lines of output are saved to a linked list</li>.
- * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent}, is raised on the start
- * and finish of a process.</li>
+ * <li>A synchronous callback, {@link LongLivedProcessLifecycleEvent},
+ * is raised on the start and finish of a process.</li>
* </ol>
*
*/
@@ -65,12 +65,12 @@ public class LongLivedProcess implements Runnable {
public static final int RECENT_LINE_LOG_LIMIT = 64;
/**
- * Const defining the time in millis between polling for new text
+ * Const defining the time in millis between polling for new text.
*/
private static final int STREAM_READER_SLEEP_TIME = 200;
/**
- * limit on the length of a stream before it triggers an automatic newline
+ * limit on the length of a stream before it triggers an automatic newline.
*/
private static final int LINE_LENGTH = 256;
private final ProcessBuilder processBuilder;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f3334cc3/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
index ec0c61d..dc591df 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceTerminatingRunnable.java
@@ -32,7 +32,7 @@ public class ServiceTerminatingRunnable implements Runnable {
private Exception exception;
/**
- * Create an instance
+ * Create an instance.
* @param owner owning service
* @param action action to execute before terminating the service
*/
@@ -44,7 +44,7 @@ public class ServiceTerminatingRunnable implements Runnable {
}
/**
- * Get the owning service
+ * Get the owning service.
* @return the service to receive notification when
* the runnable completes.
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f3334cc3/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
index 99a2c28..737197b 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/ServiceThreadFactory.java
@@ -42,9 +42,10 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public class ServiceThreadFactory implements ThreadFactory {
- private static AtomicInteger counter = new AtomicInteger(1);
+ private static final AtomicInteger counter = new AtomicInteger(1);
+
/**
- * Default format for thread names: {@value}
+ * Default format for thread names: {@value}.
*/
public static final String DEFAULT_NAMING_FORMAT = "%s-%03d";
private final String name;
@@ -68,8 +69,7 @@ public class ServiceThreadFactory implements ThreadFactory {
}
/**
- *
- * Create an instance with the default naming format
+ * Create an instance with the default naming format.
* @param name base thread name
* @param daemons flag to indicate the threads should be marked as daemons
*/
@@ -89,7 +89,7 @@ public class ServiceThreadFactory implements ThreadFactory {
}
/**
- * Create a single thread executor using this naming policy
+ * Create a single thread executor using this naming policy.
* @param name base thread name
* @param daemons flag to indicate the threads should be marked as daemons
* @return an executor
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f3334cc3/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
index 74d5623..65d14b7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/workflow/WorkflowCallbackService.java
@@ -30,7 +30,9 @@ import java.util.concurrent.TimeUnit;
/**
* A service that calls the supplied callback when it is started -after the
- * given delay. It can be configured to stop itself after the callback has
+ * given delay.
+ *
+ * It can be configured to stop itself after the callback has
* completed, marking any exception raised as the exception of this service.
* The notifications come in on a callback thread -a thread that is only
* started in this service's <code>start()</code> operation.
@@ -38,9 +40,10 @@ import java.util.concurrent.TimeUnit;
public class WorkflowCallbackService<V> extends
WorkflowScheduledExecutorService<ScheduledExecutorService> {
protected static final Logger LOG =
- LoggerFactory.getLogger(WorkflowCallbackService.class);
+ LoggerFactory.getLogger(WorkflowCallbackService.class);
+
/**
- * This is the callback
+ * This is the callback.
*/
private final Callable<V> callback;
private final int delay;
[14/31] incubator-slider git commit: SLIDER-1129 increment versions
of httpclient/http core dependencies; move to test scope only
Posted by el...@apache.org.
SLIDER-1129 increment versions of httpclient/http core dependencies; move to test scope only
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34f559a6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34f559a6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34f559a6
Branch: refs/heads/branches/branch-0.91
Commit: 34f559a6716e429da5eda8ad17aab970fb7d6641
Parents: 5c391cc
Author: Steve Loughran <st...@apache.org>
Authored: Mon May 23 19:37:04 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Mon May 23 19:37:04 2016 +0100
----------------------------------------------------------------------
pom.xml | 5 +++--
slider-core/pom.xml | 6 +-----
2 files changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f559a6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index b616e15..36f16c0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -154,7 +154,8 @@
<gson.version>2.2.2</gson.version>
<guice.version>3.0</guice.version>
<httpclient.version>3.1</httpclient.version>
- <httpclient4.version>4.2.5</httpclient4.version>
+ <httpclient4.version>4.5.2</httpclient4.version>
+ <httpcore.version>4.4.4</httpcore.version>
<jackson.version>1.9.13</jackson.version>
<jcommander.version>1.30</jcommander.version>
@@ -717,7 +718,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpcore</artifactId>
- <version>${httpclient4.version}</version>
+ <version>${httpcore.version}</version>
</dependency>
<!-- ======================================================== -->
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34f559a6/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 17728c3..6e942c0 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -335,11 +335,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpcore</artifactId>
+ <scope>test</scope>
</dependency>
<dependency>
[17/31] incubator-slider git commit: SLIDER-1130 Hadoop 2.8 YARN
changes have broken slider. See also YARN-5135: ContainerRequest labels
ignored -broken constructor
Posted by el...@apache.org.
SLIDER-1130 Hadoop 2.8 YARN changes have broken slider. See also YARN-5135: ContainerRequest labels ignored -broken constructor
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/80fe8c7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/80fe8c7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/80fe8c7e
Branch: refs/heads/branches/branch-0.91
Commit: 80fe8c7e04a933885ab0f623216940861b233b5f
Parents: 9937bbc
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 17:59:58 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue May 24 17:59:58 2016 +0100
----------------------------------------------------------------------
.../appmaster/state/OutstandingRequest.java | 2 ++
.../appstate/TestMockAppStateAAPlacement.groovy | 6 ++---
.../appstate/TestMockLabelledAAPlacement.groovy | 4 +--
.../TestOutstandingRequestValidation.groovy | 26 --------------------
...tRoleHistoryOutstandingRequestTracker.groovy | 5 +++-
.../appmaster/model/mock/MockYarnCluster.groovy | 2 +-
6 files changed, 12 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
index e0b01a1..c6524be 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/OutstandingRequest.java
@@ -283,6 +283,7 @@ public final class OutstandingRequest extends RoleHostnamePair {
*/
public synchronized AMRMClient.ContainerRequest escalate() {
Preconditions.checkNotNull(issuedRequest, "cannot escalate if request not issued " + this);
+ log.debug("Escalating {}", this.toString());
escalated = true;
// this is now the priority
@@ -352,6 +353,7 @@ public final class OutstandingRequest extends RoleHostnamePair {
}
sb.append(", node=").append(node);
sb.append(", hasLocation=").append(requestHasLocation);
+ sb.append(", label=").append(label);
sb.append(", requestedTimeMillis=").append(requestedTimeMillis);
sb.append(", mayEscalate=").append(mayEscalate);
sb.append(", escalated=").append(escalated);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
index 85c09ff..125b1f4 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateAAPlacement.groovy
@@ -57,7 +57,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
/**
- * This is the simplest AA allocation: no lables, so allocate anywhere
+ * This is the simplest AA allocation: no labels, so allocate anywhere
* @throws Throwable
*/
@Test
@@ -218,7 +218,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
// flex down so that the next request should be cancelled
aaRole.desired = 0
- // expect: no new reqests, pending count --
+ // expect: no new requests, pending count --
List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
assert aaRole.pendingAntiAffineRequests == 0
assert !aaRole.AARequestOutstanding
@@ -281,7 +281,7 @@ class TestMockAppStateAAPlacement extends BaseMockAppStateAATest
}
protected AppState.NodeUpdatedOutcome addNewNode() {
- updateNodes(MockFactory.newNodeReport("4", NodeState.RUNNING, "gpu"))
+ updateNodes(MockFactory.instance.newNodeReport("4", NodeState.RUNNING, "gpu"))
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
index b2e2a74..e159dbb 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockLabelledAAPlacement.groovy
@@ -48,8 +48,8 @@ class TestMockLabelledAAPlacement extends BaseMockAppStateAATest
super.setup()
// node 1 is GPU
- updateNodes(MockFactory.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
- updateNodes(MockFactory.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST0, NodeState.RUNNING, LABEL_GPU))
+ updateNodes(MockFactory.instance.newNodeReport(HOST1, NodeState.RUNNING, LABEL_GPU))
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
index 9202b5c..fb9d976 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestOutstandingRequestValidation.groovy
@@ -55,21 +55,6 @@ class TestOutstandingRequestValidation extends SliderTestBase {
createAndValidate(H1, "", true)
}
- @Test
- public void testLabelAndHostRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", true)
- }
-
- @Test
- public void testLabelAndHostNonRelaxed() throws Throwable {
- expectValidationFailure(H1, "gpu", false)
- }
-
- @Test
- public void testComplexLabelExpression() throws Throwable {
- expectValidationFailure(null, "gpu||ps4", true)
- }
-
/**
* Use varargs for simple list to array conversion
* @param hostnames host names
@@ -79,17 +64,6 @@ class TestOutstandingRequestValidation extends SliderTestBase {
hostnames
}
- void expectValidationFailure(
- String[] hosts,
- String labels,
- boolean relaxLocality) {
- try {
- def result = createAndValidate(hosts, labels, relaxLocality)
- fail("Expected an exception, got $result")
- } catch (InvalidContainerRequestException expected) {
- }
- }
-
void expectCreationFailure(
String[] hosts,
String labels,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
index 6969b38..0969824 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryOutstandingRequestTracker.groovy
@@ -238,12 +238,15 @@ class TestRoleHistoryOutstandingRequestTracker extends BaseMockAppStateTest {
def workerRole = lookupRole(WORKER.name)
// initial request
def yarnRequest = req1.buildContainerRequest(resource, workerRole, 0)
+ assert (req1.label == WORKERS_LABEL)
+
assert (yarnRequest.nodeLabelExpression == null)
assert (!yarnRequest.relaxLocality)
// escalation
def yarnRequest2 = req1.escalate()
- assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
+ assert yarnRequest2.nodes == null
assert (yarnRequest2.relaxLocality)
+ assert (yarnRequest2.nodeLabelExpression == WORKERS_LABEL)
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/80fe8c7e/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
index e1dfa25..f0a1494 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnCluster.groovy
@@ -182,7 +182,7 @@ public class MockYarnCluster {
containers[i] = new MockYarnClusterContainer(mci)
}
- nodeReport = MockFactory.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
+ nodeReport = MockFactory.instance.newNodeReport(hostname, nodeId, NodeState.RUNNING, "")
}
/**
[11/31] incubator-slider git commit: SLIDER-1128 Update command
logger test resources
Posted by el...@apache.org.
SLIDER-1128 Update command logger test resources
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0c1f70af
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0c1f70af
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0c1f70af
Branch: refs/heads/branches/branch-0.91
Commit: 0c1f70af978a90732ec59985f69c73d652987f85
Parents: f3334cc
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Fri May 20 05:57:27 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Fri May 20 05:57:27 2016 -0700
----------------------------------------------------------------------
.../test/app_packages/test_command_log/resources_add_on_pkg.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0c1f70af/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
index 5255f0e..3f71eb3 100644
--- a/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
+++ b/slider-core/src/test/app_packages/test_command_log/resources_add_on_pkg.json
@@ -6,7 +6,7 @@
},
"components": {
"COMMAND_LOGGER": {
- "yarn.memory": "128",
+ "yarn.memory": "256",
"yarn.role.priority": "1",
"yarn.component.instances": "1"
},
[02/31] incubator-slider git commit: SLIDER-906 Support for Docker
based application packaging with first class YARN support - Phase 2
[contributed by Thomas (Yu) Liu, Billie Rinaldi and Gour Saha]
Posted by el...@apache.org.
SLIDER-906 Support for Docker based application packaging with first class YARN support - Phase 2 [contributed by Thomas (Yu) Liu, Billie Rinaldi and Gour Saha]
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1ab0cbbd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1ab0cbbd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1ab0cbbd
Branch: refs/heads/branches/branch-0.91
Commit: 1ab0cbbd87b92d59d087a8c92a170d0be748b4b3
Parents: 4b4e24f
Author: Gour Saha <go...@apache.org>
Authored: Tue Mar 22 18:00:26 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Wed Apr 13 00:16:32 2016 -0700
----------------------------------------------------------------------
.../src/main/python/agent/ActionQueue.py | 15 +-
.../src/main/python/agent/CommandStatusDict.py | 2 +
.../src/main/python/agent/Controller.py | 2 +-
.../src/main/python/agent/YarnDockerManager.py | 248 +++++++++++++++++
slider-agent/src/main/python/agent/main.py | 2 +-
.../core/providers/windows/system.py | 1 +
.../java/org/apache/slider/api/ClusterNode.java | 2 +
.../org/apache/slider/api/ResourceKeys.java | 14 +
.../org/apache/slider/client/SliderClient.java | 10 +
.../apache/slider/common/tools/SliderUtils.java | 22 +-
.../slider/core/launch/AbstractLauncher.java | 31 ++-
.../slider/providers/agent/AgentKeys.java | 1 +
.../providers/agent/AgentProviderService.java | 264 ++++++++++++++-----
.../providers/agent/ComponentInstanceState.java | 20 ++
.../agent/application/metadata/Application.java | 2 +-
.../agent/application/metadata/Component.java | 3 +
.../application/metadata/DockerContainer.java | 65 ++++-
.../server/appmaster/SliderAppMaster.java | 1 +
.../slider/server/appmaster/state/AppState.java | 1 +
.../server/appmaster/state/RoleInstance.java | 4 +
.../web/rest/agent/ComponentStatus.java | 20 ++
.../web/rest/agent/ExecutionCommand.java | 21 +-
.../appmaster/web/rest/agent/HeartBeat.java | 4 +-
.../web/rest/agent/HeartBeatResponse.java | 1 +
.../web/rest/agent/RegistrationResponse.java | 20 +-
.../appmaster/web/rest/agent/StatusCommand.java | 11 +
.../allCompFailed/appConfig.json | 10 +
.../allCompFailed/metainfo.json | 44 ++++
.../allCompFailed/resources.json | 20 ++
.../basic/appConfig.json | 10 +
.../test_docker_on_yarn_pkg/basic/metainfo.json | 26 ++
.../basic/resources.json | 14 +
.../multiComp/appConfig.json | 10 +
.../multiComp/metainfo.json | 44 ++++
.../multiComp/resources.json | 20 ++
.../multiCompMultiContainer/appConfig.json | 10 +
.../multiCompMultiContainer/metainfo.json | 44 ++++
.../multiCompMultiContainer/resources.json | 20 ++
.../oneCompFailed/appConfig.json | 10 +
.../oneCompFailed/metainfo.json | 44 ++++
.../oneCompFailed/resources.json | 20 ++
.../agent/TestAgentProviderService.java | 10 +-
.../DockerAppLaunchedOnYarnIT.groovy | 259 ++++++++++++++++++
43 files changed, 1321 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/agent/ActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/ActionQueue.py b/slider-agent/src/main/python/agent/ActionQueue.py
index 497d4f4..107d6c3 100644
--- a/slider-agent/src/main/python/agent/ActionQueue.py
+++ b/slider-agent/src/main/python/agent/ActionQueue.py
@@ -31,6 +31,7 @@ from AgentToggleLogger import AgentToggleLogger
from CommandStatusDict import CommandStatusDict
from CustomServiceOrchestrator import CustomServiceOrchestrator
from DockerManager import DockerManager
+from YarnDockerManager import YarnDockerManager
import Constants
@@ -54,6 +55,7 @@ class ActionQueue(threading.Thread):
AUTO_RESTART = 'auto_restart'
docker_mode = False
+ yarn_docker_mode = False
def __init__(self, config, controller, agentToggleLogger):
super(ActionQueue, self).__init__()
@@ -71,6 +73,7 @@ class ActionQueue(threading.Thread):
controller,
self.queueOutAgentToggleLogger)
self.dockerManager = DockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
+ self.yarnDockerManager = YarnDockerManager(self.tmpdir, config.getWorkRootPath(), self.customServiceOrchestrator)
def stop(self):
@@ -169,7 +172,10 @@ class ActionQueue(threading.Thread):
logger.info("Running command: " + str(command))
- if 'configurations' in command and 'docker' in command['configurations']:
+ if 'configurations' in command and 'yarnDockerMode' in command and command['yarnDockerMode'] == True:
+ self.yarn_docker_mode = True
+ commandresult = self.yarnDockerManager.execute_command(command, store_config or store_command)
+ elif 'configurations' in command and 'docker' in command['configurations']:
self.docker_mode = True
commandresult = self.dockerManager.execute_command(command, store_config or store_command)
else:
@@ -235,6 +241,8 @@ class ActionQueue(threading.Thread):
component_status = None
if self.docker_mode:
component_status = self.dockerManager.query_status(command)
+ elif self.yarn_docker_mode:
+ component_status = self.yarnDockerManager.query_status(command)
else:
component_status = self.customServiceOrchestrator.requestComponentStatus(command)
@@ -254,8 +262,13 @@ class ActionQueue(threading.Thread):
" of service " + str(service) + \
" of cluster " + str(cluster))
logger.debug(pprint.pformat(result))
+ if 'ip' in component_status:
+ result['ip'] = component_status['ip']
+ if 'hostname' in component_status:
+ result['hostname'] = component_status['hostname']
if result is not None:
+ logger.debug("result of execute_status_command: " + str(result))
self.commandStatuses.put_command_status(command, result, reportResult)
except Exception, err:
traceback.print_exc()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/agent/CommandStatusDict.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CommandStatusDict.py b/slider-agent/src/main/python/agent/CommandStatusDict.py
index bcbce9e..5af75fb 100644
--- a/slider-agent/src/main/python/agent/CommandStatusDict.py
+++ b/slider-agent/src/main/python/agent/CommandStatusDict.py
@@ -73,6 +73,7 @@ class CommandStatusDict():
for key, item in self.current_state.items():
command = item[0]
report = item[1]
+ logger.debug("report inside generate report is: " + str(report))
if command ['commandType'] == ActionQueue.EXECUTION_COMMAND:
if (report['status']) != ActionQueue.IN_PROGRESS_STATUS:
resultReports.append(report)
@@ -90,6 +91,7 @@ class CommandStatusDict():
'reports': resultReports,
'componentStatus': resultComponentStatus
}
+ logger.debug("result from generate report: " + str(result))
return result
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/agent/Controller.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/Controller.py b/slider-agent/src/main/python/agent/Controller.py
index ec3bed7..8c01315 100644
--- a/slider-agent/src/main/python/agent/Controller.py
+++ b/slider-agent/src/main/python/agent/Controller.py
@@ -165,7 +165,7 @@ class Controller(threading.Thread):
print("Registered with the server")
self.responseId = int(regResp['responseId'])
self.isRegistered = True
- if 'statusCommands' in regResp.keys():
+ if ('statusCommands' in regResp.keys()) and (not regResp['statusCommands'] == None):
logger.info("Got status commands on registration " + pprint.pformat(
regResp['statusCommands']))
self.addToQueue(regResp['statusCommands'])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/agent/YarnDockerManager.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/YarnDockerManager.py b/slider-agent/src/main/python/agent/YarnDockerManager.py
new file mode 100644
index 0000000..abdb2fb
--- /dev/null
+++ b/slider-agent/src/main/python/agent/YarnDockerManager.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import logging
+import os
+import subprocess
+import Constants
+import time
+import traceback
+from resource_management import *
+
+logger = logging.getLogger()
+
+class YarnDockerManager(Script):
+ stored_status_command = ''
+ stored_command = ''
+ container_id = ''
+
+ def __init__(self, tmpdir, workroot, customServiceOrchestrator):
+ self.tmpdir = tmpdir
+ self.workroot = workroot
+ self.customServiceOrchestrator = customServiceOrchestrator
+
+ def execute_command(self, command, store_command=False):
+ returncode = ''
+ out = ''
+ err = ''
+
+ if store_command:
+ logger.info("Storing applied config: " + str(command['configurations']))
+ self.stored_command = command
+ status_command_str = self.extract_config_from_command(command, 'docker.status_command')
+ if status_command_str:
+ self.stored_status_command = status_command_str
+ logger.info("status command" + self.stored_status_command)
+ if 'hostLevelParams' in command:
+ if 'container_id' in command['hostLevelParams']:
+ self.container_id = command['hostLevelParams']['container_id']
+
+ if command['roleCommand'] == 'INSTALL':
+ with Environment(self.workroot) as env:
+ self.install_container(command, env)
+ returncode = 0; out = ''; err = ''
+ if command['roleCommand'] == 'START':
+ returncode, out, err = self.start_container(command)
+ # need check
+ return {Constants.EXIT_CODE:returncode, 'stdout':out, 'stderr':err}
+
+
+ def extract_config_files_from_command(self, command):
+ if 'containers' in command and len(command['containers']) > 0:
+ if 'configFiles' in command['containers'][0]:
+ return command['containers'][0]['configFiles']
+ return []
+
+ def extract_config_file_properties_from_command(self, command, file):
+ if 'configurations' in command:
+ if 'dictionaryName' in file and file['dictionaryName'] in command['configurations']:
+ properties = {}
+ for k,v in command['configurations'][file['dictionaryName']].iteritems():
+ properties[k] = format(v, **os.environ)
+ return properties
+ return {}
+
+ def extract_config_from_command(self, command, field):
+ value = ''
+ if 'configurations' in command:
+ if 'docker' in command['configurations']:
+ if field in command['configurations']['docker']:
+ logger.info(field + ': ' + str( command['configurations']['docker'][field]))
+ value = command['configurations']['docker'][field]
+ return value
+
+
+ # will evolve into a class hierarch, linux and windows
+ def execute_command_on_linux(self, docker_command, blocking, stdoutFile=None, stderrFile=None):
+
+ logger.info("command str: " + docker_command)
+ logger.info("command env: " + str(os.environ))
+ if stdoutFile != None or stderrFile != None:
+ proc = subprocess.Popen(docker_command,
+ stdout = stdoutFile,
+ stderr = stderrFile, universal_newlines = True, shell=True)
+ else:
+ proc = subprocess.Popen(docker_command, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell=True)
+
+ returncode = 0
+ out = ''
+ err = ''
+ if blocking is True:
+ out, err = proc.communicate()
+ returncode = proc.returncode
+ else:
+ time.sleep(5)
+ if proc.returncode is not None:
+ # this means the command has already returned
+ returncode = proc.returncode
+ out, err = proc.communicate()
+ logger.info("returncode: " + str(returncode) + " out: " + str(out) + " err: " + str(err))
+ return returncode, out, err
+
+ def enqueue_output(out, queue):
+ for line in iter(out.readline, b''):
+ queue.put(line)
+ out.close()
+
+ def get_config_file_global(self, command, file, property, useEnv=True):
+ keyName = file['dictionaryName'] + '.' + property
+ if keyName in command['configurations']['global']:
+ if useEnv:
+ return format(command['configurations']['global'][keyName], **os.environ)
+ else:
+ return command['configurations']['global'][keyName]
+ return None
+
+ def install_container(self, command, env):
+ try:
+ configFiles = self.extract_config_files_from_command(command)
+ for configFile in configFiles:
+ properties = self.extract_config_file_properties_from_command(command, configFile)
+ fileName = configFile['fileName']
+ dir = self.get_config_file_global(command, configFile, 'destDir')
+ if dir is None:
+ dir = self.workroot
+ logger.info("creating config file " + str(configFile) + " in directory "+str(dir))
+ Directory(dir, recursive = True)
+ if configFile['type'] == 'properties':
+ PropertiesFile(fileName,
+ dir=dir,
+ properties=properties)
+ elif configFile['type'] == 'env':
+ content = self.get_config_file_global(command, configFile, 'content', useEnv=False)
+ if content is not None:
+ File(os.path.join(dir, fileName),
+ content=InlineTemplate(content, **properties))
+ elif configFile['type'] == 'template':
+ templateFile = self.get_config_file_global(command, configFile, 'templateFile')
+ if templateFile is not None:
+ with open(templateFile,"r") as fp:
+ fileContent = fp.read()
+ File(os.path.join(dir, fileName),
+ content=InlineTemplate(fileContent, **properties))
+ elif configFile['type'] == 'xml':
+ XmlConfig(fileName,
+ conf_dir=dir,
+ configurations=properties)
+ except:
+ traceback.print_exc()
+
+ def start_container(self, command):
+ #extracting param needed by docker run from the command passed from AM
+ startCommand = self.extract_config_from_command(command, 'docker.startCommand')
+ #adding redirecting stdout stderr to file
+ outfilename = Constants.APPLICATION_STD_OUTPUT_LOG_FILE_PREFIX + \
+ self.container_id + Constants.APPLICATION_STD_OUTPUT_LOG_FILE_FILE_TYPE
+
+ errfilename = Constants.APPLICATION_STD_ERROR_LOG_FILE_PREFIX + \
+ self.container_id + Constants.APPLICATION_STD_ERROR_LOG_FILE_FILE_TYPE
+
+ stdoutFile = open(outfilename, 'w')
+ stderrFile = open(errfilename, 'w')
+ returncode,out,err = self.execute_command_on_linux(startCommand, False,
+ stdoutFile, stderrFile)
+ return returncode,out,err
+
+ def query_status(self, command):
+ if command['roleCommand'] == "GET_CONFIG":
+ return self.getConfig(command)
+ else:
+ returncode = ''
+ out = ''
+ err = ''
+ status_command_str = self.extract_config_from_command(command, 'docker.status_command')
+ if status_command_str:
+ self.stored_status_command = status_command_str
+ logger.info("in query_status, got stored status command" + self.stored_status_command)
+ if self.stored_status_command:
+ logger.info("stored status command to run: " + self.stored_status_command)
+ returncode, out, err = self.execute_command_on_linux(self.stored_status_command, True)
+ logger.info("status of the app in docker container: " + str(returncode) + ";" + str(out) + ";" + str(err))
+
+ return {Constants.EXIT_CODE:returncode, 'stdout':out, 'stderr':err}
+
+ def getConfig(self, command):
+ logger.info("get config command: " + str(command))
+ config = {}
+
+ if 'configurations' in self.stored_command:
+ if 'commandParams' in command and 'config_type' in command['commandParams']:
+ config_type = command['commandParams']['config_type']
+ logger.info("Requesting applied config for type {0}".format(config_type))
+ if config_type in self.stored_command['configurations']:
+ logger.info("get config result: " + self.stored_command['configurations'][config_type])
+ config = {
+ 'configurations': {config_type: self.stored_command['configurations'][config_type]}
+ }
+ else:
+ config = {
+ 'configurations': {}
+ }
+ pass
+ else:
+ logger.info("Requesting all applied config." + str(self.stored_command['configurations']))
+ config = {
+ 'configurations': self.stored_command['configurations']
+ }
+ pass
+ else:
+ config = {
+ 'configurations': {}
+ }
+
+ #query the ip and hostname of the docker container where the agent is running
+ ip_query_command = "ip addr show dev eth0 | grep \'inet \' | awk \'{print $2}\' | cut -d / -f 1"
+ proc = subprocess.Popen(ip_query_command, stdout = subprocess.PIPE, shell=True)
+ ip, err = proc.communicate()
+ if err is not None:
+ logger.error("error when retrieving ip: " + err)
+
+ hostname_query_command = "hostname"
+ proc = subprocess.Popen(hostname_query_command, stdout = subprocess.PIPE, shell=True)
+ hostname, err = proc.communicate()
+ if err is not None:
+ logger.error("error when retrieving hostname: " + err)
+
+ config['ip'] = ip.rstrip()
+ config['hostname'] = hostname.rstrip()
+
+ logger.info('response from getconfig: ' + str(config))
+ return config
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index 36aa4f4..39de1ae 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -55,7 +55,7 @@ def signal_handler(signum, frame):
os._exit(0)
logger.info('signal received, exiting.')
global controller
- if controller is not None and hasattr(controller, 'actionQueue'):
+ if controller is not None and hasattr(controller, 'actionQueue') and controller.actionQueue.yarn_docker_mode == False:
docker_mode = controller.actionQueue.docker_mode
if docker_mode:
tmpdir = controller.actionQueue.dockerManager.stop_container()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index af513a1..b85c020 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -89,6 +89,7 @@ def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=
if not wait_for_finish:
Logger.debug("No need to wait for the process to exit. Will leave the process running ...")
code = 0
+ logAnyway = False
if pid_file_name:
Logger.debug("Writing the process id %s to file %s" % (str(proc.pid), pid_file_name))
pidfile = open(pid_file_name, 'w')
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
index e05a2ef..8b0a563 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ClusterNode.java
@@ -66,6 +66,8 @@ public final class ClusterNode implements Cloneable {
*/
public boolean released;
public String host;
+ public String ip;
+ public String hostname;
public String hostUrl;
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
index f92a58d..aba544b 100644
--- a/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/ResourceKeys.java
@@ -66,6 +66,18 @@ public interface ResourceKeys {
*/
String YARN_CORES = "yarn.vcores";
+ /**
+ * Number of disks per instance to ask YARN for
+ * {@value}
+ */
+ String YARN_DISKS = "yarn.disks.count-per-instance";
+
+ /**
+ * Disk size per disk to ask YARN for
+ * {@value}
+ */
+ String YARN_DISK_SIZE = "yarn.disk.size";
+
/** {@value} */
int DEF_YARN_CORES = 1;
@@ -168,6 +180,8 @@ public interface ResourceKeys {
String YARN_LOG_INCLUDE_PATTERNS = "yarn.log.include.patterns";
String YARN_LOG_EXCLUDE_PATTERNS = "yarn.log.exclude.patterns";
+ String YARN_PROFILE_NAME = "yarn.resource-profile-name";
+
/**
* Window of time where application master's failure count
* can be reset to 0.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 2e5c45a..dd90e46 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -177,6 +177,7 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
+import java.io.FilenameFilter;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -1760,6 +1761,15 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
conf.global.putAll(newglobal);
+ for (String component : conf.components.keySet()) {
+ Map<String,String> newComponent = new HashMap<>();
+ for (Entry<String,String> entry : conf.components.get(component).entrySet()) {
+ newComponent.put(entry.getKey(), replaceTokens(entry.getValue(),
+ userName, clusterName));
+ }
+ conf.components.get(component).putAll(newComponent);
+ }
+
Map<String,List<String>> newcred = new HashMap<>();
for (Entry<String,List<String>> entry : conf.credentials.entrySet()) {
List<String> resultList = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index b9c7b8e..eae80f5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -117,6 +117,7 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.regex.Pattern;
import java.util.zip.GZIPOutputStream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -153,9 +154,14 @@ public final class SliderUtils {
public static final String PYTHON = "python";
/**
- * name of docker program
+ * type of docker standalone application
*/
public static final String DOCKER = "docker";
+ /**
+ * type of docker on yarn application
+ */
+ public static final String DOCKER_YARN = "yarn_docker";
+
public static final int NODE_LIST_LIMIT = 10;
private SliderUtils() {
@@ -1132,6 +1138,11 @@ public final class SliderUtils {
* @return a possibly empty map of environment variables.
*/
public static Map<String, String> buildEnvMap(Map<String, String> roleOpts) {
+ return buildEnvMap(roleOpts, null);
+ }
+
+ public static Map<String, String> buildEnvMap(Map<String, String> roleOpts,
+ Map<String,String> tokenMap) {
Map<String, String> env = new HashMap<>();
if (roleOpts != null) {
for (Map.Entry<String, String> entry : roleOpts.entrySet()) {
@@ -1139,7 +1150,14 @@ public final class SliderUtils {
if (key.startsWith(RoleKeys.ENV_PREFIX)) {
String envName = key.substring(RoleKeys.ENV_PREFIX.length());
if (!envName.isEmpty()) {
- env.put(envName, entry.getValue());
+ String value = entry.getValue();
+ if (tokenMap != null) {
+ for (Map.Entry<String,String> token : tokenMap.entrySet()) {
+ value = value.replaceAll(Pattern.quote(token.getKey()),
+ token.getValue());
+ }
+ }
+ env.put(envName, value);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
index 2ec3cda..0348828 100644
--- a/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
+++ b/slider-core/src/main/java/org/apache/slider/core/launch/AbstractLauncher.java
@@ -75,6 +75,11 @@ public abstract class AbstractLauncher extends Configured {
// security
protected final Credentials credentials;
protected LogAggregationContext logAggregationContext;
+ protected boolean yarnDockerMode = false;
+ protected String dockerImage;
+ protected String yarnContainerMountPoints;
+ protected String runPrivilegedContainer;
+
/**
* Create instance.
@@ -97,6 +102,10 @@ public abstract class AbstractLauncher extends Configured {
public ContainerLaunchContext getContainerLaunchContext() {
return containerLaunchContext;
}
+
+ public void setYarnDockerMode(boolean yarnDockerMode){
+ this.yarnDockerMode = yarnDockerMode;
+ }
/**
* Get the env vars to work on
@@ -181,7 +190,6 @@ public abstract class AbstractLauncher extends Configured {
*/
public ContainerLaunchContext completeContainerLaunch() throws IOException {
-
String cmdStr = SliderUtils.join(commands, " ", false);
log.debug("Completed setting up container command {}", cmdStr);
containerLaunchContext.setCommands(commands);
@@ -192,7 +200,7 @@ public abstract class AbstractLauncher extends Configured {
for (Map.Entry<String, String> envPair : envVars.entrySet()) {
log.debug(" \"{}\"=\"{}\"", envPair.getKey(), envPair.getValue());
}
- }
+ }
containerLaunchContext.setEnvironment(env);
//service data
@@ -214,6 +222,14 @@ public abstract class AbstractLauncher extends Configured {
containerLaunchContext.setTokens(CredentialUtils.marshallCredentials(
credentials));
+ if(yarnDockerMode){
+ Map<String, String> env = containerLaunchContext.getEnvironment();
+ env.put("YARN_CONTAINER_RUNTIME_TYPE", "docker");
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_IMAGE", dockerImage);//if yarnDockerMode, then dockerImage is set
+ env.put("YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER", runPrivilegedContainer);
+ log.info("yarn docker env var has been set {}", containerLaunchContext.getEnvironment().toString());
+ }
+
return containerLaunchContext;
}
@@ -480,5 +496,16 @@ public abstract class AbstractLauncher extends Configured {
return null;
}
+ public void setDockerImage(String dockerImage) {
+ this.dockerImage = dockerImage;
+ }
+
+ public void setYarnContainerMountPoints(String yarnContainerMountPoints) {
+ this.yarnContainerMountPoints = yarnContainerMountPoints;
+ }
+
+ public void setRunPrivilegedContainer(String runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 5ea95df..b027939 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -64,6 +64,7 @@ public interface AgentKeys {
String ARG_ZOOKEEPER_REGISTRY_PATH = "--zk-reg-path";
String ARG_DEBUG = "--debug";
String AGENT_MAIN_SCRIPT_ROOT = "./infra/agent/slider-agent/";
+ String AGENT_JINJA2_ROOT = "./infra/agent/slider-agent/jinja2";
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 0a1a98f..9d12f1d 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -28,9 +28,12 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.registry.client.binding.RegistryPathUtils;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.registry.client.types.yarn.PersistencePolicies;
+import org.apache.hadoop.registry.client.types.yarn.YarnRegistryAttributes;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
@@ -338,7 +341,7 @@ public class AgentProviderService extends AbstractProviderService implements
Path containerTmpDirPath) throws
IOException,
SliderException {
-
+
String appDef = SliderUtils.getApplicationDefinitionPath(instanceDefinition
.getAppConfOperations());
@@ -346,9 +349,20 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Build launch context for Agent");
log.debug(instanceDefinition.toString());
+
+ //if we are launching docker based app on yarn, then we need to pass docker image
+ if (isYarnDockerContainer(role)) {
+ launcher.setYarnDockerMode(true);
+ launcher.setDockerImage(getConfigFromMetaInfo(role, "image"));
+ launcher.setRunPrivilegedContainer(getConfigFromMetaInfo(role, "runPriviledgedContainer"));
+ launcher
+ .setYarnContainerMountPoints(getConfigFromMetaInfoWithAppConfigOverriding(
+ role, "yarn.container.mount.points"));
+ }
// Set the environment
- launcher.putEnv(SliderUtils.buildEnvMap(appComponent));
+ launcher.putEnv(SliderUtils.buildEnvMap(appComponent,
+ getStandardTokenMap(getAmState().getAppConfSnapshot(), role)));
String workDir = ApplicationConstants.Environment.PWD.$();
launcher.setEnv("AGENT_WORK_ROOT", workDir);
@@ -379,6 +393,7 @@ public class AgentProviderService extends AbstractProviderService implements
// set PYTHONPATH
List<String> pythonPaths = new ArrayList<String>();
pythonPaths.add(AgentKeys.AGENT_MAIN_SCRIPT_ROOT);
+ pythonPaths.add(AgentKeys.AGENT_JINJA2_ROOT);
String pythonPath = StringUtils.join(File.pathSeparator, pythonPaths);
launcher.setEnv(PYTHONPATH, pythonPath);
log.info("PYTHONPATH set to {}", pythonPath);
@@ -836,7 +851,7 @@ public class AgentProviderService extends AbstractProviderService implements
response.setResponseId(id + 1L);
String label = heartBeat.getHostname();
- String pkg = heartBeat.getPkg();
+ String pkg = heartBeat.getPackage();
log.debug("package received: " + pkg);
@@ -851,7 +866,7 @@ public class AgentProviderService extends AbstractProviderService implements
CommandScript cmdScript = getScriptPathForMasterPackage(roleName);
List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands();
- if (!isDockerContainer(roleName)
+ if (!isDockerContainer(roleName) && !isYarnDockerContainer(roleName)
&& (cmdScript == null || cmdScript.getScript() == null)
&& commands.size() == 0) {
log.error(
@@ -880,6 +895,51 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ List<ComponentStatus> statuses = heartBeat.getComponentStatus();
+ if (statuses != null && !statuses.isEmpty()) {
+ log.info("status from agent: " + statuses.toString());
+ try {
+ for(ComponentStatus status : statuses){
+ RoleInstance role = null;
+ if(status.getIp() != null && !status.getIp().isEmpty()){
+ role = amState.getOwnedContainer(containerId);
+ role.ip = status.getIp();
+ }
+ if(status.getHostname() != null & !status.getHostname().isEmpty()){
+ role = amState.getOwnedContainer(containerId);
+ role.hostname = status.getHostname();
+ }
+ if (role != null) {
+ // create an updated service record (including hostname and ip) and publish...
+ ServiceRecord record = new ServiceRecord();
+ record.set(YarnRegistryAttributes.YARN_ID, containerId);
+ record.description = roleName;
+ record.set(YarnRegistryAttributes.YARN_PERSISTENCE,
+ PersistencePolicies.CONTAINER);
+ // TODO: switch record attributes to use constants from YarnRegistryAttributes
+ // when it's been updated.
+ if (role.ip != null) {
+ record.set("yarn:ip", role.ip);
+ }
+ if (role.hostname != null) {
+ record.set("yarn:hostname", role.hostname);
+ }
+ yarnRegistry.putComponent(
+ RegistryPathUtils.encodeYarnID(containerId), record);
+
+ }
+ }
+
+
+ } catch (NoSuchNodeException e) {
+ // ignore - there is nothing to do if we don't find a container
+ log.warn("Owned container {} not found - {}", containerId, e);
+ } catch (IOException e) {
+ log.warn("Error updating container {} service record in registry",
+ containerId, e);
+ }
+ }
+
Boolean isMaster = isMaster(roleName);
ComponentInstanceState componentStatus = getComponentStatuses().get(label);
componentStatus.heartbeat(System.currentTimeMillis());
@@ -944,7 +1004,7 @@ public class AgentProviderService extends AbstractProviderService implements
componentStatus.getNextPkgToInstall(), command.toString());
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName)){
+ if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
addInstallDockerCommand(roleName, containerId, response, null, timeout);
} else if (scriptPath != null) {
addInstallCommand(roleName, containerId, response, scriptPath,
@@ -994,7 +1054,7 @@ public class AgentProviderService extends AbstractProviderService implements
boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName)){
+ if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
addStartDockerCommand(roleName, containerId, response, null, timeout, false);
} else if (scriptPath != null) {
addStartCommand(roleName,
@@ -1046,7 +1106,7 @@ public class AgentProviderService extends AbstractProviderService implements
&& command == Command.NOP) {
if (!componentStatus.getConfigReported()) {
log.info("Requesting applied config for {} on {}.", roleName, containerId);
- if (isDockerContainer(roleName)){
+ if (isDockerContainer(roleName) || isYarnDockerContainer(roleName)){
addGetConfigDockerCommand(roleName, containerId, response);
} else {
addGetConfigCommand(roleName, containerId, response);
@@ -1080,7 +1140,15 @@ public class AgentProviderService extends AbstractProviderService implements
private boolean isDockerContainer(String roleName) {
String type = getMetaInfo().getApplicationComponent(roleName).getType();
if (SliderUtils.isSet(type)) {
- return type.toLowerCase().equals(SliderUtils.DOCKER);
+ return type.toLowerCase().equals(SliderUtils.DOCKER) || type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
+ }
+ return false;
+ }
+
+ private boolean isYarnDockerContainer(String roleName) {
+ String type = getMetaInfo().getApplicationComponent(roleName).getType();
+ if (SliderUtils.isSet(type)) {
+ return type.toLowerCase().equals(SliderUtils.DOCKER_YARN);
}
return false;
}
@@ -2005,12 +2073,17 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.addContainerDetails(componentName, getMetaInfo());
Map<String, String> dockerConfig = new HashMap<String, String>();
- dockerConfig.put(
- "docker.command_path",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
- "commandPath"));
- dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(componentName, "image"));
+ if(isYarnDockerContainer(componentName)){
+ //put nothing
+ cmd.setYarnDockerMode(true);
+ } else {
+ dockerConfig.put(
+ "docker.command_path",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "commandPath"));
+ dockerConfig.put("docker.image_name",
+ getConfigFromMetaInfo(componentName, "image"));
+ }
configurations.put("docker", dockerConfig);
log.debug("Docker- command: {}", cmd.toString());
@@ -2122,7 +2195,7 @@ public class AgentProviderService extends AbstractProviderService implements
throws SliderException {
assert getAmState().isApplicationLive();
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- if (isDockerContainer(componentName)) {
+ if (isDockerContainer(componentName) || isYarnDockerContainer(componentName)) {
addStatusDockerCommand(componentName, containerId, response, scriptPath,
timeout);
return;
@@ -2182,9 +2255,14 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, String> dockerConfig = new HashMap<String, String>();
String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand");
if (statusCommand == null) {
- statusCommand = "docker top "
+ if(isYarnDockerContainer(componentName)){
+ //should complain the required field is null
+ cmd.setYarnDockerMode(true);
+ } else {
+ statusCommand = "docker top "
+ containerId
+ " | grep \"\"";// default value
+ }
}
dockerConfig.put("docker.status_command",statusCommand);
configurations.put("docker", dockerConfig);
@@ -2218,16 +2296,21 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, String> dockerConfig = new HashMap<String, String>();
String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(componentName, "statusCommand");
if (statusCommand == null) {
- statusCommand = "docker top "
+ if(isYarnDockerContainer(componentName)){
+ //should complain the required field is null
+ cmd.setYarnDockerMode(true);
+ } else {
+ statusCommand = "docker top "
+ containerId
+ " | grep \"\"";// default value
+ }
}
dockerConfig.put("docker.status_command",statusCommand);
configurations.put("docker", dockerConfig);
cmd.setConfigurations(configurations);
log.debug("Docker- getconfig command {}", cmd);
-
+
response.addStatusCommand(cmd);
}
@@ -2309,53 +2392,70 @@ public class AgentProviderService extends AbstractProviderService implements
log.debug("after resolution: " + appConf.toString());
Map<String, String> dockerConfig = new HashMap<String, String>();
- dockerConfig.put(
+ if (isYarnDockerContainer(componentName)) {
+ dockerConfig.put(
+ "docker.startCommand",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "start_command"));
+ cmd.setYarnDockerMode(true);
+ } else {
+ dockerConfig.put(
"docker.command_path",
getConfigFromMetaInfoWithAppConfigOverriding(componentName,
"commandPath"));
- dockerConfig.put("docker.image_name",
- getConfigFromMetaInfo(componentName, "image"));
- // options should always have -d
- String options = getConfigFromMetaInfoWithAppConfigOverriding(
- componentName, "options");
- if(options != null && !options.isEmpty()){
- options = options + " -d";
- } else {
- options = "-d";
+
+ dockerConfig.put("docker.image_name",
+ getConfigFromMetaInfo(componentName, "image"));
+ // options should always have -d
+ String options = getConfigFromMetaInfoWithAppConfigOverriding(
+ componentName, "options");
+ if(options != null && !options.isEmpty()){
+ options = options + " -d";
+ } else {
+ options = "-d";
+ }
+ dockerConfig.put("docker.options", options);
+ // options should always have -d
+ dockerConfig.put(
+ "docker.containerPort",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "containerPort"));
+ dockerConfig
+ .put(
+ "docker.hostPort",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "hostPort"));
+
+ dockerConfig.put(
+ "docker.mounting_directory",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "containerMount"));
+ dockerConfig
+ .put(
+ "docker.host_mounting_directory",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName,
+ "hostMount"));
+
+ dockerConfig.put("docker.additional_param",
+ getConfigFromMetaInfoWithAppConfigOverriding(componentName, "additionalParam"));
+
+ dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo(
+ componentName, "containerPath"));
}
- dockerConfig.put("docker.options", options);
- // options should always have -d
- dockerConfig.put(
- "docker.containerPort",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
- "containerPort"));
- dockerConfig
- .put(
- "docker.hostPort",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
- "hostPort"));
-
- dockerConfig.put(
- "docker.mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
- "containerMount"));
- dockerConfig
- .put(
- "docker.host_mounting_directory",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName,
- "hostMount"));
-
- dockerConfig.put("docker.additional_param",
- getConfigFromMetaInfoWithAppConfigOverriding(componentName, "additionalParam"));
-
- dockerConfig.put("docker.input_file.mount_path", getConfigFromMetaInfo(
- componentName, "containerPath"));
+
+ String lifetime = getConfigFromMetaInfoWithAppConfigOverriding(
+ componentName, "lifetime");
+ dockerConfig.put("docker.lifetime", lifetime);
configurations.put("docker", dockerConfig);
String statusCommand = getConfigFromMetaInfoWithAppConfigOverriding(
componentName, "statusCommand");
if (statusCommand == null) {
- statusCommand = "docker top "
- + containerId + " | grep \"\"";// default value
+ if(isYarnDockerContainer(componentName)){
+ //should complain the required field is null
+ } else {
+ statusCommand = "docker top "
+ + containerId + " | grep \"\"";
+ }
}
dockerConfig.put("docker.status_command",statusCommand);
@@ -2413,11 +2513,27 @@ public class AgentProviderService extends AbstractProviderService implements
log.debug("Docker- containers metainfo: {}", containers.toString());
if (containers.size() > 0) {
DockerContainer container = containers.get(0);
-
switch (configName) {
+ case "start_command":
+ result = container.getStartCommand();
+ break;
case "image":
result = container.getImage();
break;
+ case "network":
+ if (container.getNetwork() == null || container.getNetwork().isEmpty()) {
+ result = "none";
+ } else {
+ result = container.getNetwork();
+ }
+ break;
+ case "useNetworkScript":
+ if (container.getUseNetworkScript() == null || container.getUseNetworkScript().isEmpty()) {
+ result = "yes";
+ } else {
+ result = container.getUseNetworkScript();
+ }
+ break;
case "statusCommand":
result = container.getStatusCommand();
break;
@@ -2452,6 +2568,13 @@ public class AgentProviderService extends AbstractProviderService implements
case "additionalParam":
result = container.getAdditionalParam();// to support multi port later
break;
+ case "runPriviledgedContainer":
+ if (container.getRunPrivilegedContainer() == null) {
+ result = "false";
+ } else {
+ result = container.getRunPrivilegedContainer();
+ }
+ break;
default:
break;
}
@@ -2679,12 +2802,16 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, String> tokens = getStandardTokenMap(appConf, componentName);
Set<String> configs = new HashSet<String>();
- configs.addAll(getApplicationConfigurationTypes());
+ configs.addAll(getApplicationConfigurationTypes(componentName));
configs.addAll(getSystemConfigurationsRequested(appConf));
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
configurations, tokens, containerId, componentName);
+ if (appConf.getComponent(componentName) != null) {
+ addNamedConfiguration(configType, appConf.getComponent(componentName).options,
+ configurations, tokens, containerId, componentName);
+ }
}
//do a final replacement of re-used configs
@@ -2752,7 +2879,7 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
- protected List<String> getApplicationConfigurationTypes() {
+ protected List<String> getApplicationConfigurationTypes(String componentName) {
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
@@ -2761,6 +2888,23 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Expecting config type {}.", configFile.getDictionaryName());
configList.add(configFile.getDictionaryName());
}
+ for (Component component : getMetaInfo().getApplication().getComponents()) {
+ if (!component.getName().equals(componentName)) {
+ continue;
+ }
+ if (component.getDockerContainers() == null) {
+ continue;
+ }
+ for (DockerContainer container : component.getDockerContainers()) {
+ if (container.getConfigFiles() == null) {
+ continue;
+ }
+ for (ConfigFile configFile : container.getConfigFiles()) {
+ log.info("Expecting config type {}.", configFile.getDictionaryName());
+ configList.add(configFile.getDictionaryName());
+ }
+ }
+ }
// remove duplicates. mostly worried about 'global' being listed
return new ArrayList<String>(new HashSet<String>(configList));
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
index 55fdba6..6ee0ebb 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentInstanceState.java
@@ -46,6 +46,8 @@ public class ComponentInstanceState {
private int failuresSeen = 0;
private Boolean configReported = false;
private long lastHeartbeat = 0;
+ private String ip;
+ private String hostname;
private ContainerState containerState;
private Map<String, State> pkgStatuses;
@@ -314,7 +316,25 @@ public class ComponentInstanceState {
sb.append(", lastHeartbeat=").append(lastHeartbeat);
sb.append(", containerState=").append(containerState);
sb.append(", componentName='").append(componentName).append('\'');
+ sb.append(", ip=").append(ip);
+ sb.append(", hostname='").append(hostname).append('\'');
sb.append('}');
return sb.toString();
}
+
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index 3a445f4..63546a4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -130,7 +130,7 @@ public class Application extends AbstractMetainfoSchema {
sb.append(",\n\"version\" :").append(version);
sb.append(",\n\"components\" : {");
for (Component component : components) {
- sb.append("\n").append(component);
+ sb.append("\n").append(component.toString());
}
sb.append("\n},");
sb.append('}');
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 7099448..3f23455 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -162,6 +162,9 @@ public class Component extends AbstractComponent {
sb.append("\n\"name\": ").append(name);
sb.append(",\n\"category\": ").append(category);
sb.append(",\n\"commandScript\" :").append(commandScript);
+ for(DockerContainer dc : dockerContainers){
+ sb.append(",\n\"container\" :").append(dc.toString());
+ }
sb.append('}');
return sb.toString();
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
index ecd0166..4c61e7f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/DockerContainer.java
@@ -35,13 +35,18 @@ public class DockerContainer implements Validate {
private String name;
private String image;
+ private String network;
+ private String useNetworkScript;
private String options;
private List<DockerContainerMount> mounts = new ArrayList<>();
private List<DockerContainerPort> ports = new ArrayList<>();
private String statusCommand;
+ private String startCommand;
private String commandPath;
private String additionalParam;
+ private String runPrivilegedContainer;
private List<DockerContainerInputFile> inputFiles = new ArrayList<>();
+ private List<ConfigFile> configFiles = new ArrayList<>();
public DockerContainer() {
}
@@ -75,6 +80,22 @@ public class DockerContainer implements Validate {
this.image = image;
}
+ public String getNetwork() {
+ return network;
+ }
+
+ public void setNetwork(String network) {
+ this.network = network;
+ }
+
+ public String getUseNetworkScript() {
+ return useNetworkScript;
+ }
+
+ public void setUseNetworkScript(String useNetworkScript) {
+ this.useNetworkScript = useNetworkScript;
+ }
+
public String getOptions() {
return options;
}
@@ -95,10 +116,12 @@ public class DockerContainer implements Validate {
}
}
+ @JsonProperty("statusCommand")
public String getStatusCommand() {
return statusCommand;
}
+ @JsonProperty("statusCommand")
public void setStatusCommand(String statusCommand) {
this.statusCommand = statusCommand;
}
@@ -119,14 +142,46 @@ public class DockerContainer implements Validate {
this.additionalParam = additionalParam;
}
+ @JsonProperty("startCommand")
+ public String getStartCommand() {
+ return startCommand;
+ }
+
+ @JsonProperty("startCommand")
+ public void setStartCommand(String startCommand) {
+ this.startCommand = startCommand;
+ }
+
+ @JsonProperty("runPrivilegedContainer")
+ public String getRunPrivilegedContainer() {
+ return runPrivilegedContainer;
+ }
+
+ @JsonProperty("runPrivilegedContainer")
+ public void setRunPrivilegedContainer(String runPrivilegedContainer) {
+ this.runPrivilegedContainer = runPrivilegedContainer;
+ }
+
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
+ public void setConfigFiles(List<ConfigFile> configFiles) {
+ this.configFiles = configFiles;
+ }
+
@Override
public String toString() {
StringBuilder result = new StringBuilder("DockerContainer [name=")
- .append(name).append(", image=").append(image).append(", options=")
- .append(options).append(", mounts=").append(mounts).append(", ports=")
- .append(ports).append(", statusCommand=").append(statusCommand)
- .append(", commandPath=").append(commandPath).append(", additionalParam=")
- .append(additionalParam).append(", inputFiles=").append(inputFiles).append("]");
+ .append(name).append(", image=").append(image).append(", options=")
+ .append(options).append(", mounts=").append(mounts).append(", ports=")
+ .append(ports).append(", statusCommand=").append(statusCommand)
+ .append(", commandPath=").append(commandPath)
+ .append(", additionalParam=").append(additionalParam)
+ .append(", inputFiles=").append(inputFiles).append(", startCommand=")
+ .append(startCommand).append(", runPriviledgedContainer=")
+ .append(runPrivilegedContainer).append(", configFiles=")
+ .append(configFiles).append("]");
return result.toString();
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index d301f13..73f24c7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -2229,6 +2229,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
ContainerLaunchContext ctx,
RoleInstance instance) throws IOException {
appState.containerStartSubmitted(container, instance);
+
nmClientAsync.startContainerAsync(container, ctx);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index c8369c1..a11ce56 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1703,6 +1703,7 @@ public class AppState {
//build the map of node -> containers
Map<String, Map<String, ClusterNode>> clusterNodes =
createRoleToClusterNodeMap();
+ log.info("app state clusterNodes {} ", clusterNodes.toString());
cd.status = new HashMap<>();
cd.status.put(ClusterDescriptionKeys.KEY_CLUSTER_LIVE, clusterNodes);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
index 30a2bb0..345c67e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleInstance.java
@@ -98,6 +98,8 @@ public final class RoleInstance implements Cloneable {
*/
public String[] environment;
+ public String ip;
+ public String hostname;
public String host;
public String hostURL;
public ContainerAllocationOutcome placement;
@@ -231,6 +233,8 @@ public final class RoleInstance implements Cloneable {
node.environment = Arrays.copyOf(environment, environment.length);
}
node.exitCode = exitCode;
+ node.ip = ip;
+ node.hostname = hostname;
node.host = host;
node.hostUrl = hostURL;
if (output != null) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
index 63bdd0c..acdc234 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ComponentStatus.java
@@ -34,6 +34,8 @@ public class ComponentStatus {
String serviceName;
String clusterName;
String roleCommand;
+ String ip;
+ String hostname;
@JsonProperty("configurations")
private Map<String, Map<String, String>> configurations;
@@ -104,6 +106,24 @@ public class ComponentStatus {
", serviceName='" + serviceName + '\'' +
", clusterName='" + clusterName + '\'' +
", roleCommand='" + roleCommand + '\'' +
+ ", ip='" + ip + '\'' +
+ ", hostname='" + hostname + '\'' +
'}';
}
+
+ public String getIp() {
+ return ip;
+ }
+
+ public void setIp(String ip) {
+ this.ip = ip;
+ }
+
+ public String getHostname() {
+ return hostname;
+ }
+
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index e0d3a75..eb97a55 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -59,6 +59,7 @@ public class ExecutionCommand {
private String componentType;
private List<DockerContainer> containers = new ArrayList<>();
private String pkg;
+ private boolean yarnDockerMode = false;
public ExecutionCommand(AgentCommandType commandType) {
this.commandType = commandType;
@@ -229,7 +230,16 @@ public class ExecutionCommand {
public List<DockerContainer> getContainers() {
return containers;
}
-
+
+ @JsonProperty("yarnDockerMode")
+ public boolean isYarnDockerMode() {
+ return yarnDockerMode ;
+ }
+
+ @JsonProperty("yarnDockerMode")
+ public void setYarnDockerMode(boolean yarnDockerMode) {
+ this.yarnDockerMode = yarnDockerMode;
+ }
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
@@ -243,7 +253,8 @@ public class ExecutionCommand {
.append(", commandParams=").append(commandParams)
.append(", serviceName=").append(serviceName)
.append(", componentName=").append(componentName)
- .append(", componentType=").append(componentType).append(", pkg=")
+ .append(", componentType=").append(componentType)
+ .append(", yarnDockerMode=").append(yarnDockerMode).append(", pkg=")
.append(pkg).append("]");
return builder.toString();
}
@@ -256,11 +267,14 @@ public class ExecutionCommand {
for (DockerContainer metaContainer : component.getDockerContainers()) {
DockerContainer container = new DockerContainer();
container.setImage(metaContainer.getImage());
+ container.setNetwork(metaContainer.getNetwork());
+ container.setUseNetworkScript(metaContainer.getUseNetworkScript());
container.setName(metaContainer.getName());
container.setOptions(metaContainer.getOptions());
container.setAdditionalParam(metaContainer.getAdditionalParam());
container.setCommandPath(metaContainer.getAdditionalParam());
container.setStatusCommand(metaContainer.getStatusCommand());
+ container.setStartCommand(metaContainer.getStartCommand());
if (metaContainer.getMounts().size() > 0) {
for (DockerContainerMount metaContMount : metaContainer.getMounts()) {
DockerContainerMount contMnt = new DockerContainerMount();
@@ -286,6 +300,9 @@ public class ExecutionCommand {
container.getInputFiles().add(inpFile);
}
}
+ if (metaContainer.getConfigFiles() != null) {
+ container.setConfigFiles(metaContainer.getConfigFiles());
+ }
log.info("Docker container meta info ready: " + container.toString());
this.getContainers().add(container);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
index b7172c2..d17c465 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeat.java
@@ -125,12 +125,12 @@ public class HeartBeat {
}
@JsonProperty("package")
- public String getPkg() {
+ public String getPackage() {
return pkg;
}
@JsonProperty("package")
- public void setPkg(String pkg) {
+ public void setPackage(String pkg) {
this.pkg = pkg;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
index c118840..b500d67 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/HeartBeatResponse.java
@@ -39,6 +39,7 @@ public class HeartBeatResponse {
RegistrationCommand registrationCommand;
+ boolean yarnDockerMode = false;
boolean restartAgent = false;
boolean restartEnabled = true;
boolean hasMappedComponents = false;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
index 50a3c87..80b7a5e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/RegistrationResponse.java
@@ -58,6 +58,22 @@ public class RegistrationResponse {
public RegistrationResponse() {
}
+ public RegistrationStatus getResponse() {
+ return response;
+ }
+
+ public void setResponse(RegistrationStatus response) {
+ this.response = response;
+ }
+
+ public int getExitstatus() {
+ return exitstatus;
+ }
+
+ public void setExitstatus(int exitstatus) {
+ this.exitstatus = exitstatus;
+ }
+
public RegistrationStatus getResponseStatus() {
return response;
}
@@ -90,8 +106,8 @@ public class RegistrationResponse {
this.tags = tags;
}
- public void setExitstatus(int exitstatus) {
- this.exitstatus = exitstatus;
+ public String getLog() {
+ return log;
}
public void setLog(String log) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
index 52fb8dd..5b205b5 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/StatusCommand.java
@@ -41,6 +41,7 @@ public class StatusCommand {
private Map<String, String> commandParams = new HashMap<String, String>();
private Map<String, String> hostLevelParams = new HashMap<String, String>();
private String roleCommand;
+ private boolean yarnDockerMode;
public StatusCommand() {
this.agentCommandType = AgentCommandType.STATUS_COMMAND;
@@ -125,6 +126,16 @@ public class StatusCommand {
public void setRoleCommand(String roleCommand) {
this.roleCommand = roleCommand;
}
+
+ @JsonProperty("yarnDockerMode")
+ public boolean isYarnDockerMode() {
+ return yarnDockerMode;
+ }
+
+ @JsonProperty("yarnDockerMode")
+ public void setYarnDockerMode(boolean yarnDockerMode) {
+ this.yarnDockerMode = yarnDockerMode;
+ }
@Override
public String toString() {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json
new file mode 100644
index 0000000..8839efc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/appConfig.json
@@ -0,0 +1,10 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "YCLOUD": {
+ "ycloud.statusCommand": "ls"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json
new file mode 100644
index 0000000..79c80e9
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/metainfo.json
@@ -0,0 +1,44 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "MYYCLOUD",
+ "components": [
+ {
+ "name": "YCLOUD1",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud1",
+ "image": "doesnotexist",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49160"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "YCLOUD2",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud2",
+ "image": "doesnotexist",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49161"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json
new file mode 100644
index 0000000..985d8e3
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/allCompFailed/resources.json
@@ -0,0 +1,20 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "slider-appmaster": {},
+ "YCLOUD1": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": 2,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ },
+ "YCLOUD2": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": 2,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json
new file mode 100644
index 0000000..8839efc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/appConfig.json
@@ -0,0 +1,10 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "YCLOUD": {
+ "ycloud.statusCommand": "ls"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json
new file mode 100644
index 0000000..67d6171
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/metainfo.json
@@ -0,0 +1,26 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "MYYCLOUD",
+ "components": [
+ {
+ "name": "YCLOUD",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49160"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json
new file mode 100644
index 0000000..22ff65e
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/basic/resources.json
@@ -0,0 +1,14 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "slider-appmaster": {},
+ "YCLOUD": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": 1,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json
new file mode 100644
index 0000000..8839efc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/appConfig.json
@@ -0,0 +1,10 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "YCLOUD": {
+ "ycloud.statusCommand": "ls"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json
new file mode 100644
index 0000000..efbb638
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/metainfo.json
@@ -0,0 +1,44 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "MYYCLOUD",
+ "components": [
+ {
+ "name": "YCLOUD1",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud1",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49160"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "YCLOUD2",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud2",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49161"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json
new file mode 100644
index 0000000..0cbb88a
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiComp/resources.json
@@ -0,0 +1,20 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "slider-appmaster": {},
+ "YCLOUD1": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": 1,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ },
+ "YCLOUD2": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": 1,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json
new file mode 100644
index 0000000..8839efc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/appConfig.json
@@ -0,0 +1,10 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "YCLOUD": {
+ "ycloud.statusCommand": "ls"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json
new file mode 100644
index 0000000..efbb638
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/metainfo.json
@@ -0,0 +1,44 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "MYYCLOUD",
+ "components": [
+ {
+ "name": "YCLOUD1",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud1",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49160"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "YCLOUD2",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud2",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49161"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json
new file mode 100644
index 0000000..399a301
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/multiCompMultiContainer/resources.json
@@ -0,0 +1,20 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "slider-appmaster": {},
+ "YCLOUD1": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": 2,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ },
+ "YCLOUD2": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": 3,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json
new file mode 100644
index 0000000..8839efc
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/appConfig.json
@@ -0,0 +1,10 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "YCLOUD": {
+ "ycloud.statusCommand": "ls"
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json
new file mode 100644
index 0000000..2befb20
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/metainfo.json
@@ -0,0 +1,44 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "MYYCLOUD",
+ "components": [
+ {
+ "name": "YCLOUD1",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud1",
+ "image": "doesnotexist",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49160"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "YCLOUD2",
+ "type": "yarn_docker",
+ "dockerContainers": [
+ {
+ "name": "ycloud2",
+ "image": "ycloud-centos6:latest",
+ "startCommand": "ls",
+ "yarnContainerMountPoint": "null",
+ "ports": [
+ {
+ "containerPort": "8080",
+ "hostPort": "49161"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1ab0cbbd/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json
new file mode 100644
index 0000000..985d8e3
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_docker_on_yarn_pkg/oneCompFailed/resources.json
@@ -0,0 +1,20 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {},
+ "global": {},
+ "components": {
+ "slider-appmaster": {},
+ "YCLOUD1": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": 2,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ },
+ "YCLOUD2": {
+ "yarn.role.priority": "2",
+ "yarn.component.instances": 2,
+ "yarn.memory": "256",
+ "yarn.vcores": "1"
+ }
+ }
+}
\ No newline at end of file
[20/31] incubator-slider git commit: SLIDER-1134 App config key-value
pairs specified on the command line seeps into resources definition
Posted by el...@apache.org.
SLIDER-1134 App config key-value pairs specified on the command line seeps into resources definition
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/e6999d0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/e6999d0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/e6999d0c
Branch: refs/heads/branches/branch-0.91
Commit: e6999d0c1df987705855406791a46f80e1c3effc
Parents: eccefc1
Author: Gour Saha <go...@apache.org>
Authored: Tue May 31 07:53:21 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Tue May 31 07:53:21 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/slider/client/SliderClient.java | 1 -
.../slider/common/params/AppAndResouceOptionArgsDelegate.java | 2 +-
2 files changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e6999d0c/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index dd90e46..f74d3b7 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1593,7 +1593,6 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// resource component args
appConf.merge(cmdLineResourceOptions);
- resources.merge(cmdLineResourceOptions);
resources.mergeComponents(buildInfo.getResourceCompOptionMap());
builder.init(providerName, instanceDefinition);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/e6999d0c/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
index 248e4c2..f171708 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AppAndResouceOptionArgsDelegate.java
@@ -95,7 +95,7 @@ public class AppAndResouceOptionArgsDelegate extends AbstractArgsDelegate {
*/
public Map<String, Map<String, String>> getResourceCompOptionMap() throws
BadCommandArgumentsException {
- return convertTripleListToMaps(ARG_COMP_OPT, resCompOptTriples);
+ return convertTripleListToMaps(ARG_RES_COMP_OPT, resCompOptTriples);
}
public void setOption(String key, String value) {
[06/31] incubator-slider git commit: SLIDER-1112 allow failure of
creation of applications in test case
Posted by el...@apache.org.
SLIDER-1112 allow failure of creation of applications in test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1b833e55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1b833e55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1b833e55
Branch: refs/heads/branches/branch-0.91
Commit: 1b833e55abe4d6524171db636a5b570fe9148213
Parents: 7b560b9
Author: Yu Liu <yu...@apache.org>
Authored: Wed Apr 20 19:04:35 2016 -0700
Committer: Yu Liu <yu...@apache.org>
Committed: Wed Apr 20 19:04:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/funtest/framework/CommandTestBase.groovy | 6 ++++--
.../funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy | 2 +-
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1b833e55/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index b3bba74..f6bdd19 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -730,7 +730,8 @@ abstract class CommandTestBase extends SliderTestUtils {
String appTemplate,
String resourceTemplate,
List<String> extraArgs = [],
- File launchReportFile = null) {
+ File launchReportFile = null,
+ boolean failOnError = true) {
if (!launchReportFile) {
launchReportFile = createTempJsonFile()
@@ -766,7 +767,8 @@ abstract class CommandTestBase extends SliderTestUtils {
"env." + Constants.HADOOP_JAAS_DEBUG << "true";
commands.addAll(extraArgs)
SliderShell shell = new SliderShell(commands)
- if (0 != shell.execute()) {
+ int returnCode = shell.execute()
+ if (failOnError && 0 != returnCode) {
// app has failed.
// grab the app report of the last known instance of this app
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1b833e55/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
index ebce6df..20ff75c 100644
--- a/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/coprocessors/ApplicationWithAddonPackagesIT.groovy
@@ -119,7 +119,7 @@ public class ApplicationWithAddonPackagesIT extends AgentCommandTestBase{
APP_TEMPLATE,
APP_RESOURCE2,
[Arguments.ARG_ADDON, ADD_ON_PACKAGE_NO_COMPONENT_PKG_NAME, addOnPackageNoComponentPkgNameFilePath.toString()],
- launchReportFile)
+ launchReportFile, false)
} finally {
CommandTestBase.THAW_WAIT_TIME = temp_holder;
}
[26/31] incubator-slider git commit: SLIDER-1140: KDiag can't handle
a % sign in a krb5.conf file
Posted by el...@apache.org.
SLIDER-1140: KDiag can't handle a % sign in a krb5.conf file
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f95ba34a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f95ba34a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f95ba34a
Branch: refs/heads/branches/branch-0.91
Commit: f95ba34a4692b4d6f596cb05a6a23929c74b31ef
Parents: 48975be
Author: Steve Loughran <st...@apache.org>
Authored: Thu Jun 9 14:17:14 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jun 9 14:17:14 2016 +0100
----------------------------------------------------------------------
.../apache/hadoop/security/KerberosDiags.java | 47 ++++++++++++++----
.../apache/slider/client/TestDiagnostics.groovy | 52 ++++++++++++++++++--
2 files changed, 87 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f95ba34a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
index 1747a2b..8c572b3 100644
--- a/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
+++ b/slider-core/src/main/java/org/apache/hadoop/security/KerberosDiags.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.security;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -316,11 +317,11 @@ public class KerberosDiags implements Closeable {
}
/**
- * Dump a keytab: list all principals
+ * Dump a keytab: list all principals.
* @param keytabFile the keytab file
* @throws IOException IO problems
*/
- private void dumpKeytab(File keytabFile) throws IOException {
+ public void dumpKeytab(File keytabFile) throws IOException {
title("Examining keytab %s", keytabFile);
File kt = keytabFile.getCanonicalFile();
failif(!kt.exists(), CAT_CONFIG, "Keytab not found: %s", kt);
@@ -513,15 +514,27 @@ public class KerberosDiags implements Closeable {
}
/**
- * Print a line of output. This goes to any output file, or
+ * Format and print a line of output.
+ * This goes to any output file, or
* is logged at info. The output is flushed before and after, to
* try and stay in sync with JRE logging.
* @param format format string
* @param args any arguments
*/
- private void println(String format, Object... args) {
+ @VisibleForTesting
+ public void println(String format, Object... args) {
+ println(format(format, args));
+ }
+
+ /**
+ * Print a line of output. This goes to any output file, or
+ * is logged at info. The output is flushed before and after, to
+ * try and stay in sync with JRE logging.
+ * @param msg message string
+ */
+ @VisibleForTesting
+ private void println(String msg) {
flush();
- String msg = String.format(format, args);
if (out != null) {
out.println(msg);
} else {
@@ -538,7 +551,7 @@ public class KerberosDiags implements Closeable {
private void title(String format, Object... args) {
println("");
println("");
- String msg = "== " + String.format(format, args) + " ==";
+ String msg = "== " + format(format, args) + " ==";
println(msg);
println("");
}
@@ -575,10 +588,10 @@ public class KerberosDiags implements Closeable {
* @param file file to dump
* @throws IOException IO problems
*/
- private void dump(File file) throws IOException {
+ public void dump(File file) throws IOException {
try (FileInputStream in = new FileInputStream(file)) {
for (String line : IOUtils.readLines(in)) {
- println(line);
+ println("%s", line);
}
}
println("");
@@ -617,6 +630,22 @@ public class KerberosDiags implements Closeable {
}
/**
+ * Format a string, treating a call where there are no varags values
+ * as a string to pass through unformatted.
+ * @param message message, which is either a format string + args, or
+ * a general string
+ * @param args argument array
+ * @return a string for printing.
+ */
+ public static String format(String message, Object... args) {
+ if (args.length == 0) {
+ return message;
+ } else {
+ return String.format(message, args);
+ }
+ }
+
+ /**
* Diagnostics failures return the exit code 41, "unauthorized".
*
* They have a category, initially for testing: the category can be
@@ -631,7 +660,7 @@ public class KerberosDiags implements Closeable {
}
public KerberosDiagsFailure(String category, String message, Object... args) {
- this(category, String.format(message, args));
+ this(category, format(message, args));
}
public KerberosDiagsFailure(String category, Throwable throwable,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f95ba34a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
index a20ab23..81c4daf 100644
--- a/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/client/TestDiagnostics.groovy
@@ -18,8 +18,10 @@
package org.apache.slider.client
-import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
+import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.fs.FileUtil
+import org.apache.hadoop.security.KerberosDiags
import org.apache.hadoop.yarn.conf.YarnConfiguration
import static org.apache.slider.common.Constants.SUN_SECURITY_KRB5_DEBUG
import org.apache.slider.common.params.ActionDiagnosticArgs
@@ -28,8 +30,6 @@ import org.apache.slider.common.params.ClientArgs
import org.apache.slider.common.params.SliderActions
import org.apache.slider.common.tools.SliderUtils
import org.apache.slider.core.main.ServiceLauncher
-import org.apache.slider.test.SliderTestBase
-import org.apache.slider.test.YarnMiniClusterTestBase
import org.apache.slider.test.YarnZKMiniClusterTestBase
import org.junit.Test
@@ -72,4 +72,50 @@ class TestDiagnostics extends YarnZKMiniClusterTestBase {
assert 0 == launcher.serviceExitCode
}
+ @Test
+ public void testKDiagExceptionConstruction() throws Throwable {
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", "%02d", 3).toString().contains("03")
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", "%w").toString().contains("%w")
+ assert new KerberosDiags.KerberosDiagsFailure("CAT", new Exception(), "%w")
+ .toString().contains("%w")
+ }
+
+ @Test
+ public void testKDiagPrintln() throws Throwable {
+ assert "%w" == KerberosDiags.format("%w")
+ assert "%s" == KerberosDiags.format("%s")
+ assert "false" == KerberosDiags.format("%s", false)
+ def sw = new StringWriter()
+ def kdiag = new KerberosDiags(new Configuration(),
+ new PrintWriter(sw), [], null, "self", 16, false)
+ try {
+ kdiag.println("%02d", 3)
+ kdiag.println("%s")
+ kdiag.println("%w")
+ } finally {
+ kdiag.close()
+ }
+ def output = sw.toString()
+ assert output.contains("03")
+ assert output.contains("%s")
+ assert output.contains("%w")
+ }
+
+ @Test
+ public void testKDiagDumpFile() throws Throwable {
+ def file1 = new File("./target/kdiaginput.txt")
+
+ def s = 'invalid %w string %s'
+ file1 << s
+ def sw = new StringWriter()
+ def kdiag = new KerberosDiags(new Configuration(),
+ new PrintWriter(sw), [], null, "self", 16, false)
+ try {
+ kdiag.dump(file1)
+ } finally {
+ kdiag.close()
+ }
+ def output = sw.toString()
+ assert output.contains(s)
+ }
}
[05/31] incubator-slider git commit: SLIDER-1111 fix SSL test issues
with newer JDKs
Posted by el...@apache.org.
SLIDER-1111 fix SSL test issues with newer JDKs
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/7b560b91
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/7b560b91
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/7b560b91
Branch: refs/heads/branches/branch-0.91
Commit: 7b560b910f5754a4c0fb2918972b7af41a20b7fd
Parents: 9130f0e
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon Apr 18 11:47:23 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon Apr 18 11:47:23 2016 -0700
----------------------------------------------------------------------
.../funtest/accumulo/AccumuloSSLTestBase.groovy | 31 ++++++++++++++++----
.../server/services/security/SecurityUtils.java | 2 +-
2 files changed, 26 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b560b91/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
index 240aad1..95eb0fb 100644
--- a/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
+++ b/app-packages/accumulo/src/test/groovy/org/apache/slider/funtest/accumulo/AccumuloSSLTestBase.groovy
@@ -21,8 +21,9 @@ import org.junit.BeforeClass
import javax.net.ssl.KeyManager
import javax.net.ssl.SSLContext
+import javax.net.ssl.SSLEngine
import javax.net.ssl.TrustManager
-import javax.net.ssl.X509TrustManager
+import javax.net.ssl.X509ExtendedTrustManager
import java.security.SecureRandom
import java.security.cert.CertificateException
import java.security.cert.X509Certificate
@@ -56,16 +57,34 @@ class AccumuloSSLTestBase extends AccumuloBasicIT {
SSLContext.setDefault(ctx);
}
- private static class DefaultTrustManager implements X509TrustManager {
+ private static class DefaultTrustManager extends X509ExtendedTrustManager {
@Override
- public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+ void checkClientTrusted(X509Certificate[] x509Certificates, String s, Socket socket) throws CertificateException {
+ }
+
+ @Override
+ void checkServerTrusted(X509Certificate[] x509Certificates, String s, Socket socket) throws CertificateException {
+ }
+
+ @Override
+ void checkClientTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException {
+ }
@Override
- public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {}
+ void checkServerTrusted(X509Certificate[] x509Certificates, String s, SSLEngine sslEngine) throws CertificateException {
+ }
+
+ @Override
+ void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+ }
+
+ @Override
+ void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
+ }
@Override
- public X509Certificate[] getAcceptedIssuers() {
- return null;
+ X509Certificate[] getAcceptedIssuers() {
+ return new X509Certificate[0]
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/7b560b91/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
index 4b2c557..9beb2d8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/server/services/security/SecurityUtils.java
@@ -58,7 +58,7 @@ public class SecurityUtils {
+ "default_days = 365 \n"
+ "\n"
+ "default_crl_days = 7 \n"
- + "default_md = md5 \n"
+ + "default_md = sha256 \n"
+ "\n"
+ "policy = policy_anything \n"
+ "\n"
[22/31] incubator-slider git commit: SLIDER-1134 remove command line
appconfig options from resources
Posted by el...@apache.org.
SLIDER-1134 remove command line appconfig options from resources
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/baf6532d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/baf6532d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/baf6532d
Branch: refs/heads/branches/branch-0.91
Commit: baf6532ddf9b91e647cfee3995b12373c3a94f4d
Parents: 66257d7
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Thu Jun 2 12:05:27 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Thu Jun 2 12:05:27 2016 -0700
----------------------------------------------------------------------
.../org/apache/slider/client/SliderClient.java | 1 +
.../AbstractClusterBuildingActionArgs.java | 6 ---
.../standalone/TestBuildStandaloneAM.groovy | 50 ++++++++++++++++++++
3 files changed, 51 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baf6532d/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index f74d3b7..dd90e46 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -1593,6 +1593,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// resource component args
appConf.merge(cmdLineResourceOptions);
+ resources.merge(cmdLineResourceOptions);
resources.mergeComponents(buildInfo.getResourceCompOptionMap());
builder.init(providerName, instanceDefinition);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baf6532d/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 3833394..2a5eedc 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -212,12 +212,6 @@ public abstract class AbstractClusterBuildingActionArgs extends
ConfTree confTree = new ConfTree();
ConfTreeOperations ops = new ConfTreeOperations(confTree);
confTree.global.putAll(optionsMap);
- Map<String, Map<String, String>> roleOptionMap = getCompOptionMap();
- for (Map.Entry<String, Map<String, String>> entry : roleOptionMap.entrySet()) {
- String key = entry.getKey();
- Map<String, String> value = entry.getValue();
- ops.getOrAddComponent(key).putAll(value);
- }
return confTree;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/baf6532d/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
index 6637c73..4a97e69 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/standalone/TestBuildStandaloneAM.groovy
@@ -170,4 +170,54 @@ class TestBuildStandaloneAM extends AgentMiniClusterTestBase {
assert masterRole != null, "Role hbase-master must exist"
assert cd.roleNames.contains(master), "Role names must contain hbase-master"
}
+
+ @Test
+ public void testBuildClusterWithCompOpt() throws Throwable {
+ String clustername = createMiniCluster("", configuration, 1, true)
+
+ describe "verify that a cluster can be built with compots"
+
+ String echo = "echo"
+ ServiceLauncher<SliderClient> launcher = createOrBuildCluster(
+ SliderActions.ACTION_BUILD,
+ clustername,
+ [(echo): 1],
+ [ARG_COMP_OPT, echo, "non.resource.property", "value",
+ ARG_COMP_OPT, echo, "yarn.some.property", "yarn",
+ ARG_COMP_OPT, echo, "component.some.property", "component",
+ ARG_COMP_OPT, echo, "role.some.property", "role",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PRIORITY, "1",
+ ARG_RES_COMP_OPT, echo, ResourceKeys.COMPONENT_PLACEMENT_POLICY, "4"],
+ true,
+ false,
+ agentDefOptions)
+ SliderClient sliderClient = launcher.service
+ addToTeardown(sliderClient);
+
+ // verify the cluster exists
+ assert 0 == sliderClient.actionExists(clustername, false)
+
+ // verify that global component options propagate from the CLI
+ def aggregateConf = sliderClient.loadPersistedClusterDescription(clustername)
+ assert "value" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "non.resource.property", null)
+ assert null == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "non.resource.property", null)
+ assert "yarn" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "yarn.some.property", null)
+ assert "yarn" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "yarn.some.property", null)
+ assert "component" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "component.some.property", null)
+ assert "component" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "component.some.property", null)
+ assert "role" == aggregateConf.appConfOperations.getComponentOpt(echo,
+ "role.some.property", null)
+ assert "role" == aggregateConf.resourceOperations.getComponentOpt(echo,
+ "role.some.property", null)
+ assert 1 == aggregateConf.resourceOperations.getComponentOptInt(echo,
+ ResourceKeys.COMPONENT_PRIORITY, -1)
+ assert 4 == aggregateConf.resourceOperations.getComponentOptInt(echo,
+ ResourceKeys.COMPONENT_PLACEMENT_POLICY, -1)
+ }
}
[24/31] incubator-slider git commit: SLIDER-1129 cut out (currently
commented out) references to the httpclient and httpcore.
Posted by el...@apache.org.
SLIDER-1129 cut out (currently commented out) references to the httpclient and httpcore.
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/48975be4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/48975be4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/48975be4
Branch: refs/heads/branches/branch-0.91
Commit: 48975be4eab51b8c1916513f5113d1ffad8469f9
Parents: eb04524
Author: Steve Loughran <st...@apache.org>
Authored: Tue May 24 18:28:46 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Thu Jun 9 11:59:40 2016 +0100
----------------------------------------------------------------------
slider-core/pom.xml | 13 -------------
1 file changed, 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/48975be4/slider-core/pom.xml
----------------------------------------------------------------------
diff --git a/slider-core/pom.xml b/slider-core/pom.xml
index 8b921a5..04e9015 100644
--- a/slider-core/pom.xml
+++ b/slider-core/pom.xml
@@ -326,19 +326,6 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
-<!--
-
- <dependency>
- <groupId>commons-httpclient</groupId>
- <artifactId>commons-httpclient</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <scope>test</scope>
- </dependency>
--->
<dependency>
<groupId>commons-lang</groupId>
[13/31] incubator-slider git commit: SLIDER-1124 throw exception if
bad port range is specified
Posted by el...@apache.org.
SLIDER-1124 throw exception if bad port range is specified
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/5c391cc6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/5c391cc6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/5c391cc6
Branch: refs/heads/branches/branch-0.91
Commit: 5c391cc6caaf772f8aa4296d3444c19c6c8ed534
Parents: cfb516b
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Mon May 23 10:04:03 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Mon May 23 10:04:03 2016 -0700
----------------------------------------------------------------------
.../apache/slider/common/tools/PortScanner.java | 34 ++++++++++-----
.../server/appmaster/SliderAppMaster.java | 3 +-
.../slider/common/tools/TestPortScan.groovy | 44 ++++++++++++++++++++
3 files changed, 70 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5c391cc6/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
index 5b80f9f..64783b6 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/PortScanner.java
@@ -17,6 +17,7 @@
package org.apache.slider.common.tools;
import org.apache.slider.common.SliderExitCodes;
+import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
import java.io.IOException;
@@ -40,25 +41,38 @@ public class PortScanner {
public PortScanner() {
}
- public void setPortRange(String input) {
+ public void setPortRange(String input) throws BadConfigException {
// first split based on commas
Set<Integer> inputPorts= new TreeSet<Integer>();
String[] ranges = input.split(",");
for ( String range : ranges ) {
+ if (range.trim().isEmpty()) {
+ continue;
+ }
Matcher m = SINGLE_NUMBER.matcher(range.trim());
if (m.find()) {
inputPorts.add(Integer.parseInt(m.group()));
- } else {
- m = NUMBER_RANGE.matcher(range.trim());
- if (m.find()) {
- String[] boundaryValues = m.group(0).split("-");
- int start = Integer.parseInt(boundaryValues[0].trim());
- int end = Integer.parseInt(boundaryValues[1].trim());
- for (int i = start; i < end + 1; i++) {
- inputPorts.add(i);
- }
+ continue;
+ }
+ m = NUMBER_RANGE.matcher(range.trim());
+ if (m.find()) {
+ String[] boundaryValues = m.group(0).split("-");
+ int start = Integer.parseInt(boundaryValues[0].trim());
+ int end = Integer.parseInt(boundaryValues[1].trim());
+ if (end < start) {
+ throw new BadConfigException("End of port range is before start: "
+ + range + " in input: " + input);
}
+ for (int i = start; i < end + 1; i++) {
+ inputPorts.add(i);
+ }
+ continue;
}
+ throw new BadConfigException("Bad port range: " + range + " in input: "
+ + input);
+ }
+ if (inputPorts.size() == 0) {
+ throw new BadConfigException("No ports found in range: " + input);
}
this.remainingPortsToCheck = new ArrayList<Integer>(inputPorts);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5c391cc6/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 0776a6c..b767059 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1137,7 +1137,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
/**
* Build up the port scanner. This may include setting a port range.
*/
- private void buildPortScanner(AggregateConf instanceDefinition) {
+ private void buildPortScanner(AggregateConf instanceDefinition)
+ throws BadConfigException {
portScanner = new PortScanner();
String portRange = instanceDefinition.
getAppConfOperations().getGlobalOptions().
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/5c391cc6/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
index f009e25..4d87c75 100644
--- a/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/common/tools/TestPortScan.groovy
@@ -18,6 +18,7 @@
package org.apache.slider.common.tools
+import org.apache.slider.core.exceptions.BadConfigException
import org.apache.slider.core.exceptions.SliderException
import org.junit.Test
@@ -125,4 +126,47 @@ class TestPortScan {
server.close()
}
}
+
+ @Test(expected = BadConfigException.class)
+ public void testBadRange() {
+ PortScanner portScanner = new PortScanner()
+ // note the em dash
+ portScanner.setPortRange("2000\u20132010")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEndBeforeStart() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2001-2000")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testEmptyRange() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("")
+ }
+
+ @Test(expected = BadConfigException.class)
+ public void testBlankRange() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange(" ")
+ }
+
+ @Test
+ public void testExtraComma() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2000-2001, ")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [2000, 2001]
+ assert ports == expectedPorts
+ }
+
+ @Test
+ public void testExtraCommas() {
+ PortScanner portScanner = new PortScanner()
+ portScanner.setPortRange("2000-2001,, ,2003,")
+ List<Integer> ports = portScanner.remainingPortsToCheck
+ def expectedPorts = [2000, 2001, 2003]
+ assert ports == expectedPorts
+ }
}
[30/31] incubator-slider git commit: SLIDER-952 Remove unnecessary
(copied) images and unnecessary year in copyright header
Posted by el...@apache.org.
SLIDER-952 Remove unnecessary (copied) images and unnecessary year in copyright header
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/55193be4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/55193be4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/55193be4
Branch: refs/heads/branches/branch-0.91
Commit: 55193be4aec99c7f8b174c775cfefc10b001b532
Parents: 2b41da2
Author: Josh Elser <el...@apache.org>
Authored: Thu Jun 16 22:45:50 2016 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Thu Jun 16 22:45:50 2016 -0400
----------------------------------------------------------------------
app-packages/storm/package/files/storm-slider | 2 --
slider-agent/src/test/python/python-wrap | 1 -
.../src/main/resources/webapps/static/busy.gif | Bin 3494 -> 0 bytes
.../main/resources/webapps/static/hadoop-st.png | Bin 23946 -> 0 bytes
4 files changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55193be4/app-packages/storm/package/files/storm-slider
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/storm-slider b/app-packages/storm/package/files/storm-slider
index af492bd..7a9203e 100644
--- a/app-packages/storm/package/files/storm-slider
+++ b/app-packages/storm/package/files/storm-slider
@@ -1,7 +1,5 @@
#!/bin/bash
#
-# Copyright 2014 The Apache Software Foundation
-#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55193be4/slider-agent/src/test/python/python-wrap
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/python-wrap b/slider-agent/src/test/python/python-wrap
index 88a8c55..d95cf07 100755
--- a/slider-agent/src/test/python/python-wrap
+++ b/slider-agent/src/test/python/python-wrap
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-# Copyright 2011 The Apache Software Foundation
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55193be4/slider-core/src/main/resources/webapps/static/busy.gif
----------------------------------------------------------------------
diff --git a/slider-core/src/main/resources/webapps/static/busy.gif b/slider-core/src/main/resources/webapps/static/busy.gif
deleted file mode 100644
index 058ce1f..0000000
Binary files a/slider-core/src/main/resources/webapps/static/busy.gif and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/55193be4/slider-core/src/main/resources/webapps/static/hadoop-st.png
----------------------------------------------------------------------
diff --git a/slider-core/src/main/resources/webapps/static/hadoop-st.png b/slider-core/src/main/resources/webapps/static/hadoop-st.png
deleted file mode 100644
index b481c04..0000000
Binary files a/slider-core/src/main/resources/webapps/static/hadoop-st.png and /dev/null differ
[23/31] incubator-slider git commit: SLIDER-1138 pass memory and core
reqs to unique components
Posted by el...@apache.org.
SLIDER-1138 pass memory and core reqs to unique components
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/c7a173cd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/c7a173cd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/c7a173cd
Branch: refs/heads/branches/branch-0.91
Commit: c7a173cdcc15f675c1b4d471ff40eefb9afb1bae
Parents: baf6532
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 8 11:28:28 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 8 11:28:28 2016 -0700
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 26 +++++++++++---------
.../appstate/TestMockAppStateUniqueNames.groovy | 22 +++++++++++++++++
2 files changed, 36 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index ed87b89..b4465a8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -802,6 +802,7 @@ public class AppState {
continue;
}
if (hasUniqueNames(resources, name)) {
+ // THIS NAME IS A GROUP
int desiredInstanceCount = getDesiredInstanceCount(resources, name);
Integer groupCount = 0;
if (groupCounts.containsKey(name)) {
@@ -853,21 +854,21 @@ public class AppState {
/**
* Get the desired instance count of a role, rejecting negative values
* @param resources resource map
- * @param role role name
+ * @param roleGroup role group
* @return the instance count
* @throws BadConfigException if the count is negative
*/
private int getDesiredInstanceCount(ConfTreeOperations resources,
- String role) throws BadConfigException {
+ String roleGroup) throws BadConfigException {
int desiredInstanceCount =
- resources.getComponentOptInt(role, COMPONENT_INSTANCES, 0);
+ resources.getComponentOptInt(roleGroup, COMPONENT_INSTANCES, 0);
if (desiredInstanceCount < 0) {
- log.error("Role {} has negative desired instances : {}", role,
+ log.error("Role {} has negative desired instances : {}", roleGroup,
desiredInstanceCount);
throw new BadConfigException(
"Negative instance count (%) requested for component %s",
- desiredInstanceCount, role);
+ desiredInstanceCount, roleGroup);
}
return desiredInstanceCount;
}
@@ -1353,12 +1354,12 @@ public class AppState {
* @throws NumberFormatException if the role could not be parsed.
*/
private int getResourceRequirement(ConfTreeOperations resources,
- String name,
+ String group,
String option,
int defVal,
int maxVal) {
- String val = resources.getComponentOpt(name, option,
+ String val = resources.getComponentOpt(group, option,
Integer.toString(defVal));
Integer intVal;
if (YARN_RESOURCE_MAX.equals(val)) {
@@ -1380,14 +1381,15 @@ public class AppState {
public Resource buildResourceRequirements(RoleStatus role, Resource capability) {
// Set up resource requirements from role values
String name = role.getName();
+ String group = role.getGroup();
ConfTreeOperations resources = getResourcesSnapshot();
int cores = getResourceRequirement(resources,
- name,
+ group,
YARN_CORES,
DEF_YARN_CORES,
containerMaxCores);
capability.setVirtualCores(cores);
- int ram = getResourceRequirement(resources, name,
+ int ram = getResourceRequirement(resources, group,
YARN_MEMORY,
DEF_YARN_MEMORY,
containerMaxMemory);
@@ -1959,13 +1961,13 @@ public class AppState {
/**
* Get the node failure threshold for a specific role, falling back to
* the global one if not
- * @param roleName role name
+ * @param roleGroup role group
* @return the threshold for failures
*/
- private int getNodeFailureThresholdForRole(String roleName) {
+ private int getNodeFailureThresholdForRole(String roleGroup) {
ConfTreeOperations resources =
instanceDefinition.getResourceOperations();
- return resources.getComponentOptInt(roleName,
+ return resources.getComponentOptInt(roleGroup,
NODE_FAILURE_THRESHOLD,
nodeFailureThreshold);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/c7a173cd/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
index 5256163..d3ba798 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -71,6 +71,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "1",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "1024",
+ (ResourceKeys.YARN_CORES) : "2",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -90,6 +92,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "2",
(ResourceKeys.COMPONENT_PRIORITY): "7",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -103,6 +107,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group21").priority
assert 8 == appState.lookupRoleStatus("group22").priority
+ assert 1024 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group21").resourceRequirements.memory
+ assert 384 == appState.lookupRoleStatus("group22").resourceRequirements.memory
+ assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores
+ assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores
}
@Test
@@ -111,6 +121,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "0",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "384",
+ (ResourceKeys.YARN_CORES) : "4",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -121,6 +133,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
appState.lookupRoleStatus(6)
assert 0 == appState.lookupRoleStatus("group11").desired
assert 6 == appState.lookupRoleStatus("group11").priority
+ assert 384 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 4 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
}
@Test
@@ -129,6 +143,8 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
def opts = [
(ResourceKeys.COMPONENT_INSTANCES): "3",
(ResourceKeys.COMPONENT_PRIORITY) : "6",
+ (ResourceKeys.YARN_MEMORY) : "640",
+ (ResourceKeys.YARN_CORES) : "8",
(ResourceKeys.UNIQUE_NAMES) : "true",
]
@@ -145,6 +161,12 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 6 == appState.lookupRoleStatus("group11").priority
assert 7 == appState.lookupRoleStatus("group12").priority
assert 8 == appState.lookupRoleStatus("group13").priority
+ assert 640 == appState.lookupRoleStatus("group11").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group12").resourceRequirements.memory
+ assert 640 == appState.lookupRoleStatus("group13").resourceRequirements.memory
+ assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores
+ assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores
}
}
[28/31] incubator-slider git commit: SLIDER-1141 add status for
unique components
Posted by el...@apache.org.
SLIDER-1141 add status for unique components
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/454f2cf2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/454f2cf2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/454f2cf2
Branch: refs/heads/branches/branch-0.91
Commit: 454f2cf2e391bc261f7905181a310994976973cf
Parents: 24a5062
Author: Billie Rinaldi <bi...@gmail.com>
Authored: Wed Jun 15 07:18:42 2016 -0700
Committer: Billie Rinaldi <bi...@gmail.com>
Committed: Wed Jun 15 08:08:37 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/slider/api/RoleKeys.java | 5 ++++
.../slider/server/appmaster/state/AppState.java | 14 ++++++++--
.../appstate/TestMockAppStateUniqueNames.groovy | 29 ++++++++++++++++++++
3 files changed, 45 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
index 647ac23..812a6b3 100644
--- a/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/RoleKeys.java
@@ -30,6 +30,11 @@ public interface RoleKeys {
String ROLE_NAME = "role.name";
/**
+ * The group of a role: {@value}
+ */
+ String ROLE_GROUP = "role.group";
+
+ /**
* Status report: number actually granted : {@value}
*/
String ROLE_ACTUAL_INSTANCES = "role.actual.instances";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index b4465a8..3213d93 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -42,7 +42,6 @@ import org.apache.slider.api.ClusterDescriptionOperations;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.ResourceKeys;
-import org.apache.slider.api.RoleKeys;
import org.apache.slider.api.StatusKeys;
import org.apache.slider.api.types.ApplicationLivenessInformation;
import org.apache.slider.api.types.ComponentInformation;
@@ -1346,7 +1345,7 @@ public class AppState {
* These are returned as integers, but there is special handling of the
* string {@link ResourceKeys#YARN_RESOURCE_MAX}, which triggers
* the return of the maximum value.
- * @param name component to get from
+ * @param group component to get from
* @param option option name
* @param defVal default value
* @param maxVal value to return if the max val is requested
@@ -1800,11 +1799,20 @@ public class AppState {
for (RoleStatus role : getRoleStatusMap().values()) {
String rolename = role.getName();
+ if (hasUniqueNames(instanceDefinition.getResourceOperations(),
+ role.getGroup())) {
+ cd.setRoleOpt(rolename, COMPONENT_PRIORITY, role.getPriority());
+ cd.setRoleOpt(rolename, ROLE_GROUP, role.getGroup());
+ MapOperations groupOptions = instanceDefinition.getResourceOperations()
+ .getComponent(role.getGroup());
+ SliderUtils.mergeMapsIgnoreDuplicateKeys(cd.getRole(rolename),
+ groupOptions.options);
+ }
List<String> instances = instanceMap.get(rolename);
int nodeCount = instances != null ? instances.size(): 0;
cd.setRoleOpt(rolename, COMPONENT_INSTANCES,
role.getDesired());
- cd.setRoleOpt(rolename, RoleKeys.ROLE_ACTUAL_INSTANCES, nodeCount);
+ cd.setRoleOpt(rolename, ROLE_ACTUAL_INSTANCES, nodeCount);
cd.setRoleOpt(rolename, ROLE_REQUESTED_INSTANCES, role.getRequested());
cd.setRoleOpt(rolename, ROLE_RELEASING_INSTANCES, role.getReleasing());
cd.setRoleOpt(rolename, ROLE_FAILED_INSTANCES, role.getFailed());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/454f2cf2/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
index d3ba798..84d6bc7 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockAppStateUniqueNames.groovy
@@ -22,6 +22,7 @@ import groovy.transform.CompileStatic
import groovy.util.logging.Slf4j
import org.apache.hadoop.fs.Path
import org.apache.slider.api.ResourceKeys
+import org.apache.slider.api.RoleKeys
import org.apache.slider.core.conf.AggregateConf
import org.apache.slider.core.conf.ConfTreeOperations
import org.apache.slider.core.exceptions.BadConfigException
@@ -86,6 +87,25 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
return new ConfTreeOperations(resources)
}
+ private static void checkRole(MockAppState appState,
+ String roleName,
+ String roleGroup,
+ Map<String, String> expectedOpts) {
+
+ for (String key : expectedOpts.keySet()) {
+ if (ResourceKeys.COMPONENT_PRIORITY.equals(key) ||
+ ResourceKeys.COMPONENT_INSTANCES.equals(key)) {
+ continue
+ }
+ assert expectedOpts.get(key).equals(appState.getClusterStatus()
+ .getMandatoryRoleOpt(roleName, key))
+ }
+ assert 1 == appState.getClusterStatus().getMandatoryRoleOptInt(
+ roleName, ResourceKeys.COMPONENT_INSTANCES)
+ assert roleGroup.equals(appState.getClusterStatus().getMandatoryRoleOpt(
+ roleName, RoleKeys.ROLE_GROUP))
+ }
+
@Test
public void testDynamicFlexAddRole() throws Throwable {
def cd = init()
@@ -113,6 +133,10 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 2 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
assert 4 == appState.lookupRoleStatus("group21").resourceRequirements.virtualCores
assert 4 == appState.lookupRoleStatus("group22").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group21", "group2", opts)
+ checkRole(appState, "group22", "group2", opts)
}
@Test
@@ -167,6 +191,11 @@ class TestMockAppStateUniqueNames extends BaseMockAppStateTest
assert 8 == appState.lookupRoleStatus("group11").resourceRequirements.virtualCores
assert 8 == appState.lookupRoleStatus("group12").resourceRequirements.virtualCores
assert 8 == appState.lookupRoleStatus("group13").resourceRequirements.virtualCores
+
+ appState.refreshClusterStatus()
+ checkRole(appState, "group11", "group1", opts)
+ checkRole(appState, "group12", "group1", opts)
+ checkRole(appState, "group13", "group1", opts)
}
}
[19/31] incubator-slider git commit: SLIDER-1133 Provide support to
accept metainfo as a JSON blob from the command line
Posted by el...@apache.org.
SLIDER-1133 Provide support to accept metainfo as a JSON blob from the command line
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/eccefc1e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/eccefc1e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/eccefc1e
Branch: refs/heads/branches/branch-0.91
Commit: eccefc1e407ffd90ae81612c713787bf712eab9e
Parents: 44f4a54
Author: Gour Saha <go...@apache.org>
Authored: Fri May 27 11:37:14 2016 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Fri May 27 11:37:14 2016 -0700
----------------------------------------------------------------------
.../AbstractClusterBuildingActionArgs.java | 6 ++-
.../apache/slider/common/params/Arguments.java | 1 +
.../core/persist/AppDefinitionPersister.java | 47 ++++++++++++++++----
.../agent/TestAppDefinitionPersister.java | 43 ++++++++++++++++--
4 files changed, 85 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eccefc1e/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 1c694bd..3833394 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -87,9 +87,13 @@ public abstract class AbstractClusterBuildingActionArgs extends
public File template;
@Parameter(names = {ARG_METAINFO},
- description = "Application meta info")
+ description = "Application meta info file")
public File appMetaInfo;
+ @Parameter(names = {ARG_METAINFO_JSON},
+ description = "Application meta info JSON blob")
+ public String appMetaInfoJson;
+
@Parameter(names = {ARG_APPDEF},
description = "Application def (folder or a zip package)")
public File appDef;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eccefc1e/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index bac20d7..0a8388d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -87,6 +87,7 @@ public interface Arguments {
String ARG_MANAGER_SHORT = "--m";
String ARG_MESSAGE = "--message";
String ARG_METAINFO = "--metainfo";
+ String ARG_METAINFO_JSON = "--metainfojson";
String ARG_NAME = "--name";
String ARG_OPTION = "--option";
String ARG_OPTION_SHORT = "-O";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eccefc1e/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
index 2448c72..8efaa5b 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.Arguments;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.conf.ConfTreeOperations;
@@ -35,6 +36,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -109,28 +111,57 @@ public class AppDefinitionPersister {
ConfTreeOperations appConf)
throws BadConfigException, IOException, BadCommandArgumentsException {
// if metainfo is provided add to the app instance
- if (buildInfo.appMetaInfo != null) {
-
- if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
- throw new BadConfigException("--metainfo file cannot be read.");
+ if (buildInfo.appMetaInfo != null || buildInfo.appMetaInfoJson != null) {
+ if (buildInfo.appMetaInfo != null && buildInfo.appMetaInfoJson != null) {
+ throw new BadConfigException("Both %s and %s cannot be specified",
+ Arguments.ARG_METAINFO, Arguments.ARG_METAINFO_JSON);
}
+ // Now we know that only one of either file or JSON is used
+ boolean isFileUsed = buildInfo.appMetaInfo != null ? true : false;
+ String argUsed = isFileUsed ? Arguments.ARG_METAINFO
+ : Arguments.ARG_METAINFO_JSON;
+
if (buildInfo.appDef != null) {
- throw new BadConfigException("both --metainfo and --appdef may not be specified.");
+ throw new BadConfigException("Both %s and %s cannot be specified",
+ argUsed, Arguments.ARG_APPDEF);
}
if (SliderUtils.isSet(appConf.getGlobalOptions().get(AgentKeys.APP_DEF))) {
- throw new BadConfigException("application.def must not be set if --metainfo is provided.");
+ throw new BadConfigException(
+ "%s cannot not be set if %s is specified in the cmd line ",
+ AgentKeys.APP_DEF, argUsed);
+ }
+
+ if (isFileUsed) {
+ if (!buildInfo.appMetaInfo.canRead() || !buildInfo.appMetaInfo.isFile()) {
+ throw new BadConfigException(
+ "Path specified with %s either cannot be read or is not a file",
+ Arguments.ARG_METAINFO);
+ }
+ } else {
+ if (StringUtils.isEmpty(buildInfo.appMetaInfoJson.trim())) {
+ throw new BadConfigException("Empty string specified with %s",
+ Arguments.ARG_METAINFO_JSON);
+ }
}
File tempDir = Files.createTempDir();
File pkgSrcDir = new File(tempDir, "default");
pkgSrcDir.mkdirs();
- Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json"));
+ File destMetaInfo = new File(pkgSrcDir, "metainfo.json");
+ if (isFileUsed) {
+ Files.copy(buildInfo.appMetaInfo, destMetaInfo);
+ } else {
+ Files.write(
+ buildInfo.appMetaInfoJson.getBytes(Charset.forName("UTF-8")),
+ destMetaInfo);
+ }
Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
log.info("Using default app def path {}", appDirPath.toString());
- appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, SliderKeys.DEFAULT_APP_PKG));
+ appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir,
+ SliderKeys.DEFAULT_APP_PKG));
Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
log.info("Setting app package to {}.", appDefPath);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/eccefc1e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
index eaf496c..dedf4f6 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAppDefinitionPersister.java
@@ -85,7 +85,9 @@ public class TestAppDefinitionPersister {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("--metainfo file cannot be read"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "Path specified with "
+ + "--metainfo either cannot be read or is not a file"));
}
try (PrintWriter writer = new PrintWriter(metainfo.getAbsolutePath(), "UTF-8")) {
@@ -98,18 +100,53 @@ public class TestAppDefinitionPersister {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("both --metainfo and --appdef may not be specified"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --appdef cannot be specified"));
+ }
+
+ // both --metainfojson and --appdef cannot be specified
+ buildInfo.appMetaInfo = null;
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfojson and --appdef cannot be specified"));
}
buildInfo.appDef = null;
+ buildInfo.appMetaInfoJson = "";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Empty string specified with --metainfojson"));
+ }
+ buildInfo.appMetaInfo = metainfo;
+
+ // both --metainfo and --metainfojson cannot be specified
+ buildInfo.appMetaInfoJson = "{}";
+ try {
+ adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
+ } catch (BadConfigException bce) {
+ log.info(bce.getMessage());
+ Assert.assertTrue(bce.getMessage().contains(
+ "Both --metainfo and --metainfojson cannot be specified"));
+ }
+ buildInfo.appMetaInfoJson = null;
+
appConf.getGlobalOptions().set(AgentKeys.APP_DEF, metainfo.getAbsolutePath());
try {
adp.processSuppliedDefinitions(clustername, buildInfo, appConf);
} catch (BadConfigException bce) {
log.info(bce.getMessage());
- Assert.assertTrue(bce.getMessage().contains("application.def must not be set if --metainfo is provided"));
+ Assert.assertTrue(bce.getMessage().contains(
+ "application.def cannot "
+ + "not be set if --metainfo is specified in the cmd line"));
}
appConf.getGlobalOptions().remove(AgentKeys.APP_DEF);