You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2014/08/06 12:50:41 UTC
[01/18] git commit: SLIDER-90: HA NN, RM support
Repository: incubator-slider
Updated Branches:
refs/heads/feature/SLIDER-284-hadoop2.5 248b6ddc3 -> ef22cb296
SLIDER-90: HA NN, RM support
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/0b52d4b0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/0b52d4b0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/0b52d4b0
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 0b52d4b0e2fc0a3a5e98c34bb6248a84e359c3db
Parents: 7ab9ce9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 12:04:41 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 12:04:41 2014 +0100
----------------------------------------------------------------------
.../java/org/apache/slider/client/SliderClient.java | 10 ++++------
.../slider/common/params/AbstractActionArgs.java | 14 +++++---------
.../java/org/apache/slider/common/params/ArgOps.java | 7 ++++---
.../org/apache/slider/common/params/ClientArgs.java | 5 +++--
.../org/apache/slider/common/params/CommonArgs.java | 11 +++++------
.../slider/server/appmaster/SliderAppMaster.java | 4 ++--
6 files changed, 23 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 40e01bb..ff2e865 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -182,7 +182,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
Configuration clientConf = SliderUtils.loadClientConfigurationResource();
ConfigHelper.mergeConfigurations(conf, clientConf, CLIENT_RESOURCE);
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
// init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
SliderUtils.forceLogin();
@@ -330,9 +330,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
client.createPath(zkPath, "", ZooDefs.Ids.OPEN_ACL_UNSAFE,
CreateMode.PERSISTENT);
return zkPath;
- } catch (InterruptedException e) {
- log.warn("Unable to create zk node {}", zkPath, e);
- } catch (KeeperException e) {
+ } catch (InterruptedException | KeeperException e) {
log.warn("Unable to create zk node {}", zkPath, e);
}
}
@@ -1008,8 +1006,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
commandLine.add(Arguments.ARG_RM_ADDR, rmAddr);
}
- if (serviceArgs.getFilesystemURL() != null) {
- commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemURL());
+ if (serviceArgs.getFilesystemBinding() != null) {
+ commandLine.add(Arguments.ARG_FILESYSTEM, serviceArgs.getFilesystemBinding());
}
addConfOptionToCLI(commandLine, config, REGISTRY_PATH,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
index 44bc239..3a8cea4 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractActionArgs.java
@@ -25,7 +25,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -41,26 +40,23 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
}
/**
- * URI of the filesystem
+ * URI/binding to the filesystem
*/
@Parameter(names = {ARG_FILESYSTEM, ARG_FILESYSTEM_LONG},
- description = "Filesystem URI",
- converter = URIArgumentConverter.class)
- public URI filesystemURL;
+ description = "Filesystem Binding")
+ public String filesystemBinding;
@Parameter(names = {ARG_BASE_PATH},
description = "Slider base path on the filesystem",
converter = PathArgumentConverter.class)
public Path basePath;
-
/**
* This is the default parameter
*/
@Parameter
public final List<String> parameters = new ArrayList<>();
-
/**
* get the name: relies on arg 1 being the cluster name in all operations
* @return the name argument, null if there is none
@@ -92,7 +88,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
@Parameter(names = {ARG_MANAGER_SHORT, ARG_MANAGER},
- description = "hostname:port of the YARN resource manager")
+ description = "Binding (usually hostname:port) of the YARN resource manager")
public String manager;
@@ -110,7 +106,7 @@ public abstract class AbstractActionArgs extends ArgOps implements Arguments {
/**
* Get the name of the action
- * @return
+ * @return the action name
*/
public abstract String getActionName() ;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
index 0837dd2..b53ebd3 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ArgOps.java
@@ -58,11 +58,12 @@ public class ArgOps {
return triple(msg, min, min);
}
- public static void applyFileSystemURL(URI filesystemURL, Configuration conf) {
- if (filesystemURL != null) {
+ public static void applyFileSystemBinding(String filesystemBinding,
+ Configuration conf) {
+ if (filesystemBinding != null) {
//filesystem argument was set -this overwrites any defaults in the
//configuration
- FileSystem.setDefaultUri(conf, filesystemURL);
+ FileSystem.setDefaultUri(conf, filesystemBinding);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index 6e44402..ca854f1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -103,9 +103,10 @@ public class ClientArgs extends CommonArgs {
log.debug("Setting RM to {}", getManager());
conf.set(YarnConfiguration.RM_ADDRESS, getManager());
}
- if ( getBasePath() != null ) {
+ if (getBasePath() != null) {
log.debug("Setting basePath to {}", getBasePath());
- conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH, getBasePath().toString());
+ conf.set(SliderXmlConfKeys.KEY_SLIDER_BASE_PATH,
+ getBasePath().toString());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
index 4cee1d1..83e658d 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/CommonArgs.java
@@ -30,7 +30,6 @@ import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.net.URI;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
@@ -215,12 +214,12 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
/**
- * If the Filesystem URL was provided, it overrides anything in
+ * If the Filesystem binding was provided, it overrides anything in
* the configuration
* @param conf configuration
*/
- public void applyFileSystemURL(Configuration conf) {
- ArgOps.applyFileSystemURL(getFilesystemURL(), conf);
+ public void applyFileSystemBinding(Configuration conf) {
+ ArgOps.applyFileSystemBinding(getFilesystemBinding(), conf);
}
public boolean isDebug() {
@@ -228,8 +227,8 @@ public abstract class CommonArgs extends ArgOps implements SliderActions,
}
- public URI getFilesystemURL() {
- return coreAction.filesystemURL;
+ public String getFilesystemBinding() {
+ return coreAction.filesystemBinding;
}
public Path getBasePath() { return coreAction.basePath; }
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/0b52d4b0/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 3b061d5..7e0ae5e 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -340,7 +340,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderAMCreateAction createAction = (SliderAMCreateAction) action;
//sort out the location of the AM
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
String rmAddress = createAction.getRmAddress();
if (rmAddress != null) {
@@ -348,7 +348,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
SliderUtils.setRmSchedulerAddress(conf, rmAddress);
}
serviceArgs.applyDefinitions(conf);
- serviceArgs.applyFileSystemURL(conf);
+ serviceArgs.applyFileSystemBinding(conf);
//init security with our conf
if (SliderUtils.isHadoopClusterSecure(conf)) {
log.info("Secure mode with kerberos realm {}",
[08/18] git commit: SLIDER-270 more work on the double flex test case
Posted by st...@apache.org.
SLIDER-270 more work on the double flex test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/291a82ef
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/291a82ef
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/291a82ef
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 291a82ef1e99a6590e245ebce7c0d1d861eac263
Parents: 040ef79
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:09:00 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:09:00 2014 +0100
----------------------------------------------------------------------
.../org/apache/slider/test/YarnMiniClusterTestBase.groovy | 7 ++++---
.../hbase/minicluster/HBaseMiniClusterTestBase.groovy | 1 +
.../minicluster/flexing/TestClusterFlexDownMultiple.groovy | 4 ++--
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
index 49dc02e..ed66190 100644
--- a/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/test/YarnMiniClusterTestBase.groovy
@@ -136,6 +136,9 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
}
}
+ protected String buildClustername(String clustername) {
+ return clustername ?: createClusterName()
+ }
/**
* Create the cluster name from the method name and an auto-incrementing
@@ -240,9 +243,7 @@ public abstract class YarnMiniClusterTestBase extends ServiceLauncherBaseTest {
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 64);
conf.set(YarnConfiguration.RM_SCHEDULER, FIFO_SCHEDULER);
SliderUtils.patchConfiguration(conf)
- if (!name) {
- name = createClusterName()
- }
+ name = buildClustername(name)
miniCluster = new MiniYARNCluster(name, noOfNodeManagers, numLocalDirs, numLogDirs)
miniCluster.init(conf)
miniCluster.start();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index c9f0a4f..6a69e17 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -284,6 +284,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int workers,
int workerFlexTarget,
boolean testHBaseAfter) {
+ clustername = buildClustername(clustername);
SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
//now flex
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/291a82ef/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
index 43ade73..97a9b35 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -34,13 +34,13 @@ class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
@Test
public void testClusterFlexDownMultiple() throws Throwable {
def clusterName = createClusterName();
- SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+ SliderClient sliderClient = startHBaseCluster(clusterName, 1, 3)
assert flexCluster(
sliderClient,
clusterName,
1,
- 3,
+ 2,
true)
assert flexCluster(
[15/18] git commit: SLIDER-219. Multiple dynamic ports may be asked
as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/1894a0ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/1894a0ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/1894a0ca
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 1894a0ca206d4d024f7b3062133ab1039ca5988b
Parents: a9a91b4
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Jul 29 23:33:00 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached/appConfig.json | 3 +-
app-packages/memcached/metainfo.xml | 6 ++
.../python/agent/CustomServiceOrchestrator.py | 57 ++++++++--
.../agent/TestCustomServiceOrchestrator.py | 107 +++++++++++++++++--
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../src/test/python/agent/TestRegistration.py | 4 +-
.../providers/agent/AgentProviderService.java | 3 +
.../agent/application/metadata/Component.java | 13 +++
.../application/metadata/MetainfoParser.java | 4 +
.../agent/TestAgentProviderService.java | 25 ++++-
11 files changed, 202 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index d205591..11fb863 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -10,7 +10,8 @@
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
- "site.global.memory_val": "200M"
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index d435b84..0550b7d 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -27,6 +27,12 @@
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${THIS_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 4eb22bf..7d740c2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -52,9 +52,9 @@ class CustomServiceOrchestrator():
self.tmp_dir = config.getResolvedPath(AgentConfig.APP_TASK_DIR)
self.python_executor = PythonExecutor(self.tmp_dir, config)
self.status_commands_stdout = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stdout.txt'))
+ 'status_command_stdout.txt'))
self.status_commands_stderr = os.path.realpath(posixpath.join(self.tmp_dir,
- 'status_command_stderr.txt'))
+ 'status_command_stderr.txt'))
self.public_fqdn = hostname.public_hostname()
self.applied_configs = {}
# Clean up old status command files if any
@@ -81,7 +81,7 @@ class CustomServiceOrchestrator():
script_tuple = (script_path, self.base_dir)
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
- "structured-out-{0}.json".format(task_id)))
+ "structured-out-{0}.json".format(task_id)))
if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
# We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
@@ -97,9 +97,9 @@ class CustomServiceOrchestrator():
for py_file, current_base_dir in filtered_py_file_list:
script_params = [command_name, json_path, current_base_dir]
python_paths = [os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
- "infra", "agent", "slider-agent", "jinja2")),
+ "infra", "agent", "slider-agent", "jinja2")),
os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
- "infra", "agent", "slider-agent"))]
+ "infra", "agent", "slider-agent"))]
if platform.system() != "Windows":
environment_vars = [("PYTHONPATH", ":".join(python_paths))]
else:
@@ -137,8 +137,8 @@ class CustomServiceOrchestrator():
# Irrespective of the outcome report the folder paths
if command_name == 'INSTALL':
ret[Constants.FOLDERS] = {
- Constants.AGENT_LOG_ROOT : self.config.getLogPath(),
- Constants.AGENT_WORK_ROOT : self.config.getWorkRootPath()
+ Constants.AGENT_LOG_ROOT: self.config.getLogPath(),
+ Constants.AGENT_WORK_ROOT: self.config.getWorkRootPath()
}
return ret
@@ -249,9 +249,7 @@ class CustomServiceOrchestrator():
value = value.replace("${AGENT_LOG_ROOT}",
self.config.getLogPath())
if port_allocation_req in value:
- port = self.allocate_port()
- value = value.replace(port_allocation_req, str(port))
- logger.info("Allocated port " + str(port) + " for " + port_allocation_req)
+ value = self.allocate_ports(value, port_allocation_req)
allocated_ports[k] = value
command['configurations'][key][k] = value
pass
@@ -266,7 +264,34 @@ class CustomServiceOrchestrator():
pass
- def allocate_port(self):
+ def allocate_ports(self, value, port_req_pattern):
+ default_port_pattern = "{DEFAULT_"
+ index = value.find(port_req_pattern)
+ while index != -1:
+ replaced_pattern = port_req_pattern
+ def_port = None
+ if index == value.find(port_req_pattern + default_port_pattern):
+ replaced_pattern = port_req_pattern + default_port_pattern
+ start_index = index + len(replaced_pattern)
+ end_index = value.find("}", start_index)
+ def_port_str = value[start_index:end_index]
+ def_port = int(def_port_str)
+ replaced_pattern = replaced_pattern + def_port_str + "}"
+ pass
+ port = self.allocate_port(def_port)
+ value = value.replace(replaced_pattern, str(port), 1)
+ logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
+ index = value.find(port_req_pattern)
+ pass
+ return value
+ pass
+
+
+ def allocate_port(self, default_port=None):
+ if default_port != None:
+ if self.is_port_available(default_port):
+ return default_port
+
MAX_ATTEMPT = 5
iter = 0
port = -1
@@ -284,4 +309,14 @@ class CustomServiceOrchestrator():
logger.info("Allocated dynamic port: " + str(port))
return port
+ def is_port_available(self, port):
+ try:
+ sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ sock.settimeout(0.2)
+ sock.connect(('127.0.0.1', port))
+ sock.close()
+ except:
+ return True
+ return False
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index d2439b1..3b41bd7 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -34,6 +34,7 @@ from CustomServiceOrchestrator import CustomServiceOrchestrator
from mock.mock import MagicMock, patch
import StringIO
import sys
+from socket import socket
class TestCustomServiceOrchestrator(TestCase):
@@ -178,13 +179,13 @@ class TestCustomServiceOrchestrator(TestCase):
pass
- @patch.object(CustomServiceOrchestrator, "allocate_port")
+ @patch.object(CustomServiceOrchestrator, "allocate_ports")
@patch.object(CustomServiceOrchestrator, "resolve_script_path")
@patch.object(PythonExecutor, "run_file")
def test_runCommand_get_port(self,
run_file_mock,
resolve_script_path_mock,
- allocate_port_mock):
+ allocate_ports_mock):
command = {
'role': 'HBASE_REGIONSERVER',
'hostLevelParams': {
@@ -212,7 +213,7 @@ class TestCustomServiceOrchestrator(TestCase):
config.getWorkRootPath.return_value = tempdir
config.getLogPath.return_value = tempdir
- allocate_port_mock.return_value = 10233
+ allocate_ports_mock.return_value = str(10233)
resolve_script_path_mock.return_value = "/basedir/scriptpath"
dummy_controller = MagicMock()
@@ -230,6 +231,100 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(run_file_mock.call_count, 1)
+ @patch.object(socket, "close")
+ @patch.object(socket, "connect")
+ def test_allocate_port_def(self, socket_connect_mock, socket_close_mock):
+ e = OSError()
+ socket_connect_mock.side_effect = e
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ ret = orchestrator.allocate_port(10)
+ self.assertEqual(ret, 10)
+
+ @patch.object(socket, "getsockname")
+ @patch.object(socket, "bind")
+ @patch.object(socket, "close")
+ @patch.object(socket, "connect")
+ def test_allocate_port_new(self, socket_connect_mock, socket_close_mock,
+ socket_bind_mock, socket_getsockname_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ socket_getsockname_mock.return_value = [100, 101]
+ ret = orchestrator.allocate_port(10)
+ self.assertEqual(ret, 101)
+
+ @patch.object(socket, "getsockname")
+ @patch.object(socket, "bind")
+ def test_allocate_port_no_def(self, socket_bind_mock, socket_getsockname_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+ socket_getsockname_mock.return_value = [100, 102]
+ ret = orchestrator.allocate_port()
+ self.assertEqual(ret, 102)
+
+
+ @patch.object(CustomServiceOrchestrator, "allocate_port")
+ def test_allocate_port_combinations(self, allocate_port_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+
+ allocate_port_mock.side_effect = [101, 102, 103]
+ ret = orchestrator.allocate_ports("1000", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1000")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "101")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT},${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "102,103")
+
+
+ @patch.object(CustomServiceOrchestrator, "is_port_available")
+ def test_allocate_port_combinations2(self, is_port_available_mock):
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller)
+
+ is_port_available_mock.return_value = True
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1005")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}-${A.ALLOCATED_PORT}{DEFAULT_1006}",
+ "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1005-1006")
+
+
@patch("hostname.public_hostname")
@patch("os.path.isfile")
@patch("os.unlink")
@@ -353,8 +448,8 @@ class TestCustomServiceOrchestrator(TestCase):
status = orchestrator.requestComponentStatus(status_command)
self.assertEqual(CustomServiceOrchestrator.DEAD_STATUS, status['exitcode'])
- @patch.object(CustomServiceOrchestrator, "allocate_port")
- def test_finalize_command(self, mock_allocate_port):
+ @patch.object(CustomServiceOrchestrator, "allocate_ports")
+ def test_finalize_command(self, mock_allocate_ports):
dummy_controller = MagicMock()
tempdir = tempfile.gettempdir()
tempWorkDir = tempdir + "W"
@@ -363,7 +458,7 @@ class TestCustomServiceOrchestrator(TestCase):
config.getResolvedPath.return_value = tempdir
config.getWorkRootPath.return_value = tempWorkDir
config.getLogPath.return_value = tempdir
- mock_allocate_port.return_value = "10023"
+ mock_allocate_ports.return_value = "10023"
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
command = {}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestGrep.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestGrep.py b/slider-agent/src/test/python/agent/TestGrep.py
index 75f0093..351befb 100644
--- a/slider-agent/src/test/python/agent/TestGrep.py
+++ b/slider-agent/src/test/python/agent/TestGrep.py
@@ -19,7 +19,7 @@ limitations under the License.
'''
from unittest import TestCase
-from agent.Grep import Grep
+from Grep import Grep
import socket
import os, sys
import logging
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index dc3a199..bc8d2a3 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -26,11 +26,11 @@ import unittest
from agent import ProcessHelper, main
import logging
import signal
-from agent.AgentConfig import AgentConfig
+from AgentConfig import AgentConfig
import ConfigParser
import os
import tempfile
-from agent.Controller import Controller
+from Controller import Controller
from optparse import OptionParser
logger = logging.getLogger()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-agent/src/test/python/agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 8647d0d..5245af9 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -25,8 +25,8 @@ import errno
import tempfile
from mock.mock import patch
from mock.mock import MagicMock
-from agent.Register import Register
-from agent.AgentConfig import AgentConfig
+from Register import Register
+from AgentConfig import AgentConfig
class TestRegistration(TestCase):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index a73aaab..f878b36 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -109,6 +109,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
+ private static final String COMPONENT_DATA_TAG = "ComponentData";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -126,6 +127,7 @@ public class AgentProviderService extends AbstractProviderService implements
return size() > MAX_LOG_ENTRIES;
}
});
+ private Map<String, Map<String, String>> componentData = new ConcurrentHashMap();
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@ -567,6 +569,7 @@ public class AgentProviderService extends AbstractProviderService implements
publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
}
+
/**
* Process return status for component instances
* @param heartBeat
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 6cd08e0..9dba87c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -16,6 +16,9 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import java.util.ArrayList;
+import java.util.List;
+
/**
*
*/
@@ -26,9 +29,11 @@ public class Component {
String minInstanceCount;
String maxInstanceCount;
CommandScript commandScript;
+ List<Export> exports;
public Component() {
publishConfig = Boolean.FALSE.toString();
+ exports = new ArrayList<>();
}
public String getName() {
@@ -79,6 +84,14 @@ public class Component {
this.commandScript = commandScript;
}
+ public void addExport(Export export) {
+ exports.add(export);
+ }
+
+ public List<Export> getExports() {
+ return exports;
+ }
+
@Override
public String toString() {
final StringBuilder sb =
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index c7922a7..d2801d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -58,6 +58,10 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/component/publishConfig");
digester.addBeanPropertySetter("*/component/minInstanceCount");
digester.addBeanPropertySetter("*/component/maxInstanceCount");
+ digester.addObjectCreate("*/component/export", Export.class);
+ digester.addBeanPropertySetter("*/component/export/name");
+ digester.addBeanPropertySetter("*/component/export/value");
+ digester.addSetNext("*/component/export", "addExport");
digester.addSetNext("*/component", "addComponent");
digester.addObjectCreate("*/commandScript", CommandScript.class);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/1894a0ca/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 0a42e94..c34d0ea 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -151,6 +151,16 @@ public class TestAgentProviderService {
+ " <script>scripts/hbase_regionserver.py</script>\n"
+ " <scriptType>PYTHON</scriptType>\n"
+ " </commandScript>\n"
+ + " <exports>\n"
+ + " <export>\n"
+ + " <name>PropertyA</name>\n"
+ + " <value>${THIS_HOST}:${site.global.listen_port}</value>\n"
+ + " </export>\n"
+ + " <export>\n"
+ + " <name>PropertyB</name>\n"
+ + " <value>AConstant</value>\n"
+ + " </export>\n"
+ + " </exports>\n"
+ " </component>\n"
+ " </components>\n"
+ " <osSpecifics>\n"
@@ -414,6 +424,7 @@ public class TestAgentProviderService {
Assert.assertEquals(component.getMaxInstanceCount(), "2");
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
Assert.assertEquals(component.getCategory(), "MASTER");
+ Assert.assertEquals(component.getExports().size(), 0);
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
@@ -421,16 +432,24 @@ public class TestAgentProviderService {
Assert.assertNull(component.getMaxInstanceCount());
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
Assert.assertEquals(component.getCategory(), "SLAVE");
+ Assert.assertEquals(component.getExports().size(), 2);
+ List<Export> es = component.getExports();
+ Export e = es.get(0);
+ Assert.assertEquals(e.getName(), "PropertyA");
+ Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}");
+ e = es.get(1);
+ Assert.assertEquals(e.getName(), "PropertyB");
+ Assert.assertEquals(e.getValue(), "AConstant");
found++;
}
}
Assert.assertEquals(found, 2);
- assert application.getExportGroups().size() == 1;
+ Assert.assertEquals(application.getExportGroups().size(), 1);
List<ExportGroup> egs = application.getExportGroups();
ExportGroup eg = egs.get(0);
- assert eg.getName().equals("QuickLinks");
- assert eg.getExports().size() == 2;
+ Assert.assertEquals(eg.getName(), "QuickLinks");
+ Assert.assertEquals(eg.getExports().size(), 2);
found = 0;
for (Export export : eg.getExports()) {
[12/18] git commit: SLIDER-271. When a container is releases the
AppMaster should not monitor heartbeats from that container SLIDER-293. On
Windows slider agent should be able to execute commands and not wait for
completion SLIDER-294. Clean up memcached
Posted by st...@apache.org.
SLIDER-271. When a container is releases the AppMaster should not monitor heartbeats from that container
SLIDER-293. On Windows slider agent should be able to execute commands and not wait for completion
SLIDER-294. Clean up memcached app packages for Windows and Linux
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/405aa51e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/405aa51e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/405aa51e
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 405aa51e203f611a385a0e923fe68eb0d7d86fc0
Parents: 640233d
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Aug 5 18:31:16 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/appConfig.json | 6 +-
app-packages/memcached-win/metainfo.xml | 7 +-
.../memcached-win/package/scripts/memcached.py | 2 +-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 6 +-
app-packages/memcached/metainfo.xml | 1 -
.../core/providers/windows/system.py | 54 ++++++++++++++-
.../providers/agent/AgentProviderService.java | 71 +++++++++++---------
.../agent/TestAgentProviderService.java | 17 ++++-
9 files changed, 121 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/appConfig.json b/app-packages/memcached-win/appConfig.json
index 8c00660..63835ec 100644
--- a/app-packages/memcached-win/appConfig.json
+++ b/app-packages/memcached-win/appConfig.json
@@ -5,12 +5,16 @@
"global": {
"application.def": "/slider/jmemcached-1.0.0.zip",
"java_home": "C:\\java",
+
"site.global.app_user": "hadoop",
"site.global.app_root": "${AGENT_WORK_ROOT}\\app\\install",
+
"site.global.additional_cp": "C:\\hdp\\hadoop-2.4.0.2.1.3.0-1990\\share\\hadoop\\common\\lib\\*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
- "site.global.memory_val": "200M"
+ "site.global.memory_val": "200M",
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
+
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
index dc2d67b..d056c0a 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -28,10 +28,15 @@
<component>
<name>MEMCACHED</name>
<category>MASTER</category>
+ <exports>
+ <export>
+ <name>host_port</name>
+ <value>${THIS_HOST}:${site.global.listen_port}</value>
+ </export>
+ </exports>
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
</commandScript>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/memcached.py b/app-packages/memcached-win/package/scripts/memcached.py
index 222a62c..1bb05dc 100644
--- a/app-packages/memcached-win/package/scripts/memcached.py
+++ b/app-packages/memcached-win/package/scripts/memcached.py
@@ -34,7 +34,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
- process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val}")
+ process_cmd = format("{java64_home}\\bin\\java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}\\*;{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached-win/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/package/scripts/params.py b/app-packages/memcached-win/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached-win/package/scripts/params.py
+++ b/app-packages/memcached-win/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index 78cb486..b179723 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -3,15 +3,17 @@
"metadata": {
},
"global": {
- "application.def": "/slider/jmemcached-1.0.0.zip",
+ "application.def": "package/jmemcached-1.0.0.zip",
"java_home": "/usr/jdk64/jdk1.7.0_45",
+
"site.global.app_user": "yarn",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/jmemcached-1.0.0",
+
"site.global.additional_cp": "/usr/lib/hadoop/lib/*",
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 08a504e..525816e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -37,7 +37,6 @@
<commandScript>
<script>scripts/memcached.py</script>
<scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
</commandScript>
</component>
</components>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
index 1e8142e..fa29d7d 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/windows/system.py
@@ -23,15 +23,26 @@ from resource_management.core.providers import Provider
from resource_management.core.logger import Logger
from resource_management.core.base import Fail
from resource_management.core import ExecuteTimeoutException
+from multiprocessing import Queue
import time
import os
import subprocess
+import shutil
def _call_command(command, logoutput=False, cwd=None, env=None, wait_for_finish=True, timeout=None):
# TODO implement timeout, logoutput, wait_for_finish
+ Logger.info("Executing %s" % (command))
proc = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
cwd=cwd, env=env, shell=False)
+ if not wait_for_finish:
+ return None, None
+
+ if timeout:
+ q = Queue()
+ t = threading.Timer( timeout, on_timeout, [proc, q] )
+ t.start()
+
out = proc.communicate()[0].strip()
code = proc.returncode
if logoutput and out:
@@ -83,7 +94,8 @@ class FileProvider(Provider):
if content:
fp.write(content)
- _set_file_acl(self.resource.path, self.resource.owner, self.resource.mode)
+ if self.resource.owner and self.resource.mode:
+ _set_file_acl(self.resource.path, self.resource.owner, self.resource.mode)
def action_delete(self):
path = self.resource.path
@@ -133,10 +145,46 @@ class ExecuteProvider(Provider):
time.sleep(self.resource.try_sleep)
except ExecuteTimeoutException:
err_msg = ("Execution of '%s' was killed due timeout after %d seconds") % (
- self.resource.command, self.resource.timeout)
+ self.resource.command, self.resource.timeout)
if self.resource.on_timeout:
Logger.info("Executing '%s'. Reason: %s" % (self.resource.on_timeout, err_msg))
_call_command(self.resource.on_timeout)
else:
- raise Fail(err_msg)
\ No newline at end of file
+ raise Fail(err_msg)
+
+
+class DirectoryProvider(Provider):
+ def action_create(self):
+ path = DirectoryProvider._trim_uri(self.resource.path)
+ if not os.path.exists(path):
+ Logger.info("Creating directory %s" % self.resource)
+ if self.resource.recursive:
+ os.makedirs(path)
+ else:
+ dirname = os.path.dirname(path)
+ if not os.path.isdir(dirname):
+ raise Fail("Applying %s failed, parent directory %s doesn't exist" % (self.resource, dirname))
+
+ os.mkdir(path)
+
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, file %s already exists" % (self.resource, path))
+
+ if self.resource.owner and self.resource.mode:
+ _set_file_acl(path, self.resource.owner, self.resource.mode)
+
+ def action_delete(self):
+ path = self.resource.path
+ if os.path.exists(path):
+ if not os.path.isdir(path):
+ raise Fail("Applying %s failed, %s is not a directory" % (self.resource, path))
+
+ Logger.info("Removing directory %s and all its content" % self.resource)
+ shutil.rmtree(path)
+
+ @staticmethod
+ def _trim_uri(file_uri):
+ if file_uri.startswith("file:///"):
+ return file_uri[8:]
+ return file_uri
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 682efba..6cfadf3 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -116,24 +116,26 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
+
private final Object syncLock = new Object();
- private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
- private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
private AtomicInteger taskId = new AtomicInteger(0);
private volatile Metainfo metainfo = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
+ private Boolean canAnyMasterPublish = null;
+ private AgentLaunchParameter agentLaunchParameter = null;
+
+ private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
+ private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
+ private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private Map<String, String> workFolders =
Collections.synchronizedMap(new LinkedHashMap<String, String>(MAX_LOG_ENTRIES, 0.75f, false) {
protected boolean removeEldestEntry(Map.Entry eldest) {
return size() > MAX_LOG_ENTRIES;
}
});
- private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
- private Boolean canAnyMasterPublish = null;
- private AgentLaunchParameter agentLaunchParameter = null;
/**
* Create an instance of AgentProviderService
@@ -286,11 +288,11 @@ public class AgentProviderService extends AbstractProviderService implements
launcher.addCommand(operation.build());
// initialize the component instance state
- componentStatuses.put(label,
- new ComponentInstanceState(
- role,
- container.getId().toString(),
- getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
+ getComponentStatuses().put(label,
+ new ComponentInstanceState(
+ role,
+ container.getId().toString(),
+ getClusterInfoPropertyValue(OptionKeys.APPLICATION_NAME)));
}
/**
@@ -329,12 +331,13 @@ public class AgentProviderService extends AbstractProviderService implements
public RegistrationResponse handleRegistration(Register registration) {
RegistrationResponse response = new RegistrationResponse();
String label = registration.getHostname();
- if (componentStatuses.containsKey(label)) {
+ if (getComponentStatuses().containsKey(label)) {
response.setResponseStatus(RegistrationStatus.OK);
- componentStatuses.get(label).setLastHeartbeat(System.currentTimeMillis());
+ getComponentStatuses().get(label).setLastHeartbeat(System.currentTimeMillis());
} else {
response.setResponseStatus(RegistrationStatus.FAILED);
response.setLog("Label not recognized.");
+ log.warn("Received registration request from unknown label {}", label);
}
return response;
}
@@ -362,12 +365,12 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
- if (!componentStatuses.containsKey(label)) {
+ if (!getComponentStatuses().containsKey(label)) {
return response;
}
Boolean isMaster = isMaster(roleName);
- ComponentInstanceState componentStatus = componentStatuses.get(label);
+ ComponentInstanceState componentStatus = getComponentStatuses().get(label);
componentStatus.setLastHeartbeat(System.currentTimeMillis());
// If no Master can explicitly publish then publish if its a master
// Otherwise, wait till the master that can publish is ready
@@ -405,7 +408,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (id < waitForCount) {
log.info("Waiting until heartbeat count {}. Current val: {}", waitForCount, id);
- componentStatuses.put(roleName, componentStatus);
+ getComponentStatuses().put(roleName, componentStatus);
return response;
}
@@ -418,7 +421,7 @@ public class AgentProviderService extends AbstractProviderService implements
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
- boolean canExecute = commandOrder.canExecute(roleName, command, componentStatuses.values());
+ boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
addStartCommand(roleName, containerId, response, scriptPath);
@@ -474,17 +477,21 @@ public class AgentProviderService extends AbstractProviderService implements
public void notifyContainerCompleted(ContainerId containerId) {
if (containerId != null) {
String containerIdStr = containerId.toString();
- if (this.componentInstanceData.containsKey(containerIdStr)) {
- synchronized (this.componentInstanceData) {
- this.componentInstanceData.remove(containerIdStr);
- }
+ if (getComponentInstanceData().containsKey(containerIdStr)) {
+ getComponentInstanceData().remove(containerIdStr);
log.info("Removing container specific data for {}", containerIdStr);
publishComponentInstanceData();
}
if (this.allocatedPorts.containsKey(containerIdStr)) {
- synchronized (this.allocatedPorts) {
- this.allocatedPorts.remove(containerIdStr);
+ this.allocatedPorts.remove(containerIdStr);
+ }
+
+ synchronized (this.componentStatuses) {
+ for (String label : getComponentStatuses().keySet()) {
+ if (label.startsWith(containerIdStr)) {
+ getComponentStatuses().remove(label);
+ }
}
}
}
@@ -589,7 +596,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return if release is requested successfully
*/
protected boolean releaseContainer(String label) {
- componentStatuses.remove(label);
+ getComponentStatuses().remove(label);
try {
getAppMaster().refreshContainer(getContainerId(label), true);
} catch (SliderException e) {
@@ -640,7 +647,7 @@ public class AgentProviderService extends AbstractProviderService implements
boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
Set<String> exportedConfigs = new HashSet();
String exportedConfigsStr = application.getExportedConfigs();
- boolean exportedConfigSpecified = exportedConfigsStr != null;
+ boolean exportedAllConfigs = exportedConfigsStr == null || exportedConfigsStr.isEmpty();
if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
for (String exportedConfig : application.getExportedConfigs().split(",")) {
if (exportedConfig.trim().length() > 0) {
@@ -653,8 +660,8 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
- if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
- !exportedConfigSpecified) {
+ if ((!exportedAllConfigs && exportedConfigs.contains(key)) ||
+ exportedAllConfigs) {
Map<String, String> configs = status.getConfigs().get(key);
publishApplicationInstanceData(key, key, configs.entrySet());
}
@@ -745,13 +752,13 @@ public class AgentProviderService extends AbstractProviderService implements
if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
- if (!this.componentInstanceData.containsKey(containerId)) {
+ if (!getComponentInstanceData().containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<>();
} else {
- perContainerData = this.componentInstanceData.get(containerId);
+ perContainerData = getComponentInstanceData().get(containerId);
}
perContainerData.putAll(toPublish);
- this.componentInstanceData.put(containerId, perContainerData);
+ getComponentInstanceData().put(containerId, perContainerData);
publishComponentInstanceData();
}
}
@@ -759,10 +766,10 @@ public class AgentProviderService extends AbstractProviderService implements
private void publishComponentInstanceData() {
Map<String, String> dataToPublish = new HashMap<>();
synchronized (this.componentInstanceData) {
- for (String container : this.componentInstanceData.keySet()) {
- for (String prop : this.componentInstanceData.get(container).keySet()) {
+ for (String container : getComponentInstanceData().keySet()) {
+ for (String prop : getComponentInstanceData().get(container).keySet()) {
dataToPublish.put(
- container + "." + prop, this.componentInstanceData.get(container).get(prop));
+ container + "." + prop, getComponentInstanceData().get(container).get(prop));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/405aa51e/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 4b2e253..922de68 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -820,20 +820,31 @@ public class TestAgentProviderService {
mockAps.getComponentInstanceData().put("cid2", new HashMap<String, String>());
mockAps.getComponentInstanceData().put(id, new HashMap<String, String>());
+ mockAps.getComponentStatuses().put("cid2_HM", new ComponentInstanceState("HM", "cid2", "aid"));
+ mockAps.getComponentStatuses().put(id + "_HM", new ComponentInstanceState("HM", id, "aid"));
+
Assert.assertNotNull(mockAps.getComponentInstanceData().get(id));
Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get(id + "_HM"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get("cid2_HM"));
+
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
- mockAps.notifyContainerCompleted(new MyContainerId(1));
- Assert.assertNull(mockAps.getComponentInstanceData().get(id));
- Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+ // Make the call
+ mockAps.notifyContainerCompleted(new MyContainerId(1));
Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+
+ Assert.assertNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertNull(mockAps.getComponentStatuses().get(id + "_HM"));
+ Assert.assertNotNull(mockAps.getComponentStatuses().get("cid2_HM"));
}
@Test
[03/18] git commit: SLIDER-283 Add role for HBase THRIFT gateway
Posted by st...@apache.org.
SLIDER-283 Add role for HBase THRIFT gateway
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/a9a91b46
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/a9a91b46
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/a9a91b46
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: a9a91b46662ecf969cbc7c5e69cef214fc91e086
Parents: 0b52d4b
Author: tedyu <yu...@gmail.com>
Authored: Tue Aug 5 09:32:53 2014 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue Aug 5 09:32:53 2014 -0700
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/metainfo.xml | 15 +++++
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift.py | 62 ++++++++++++++++++++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 5 ++
.../providers/hbase/HBaseClientProvider.java | 2 +-
.../slider/providers/hbase/HBaseKeys.java | 5 ++
.../providers/hbase/HBaseProviderService.java | 5 ++
.../slider/providers/hbase/HBaseRoles.java | 3 +
10 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 8b0825d..f0ab144 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -23,7 +23,7 @@
"site.global.ganglia_server_host": "${NN_HOST}",
"site.global.ganglia_server_port": "8667",
"site.global.ganglia_server_id": "Application1",
- "site.global.hbase_rest_port": "${HBASE_REST.ALLOCATED_PORT}",
+ "site.global.hbase_thrift_port": "${HBASE_THRIFT.ALLOCATED_PORT}",
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/hbase/metainfo.xml b/app-packages/hbase/metainfo.xml
index c14b06f..ee6ddea 100644
--- a/app-packages/hbase/metainfo.xml
+++ b/app-packages/hbase/metainfo.xml
@@ -45,6 +45,10 @@
<value>http://${HBASE_REST_HOST}:${site.global.hbase_rest_port}</value>
</export>
<export>
+ <name>org.apache.slider.hbase.thrift</name>
+ <value>http://${HBASE_THRIFT_HOST}:${site.global.hbase_thrift_port}</value>
+ </export>
+ <export>
<name>org.apache.slider.metrics</name>
<value>http://${site.global.ganglia_server_host}/cgi-bin/rrd.py?c=${site.global.ganglia_server_id}</value>
</export>
@@ -63,6 +67,7 @@
<commandOrder>
<command>HBASE_MASTER-START</command>
<requires>HBASE_REST-INSTALLED</requires>
+ <requires>HBASE_THRIFT-INSTALLED</requires>
</commandOrder>
</commandOrders>
<components>
@@ -99,6 +104,16 @@
</component>
<component>
+ <name>HBASE_THRIFT</name>
+ <category>SLAVE</category>
+ <minInstanceCount>1</minInstanceCount>
+ <commandScript>
+ <script>scripts/hbase_thrift.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ </component>
+
+ <component>
<name>HBASE_CLIENT</name>
<category>CLIENT</category>
<minInstanceCount>0</minInstanceCount>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_service.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_service.py b/app-packages/hbase/package/scripts/hbase_service.py
index cb9c599..01d38b3 100644
--- a/app-packages/hbase/package/scripts/hbase_service.py
+++ b/app-packages/hbase/package/scripts/hbase_service.py
@@ -37,6 +37,8 @@ def hbase_service(
daemon_cmd = format("{cmd} start {role}")
if name == 'rest':
daemon_cmd = format("{daemon_cmd} -p {rest_port}")
+ elif name == 'thrift':
+ daemon_cmd = format("{daemon_cmd} -p {thrift_port}")
no_op_test = format("ls {pid_file} >/dev/null 2>&1 && ps `cat {pid_file}` >/dev/null 2>&1")
elif action == 'stop':
daemon_cmd = format("{cmd} stop {role} && rm -f {pid_file}")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/hbase_thrift.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/hbase_thrift.py b/app-packages/hbase/package/scripts/hbase_thrift.py
new file mode 100644
index 0000000..84bfc62
--- /dev/null
+++ b/app-packages/hbase/package/scripts/hbase_thrift.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from hbase import hbase
+from hbase_service import hbase_service
+
+
+class HbaseThrift(Script):
+ def install(self, env):
+ self.install_packages(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+
+ hbase(name='thrift')
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env) # for security
+
+ hbase_service( 'thrift',
+ action = 'start'
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ hbase_service( 'thrift',
+ action = 'stop'
+ )
+
+ def status(self, env):
+ import status_params
+ env.set_params(status_params)
+ pid_file = format("{pid_dir}/hbase-{hbase_user}-thrift.pid")
+ check_process_status(pid_file)
+
+if __name__ == "__main__":
+ HbaseThrift().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/hbase/package/scripts/params.py b/app-packages/hbase/package/scripts/params.py
index ef48782..2d05289 100644
--- a/app-packages/hbase/package/scripts/params.py
+++ b/app-packages/hbase/package/scripts/params.py
@@ -58,6 +58,7 @@ ganglia_server_host = default('/configurations/global/ganglia_server_host', '')
ganglia_server_port = default('/configurations/global/ganglia_server_port', '8663')
rest_port = config['configurations']['global']['hbase_rest_port']
+thrift_port = config['configurations']['global']['hbase_thrift_port']
if security_enabled:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/app-packages/hbase/resources.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/resources.json b/app-packages/hbase/resources.json
index 9925c9e..2d2b4f9 100644
--- a/app-packages/hbase/resources.json
+++ b/app-packages/hbase/resources.json
@@ -22,5 +22,10 @@
"yarn.component.instances": "1",
"yarn.memory": "256"
}
+ "HBASE_THRIFT": {
+ "yarn.role.priority": "3",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
index 39ab16f..fde2bf7 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseClientProvider.java
@@ -239,7 +239,7 @@ public class HBaseClientProvider extends AbstractClientProvider implements
providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_MASTER,
0, -1);
- providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_REST_GATEWAY,
+ providerUtils.validateNodeCount(instanceDefinition, HBaseKeys.ROLE_THRIFT_GATEWAY,
0, -1);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
index f91e527..9c38fb2 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseKeys.java
@@ -28,11 +28,16 @@ public interface HBaseKeys {
String ROLE_REST_GATEWAY = "REST";
+ String ROLE_THRIFT_GATEWAY = "THRIFT";
+
/** {@value */
String REGION_SERVER = "regionserver";
/** {@value */
String REST_GATEWAY = "rest";
+
+ /** {@value */
+ String THRIFT_GATEWAY = "thrift";
/**
* What is the command for hbase to print a version: {@value}
*/
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
index 6fa7ca8..953ee79 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseProviderService.java
@@ -196,6 +196,11 @@ public class HBaseProviderService extends AbstractProviderService implements
logfile = "/rest-gateway.txt";
break;
+ case ROLE_THRIFT_GATEWAY:
+ roleCommand = THRIFT_GATEWAY;
+
+ logfile = "/thrift-gateway.txt";
+ break;
default:
throw new SliderInternalStateException("Cannot start role %s", role);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/a9a91b46/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
index 41beb0f..2cc457b 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
+++ b/slider-providers/hbase/slider-hbase-provider/src/main/java/org/apache/slider/providers/hbase/HBaseRoles.java
@@ -39,6 +39,8 @@ public class HBaseRoles {
public static final int KEY_REST_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 3;
+ public static final int KEY_THRIFT_GATEWAY = SliderKeys.ROLE_AM_PRIORITY_INDEX + 4;
+
/**
* Initialize role list
*/
@@ -46,6 +48,7 @@ public class HBaseRoles {
ROLES.add(new ProviderRole(HBaseKeys.ROLE_WORKER, KEY_WORKER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_MASTER, KEY_MASTER));
ROLES.add(new ProviderRole(HBaseKeys.ROLE_REST_GATEWAY, KEY_REST_GATEWAY));
+ ROLES.add(new ProviderRole(HBaseKeys.ROLE_THRIFT_GATEWAY, KEY_THRIFT_GATEWAY));
}
[16/18] git commit: SLIDER-270: one-liner fix to actually resolve the
problem
Posted by st...@apache.org.
SLIDER-270: one-liner fix to actually resolve the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/8d6c3fad
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/8d6c3fad
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/8d6c3fad
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 8d6c3fad0e1edc9c482f43305f82c976e3d53aa9
Parents: df963f2
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 09:32:36 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 09:33:07 2014 +0100
----------------------------------------------------------------------
.../apache/slider/server/appmaster/state/AppState.java | 13 ++++++++++---
.../server/appmaster/model/mock/MockYarnEngine.groovy | 2 +-
2 files changed, 11 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 2b17c3a..db74ea8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -786,7 +786,10 @@ public class AppState {
public RoleInstance getActiveContainer(ContainerId id) {
return activeContainers.get(id);
}
-
+
+ private RoleInstance removeActiveContainer(ContainerId id) {
+ return activeContainers.remove(id);
+ }
public synchronized List<RoleInstance> cloneLiveContainerInfoList() {
List<RoleInstance> allRoleInstances;
@@ -1265,11 +1268,15 @@ public class AppState {
completionOfNodeNotInLiveListEvent.incrementAndGet();
}
+
+ // and the active node list if present
+ removeActiveContainer(containerId);
+
// finally, verify the node doesn't exist any more
assert !containersBeingReleased.containsKey(
- containerId) : "container in release queue";
+ containerId) : "container still in release queue";
assert !getLiveNodes().containsKey(
- containerId) : " container in live nodes";
+ containerId) : " container still in live nodes";
assert getActiveContainer(containerId) ==
null : "Container still in active container list";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/8d6c3fad/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 763f653..b57a79a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -104,7 +104,7 @@ class MockYarnEngine {
if (op instanceof ContainerReleaseOperation) {
ContainerReleaseOperation cro = (ContainerReleaseOperation) op
ContainerId cid = cro.containerId
- releaseContainer(cid);
+ assert releaseContainer(cid);
released.add(cid)
} else {
ContainerRequestOperation req = (ContainerRequestOperation) op
[17/18] git commit: Merge branch
'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into
develop
Posted by st...@apache.org.
Merge branch 'feature/SLIDER-270_Calling_flex_(down)_the_second_time_does_not_work' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/d68be5aa
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/d68be5aa
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/d68be5aa
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: d68be5aa5eaa8c2e5a2d71097eb2907968c512bd
Parents: 405aa51 8d6c3fa
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 09:33:34 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 09:33:34 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
.../server/appmaster/SliderAppMaster.java | 16 +-
.../slider/server/appmaster/state/AppState.java | 142 +++++++++++-------
.../server/appmaster/state/RoleHistory.java | 6 +-
.../server/appmaster/state/RoleStatus.java | 4 +-
.../appstate/TestAppStateDynamicRoles.groovy | 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 ++++++++++++++++
.../model/appstate/TestMockRMOperations.groovy | 37 -----
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../model/mock/BaseMockAppStateTest.groovy | 57 ++++++-
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
.../minicluster/HBaseMiniClusterTestBase.groovy | 150 ++++++++++++-------
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 +++++++
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++++++
.../flexing/TestHMasterFlex1To2.groovy | 45 ------
.../src/test/resources/log4j.properties | 1 +
22 files changed, 497 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/d68be5aa/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
[09/18] git commit: SLIDER-270 assertion checking has found the
location of the problem
Posted by st...@apache.org.
SLIDER-270 assertion checking has found the location of the problem
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df963f24
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df963f24
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df963f24
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: df963f24b474f33d00719f91d8b50e83a8f55b6f
Parents: 291a82e
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:35:28 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:35:54 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 25 ++++++++++++++------
.../server/appmaster/state/RoleHistory.java | 6 ++---
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../src/test/resources/log4j.properties | 1 +
4 files changed, 23 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 74e91a0..2b17c3a 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -1172,7 +1172,7 @@ public class AppState {
RoleInstance roleInstance;
if (containersBeingReleased.containsKey(containerId)) {
- log.info("Container was queued for release");
+ log.info("Container was queued for release : {}", containerId);
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
int releasing = roleStatus.decReleasing();
@@ -1183,7 +1183,7 @@ public class AppState {
actual,
releasing,
completedCount);
- roleHistory.onReleaseCompleted(container);
+ roleHistory.onReleaseCompleted(container, true);
} else if (surplusNodes.remove(containerId)) {
//its a surplus one being purged
@@ -1251,18 +1251,28 @@ public class AppState {
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
+ log.info("Removing node ID {}", id);
RoleInstance node = getLiveNodes().remove(id);
- if (node == null) {
- log.warn("Received notification of completion of unknown node {}", id);
- completionOfNodeNotInLiveListEvent.incrementAndGet();
-
- } else {
+ if (node != null) {
node.state = ClusterDescription.STATE_DESTROYED;
node.exitCode = status.getExitStatus();
node.diagnostics = status.getDiagnostics();
getCompletedNodes().put(id, node);
result.roleInstance = node;
+ } else {
+ // not in the list
+ log.warn("Received notification of completion of unknown node {}", id);
+ completionOfNodeNotInLiveListEvent.incrementAndGet();
+
}
+ // finally, verify the node doesn't exist any more
+ assert !containersBeingReleased.containsKey(
+ containerId) : "container in release queue";
+ assert !getLiveNodes().containsKey(
+ containerId) : " container in live nodes";
+ assert getActiveContainer(containerId) ==
+ null : "Container still in active container list";
+
return result;
}
@@ -1422,6 +1432,7 @@ public class AppState {
* @throws SliderInternalStateException if the operation reveals that
* the internal state of the application is inconsistent.
*/
+ @SuppressWarnings("SynchronizationOnLocalVariableOrMethodParameter")
public List<AbstractRMOperation> reviewOneRole(RoleStatus role)
throws SliderInternalStateException, TriggerClusterTeardownException {
List<AbstractRMOperation> operations = new ArrayList<>();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
index 0cd2b39..de7e9d8 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleHistory.java
@@ -29,7 +29,6 @@ import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.ProviderRole;
import org.apache.slider.server.avro.RoleHistoryHeader;
import org.apache.slider.server.avro.RoleHistoryWriter;
-import org.mortbay.log.Log;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -659,10 +658,11 @@ public class RoleHistory {
/**
* App state notified of a container completed
* @param container completed container
+ * @param wasReleased
* @return true if the node was queued
*/
- public boolean onReleaseCompleted(Container container) {
- return markContainerFinished(container, true, false);
+ public boolean onReleaseCompleted(Container container, boolean wasReleased) {
+ return markContainerFinished(container, wasReleased, false);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
index 795b48f..1a71aed 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/history/TestRoleHistoryContainerEvents.groovy
@@ -158,7 +158,7 @@ class TestRoleHistoryContainerEvents extends BaseMockAppStateTest {
assert roleEntry.active == 0
// release completed
- roleHistory.onReleaseCompleted(container)
+ roleHistory.onReleaseCompleted(container, true)
assert roleEntry.releasing == 0
assert roleEntry.live == 0
assert roleEntry.active == 0
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df963f24/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
index a552a55..1cc355a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/resources/log4j.properties
@@ -57,3 +57,4 @@ log4j.logger.org.apache.hadoop.security.token.delegation=WARN
log4j.logger.org.apache.hadoop.yarn.util.AbstractLivelinessMonitor=WARN
log4j.logger.org.apache.hadoop.yarn.server.nodemanager.security=WARN
log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMNMInfo=WARN
+log4j.logger.org.apache.hadoop.hbase.client.HConnectionManager=WARN
\ No newline at end of file
[13/18] git commit: SLIDER-272. Application should specify the config
types to publish
Posted by st...@apache.org.
SLIDER-272. Application should specify the config types to publish
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/640233dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/640233dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/640233dd
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 640233dd355f8ac3fe917a307ca3251caf6593e7
Parents: 50dc76c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 3 21:48:32 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached-win/metainfo.xml | 1 +
app-packages/memcached/metainfo.xml | 1 +
.../providers/agent/AgentProviderService.java | 19 +++++++++++++++++--
.../agent/application/metadata/Application.java | 9 +++++++++
.../application/metadata/MetainfoParser.java | 1 +
.../agent/TestAgentProviderService.java | 2 ++
6 files changed, 31 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached-win/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached-win/metainfo.xml b/app-packages/memcached-win/metainfo.xml
index a0de8c2..dc2d67b 100644
--- a/app-packages/memcached-win/metainfo.xml
+++ b/app-packages/memcached-win/metainfo.xml
@@ -22,6 +22,7 @@
<name>MEMCACHED</name>
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
+ <exportedConfigs>None</exportedConfigs>
<components>
<component>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/app-packages/memcached/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/memcached/metainfo.xml b/app-packages/memcached/metainfo.xml
index 0550b7d..08a504e 100644
--- a/app-packages/memcached/metainfo.xml
+++ b/app-packages/memcached/metainfo.xml
@@ -22,6 +22,7 @@
<name>MEMCACHED</name>
<comment>Memcache is a network accessible key/value storage system, often used as a distributed cache.</comment>
<version>1.0.0</version>
+ <exportedConfigs>None</exportedConfigs>
<components>
<component>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 28c874b..682efba 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -88,6 +88,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
+import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
@@ -637,12 +638,26 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
List<ExportGroup> exportGroups = application.getExportGroups();
boolean hasExportGroups = exportGroups != null && !exportGroups.isEmpty();
+ Set<String> exportedConfigs = new HashSet();
+ String exportedConfigsStr = application.getExportedConfigs();
+ boolean exportedConfigSpecified = exportedConfigsStr != null;
+ if (application.getExportedConfigs() != null && application.getExportedConfigs().length() > 0) {
+ for (String exportedConfig : application.getExportedConfigs().split(",")) {
+ if (exportedConfig.trim().length() > 0) {
+ exportedConfigs.add(exportedConfig.trim());
+ }
+ }
+ }
+
for (ComponentStatus status : statuses) {
log.info("Status report: " + status.toString());
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
- Map<String, String> configs = status.getConfigs().get(key);
- publishApplicationInstanceData(key, key, configs.entrySet());
+ if ((exportedConfigSpecified && exportedConfigs.contains(key)) ||
+ !exportedConfigSpecified) {
+ Map<String, String> configs = status.getConfigs().get(key);
+ publishApplicationInstanceData(key, key, configs.entrySet());
+ }
}
if (hasExportGroups) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index b007313..016b4d4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -26,6 +26,7 @@ public class Application {
String name;
String comment;
String version;
+ String exportedConfigs;
List<Component> components;
List<ExportGroup> exportGroups;
List<OSSpecific> osSpecifics;
@@ -63,6 +64,14 @@ public class Application {
this.version = version;
}
+ public String getExportedConfigs() {
+ return exportedConfigs;
+ }
+
+ public void setExportedConfigs(String exportedConfigs) {
+ this.exportedConfigs = exportedConfigs;
+ }
+
public ConfigurationDependencies getConfigDependencies() {
return configDependencies;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index d2801d4..deb7a6a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -38,6 +38,7 @@ public class MetainfoParser {
digester.addBeanPropertySetter("*/application/name");
digester.addBeanPropertySetter("*/application/comment");
digester.addBeanPropertySetter("*/application/version");
+ digester.addBeanPropertySetter("*/application/exportedConfigs");
digester.addObjectCreate("*/commandOrder", CommandOrder.class);
digester.addBeanPropertySetter("*/commandOrder/command");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/640233dd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 10b1eb7..4b2e253 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -106,6 +106,7 @@ public class TestAgentProviderService {
+ " <version>0.96.0.2.1.1</version>\n"
+ " <type>YARN-APP</type>\n"
+ " <minHadoopVersion>2.1.0</minHadoopVersion>\n"
+ + " <exportedConfigs>hbase-site,global</exportedConfigs>\n"
+ " <exportGroups>\n"
+ " <exportGroup>\n"
+ " <name>QuickLinks</name>\n"
@@ -454,6 +455,7 @@ public class TestAgentProviderService {
Application application = metainfo.getApplication();
log.info("Service: " + application.toString());
Assert.assertEquals(application.getName(), "HBASE");
+ Assert.assertEquals(application.getExportedConfigs(), "hbase-site,global");
Assert.assertEquals(application.getComponents().size(), 2);
List<Component> components = application.getComponents();
int found = 0;
[10/18] git commit: SLIDER-219. Multiple dynamic ports may be asked
as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/dca4818c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/dca4818c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/dca4818c
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: dca4818c551c8fff655c8898c0786aae2fb0fb6a
Parents: ad2c798
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Fri Aug 1 11:52:35 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
.../slider/providers/ProviderService.java | 7 +
.../providers/agent/AgentProviderService.java | 329 ++++++++++---------
.../server/appmaster/SliderAppMaster.java | 2 +
.../agent/TestAgentProviderService.java | 49 ++-
4 files changed, 224 insertions(+), 163 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
index 56e24e9..b9fa34c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/ProviderService.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
@@ -69,6 +70,12 @@ public interface ProviderService extends ProviderCore, Service,
SliderException;
/**
+ * Notify the providers of container completion
+ * @param containerId
+ */
+ void notifyContainerCompleted(ContainerId containerId);
+
+ /**
* Execute a process in the AM
* @param instanceDefinition cluster description
* @param confDir configuration directory
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index b39601a..bbe2001 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
@@ -110,6 +111,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
+ private static final String SHARED_PORT_TAG = "{SHARED}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -290,116 +292,6 @@ public class AgentProviderService extends AbstractProviderService implements
}
/**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- * @param instanceDefinition
- */
- private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
- String hbMonitorInterval = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
- Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
- try {
- setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
- }catch (NumberFormatException e) {
- log.warn(
- "Bad value {} for {}. Defaulting to ",
- hbMonitorInterval,
- HEARTBEAT_MONITOR_INTERVAL,
- DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
- }
- }
-
- /**
- * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
- * @param instanceDefinition
- */
- private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
- String launchParameterStr = instanceDefinition.getAppConfOperations().
- getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
- agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
- }
-
- @VisibleForTesting
- protected Metainfo getMetainfo() {
- return this.metainfo;
- }
-
- @VisibleForTesting
- protected Map<String, ComponentInstanceState> getComponentStatuses() {
- return componentStatuses;
- }
-
- @VisibleForTesting
- protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
- return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
- }
-
- @VisibleForTesting
- protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
- this.heartbeatMonitorInterval = heartbeatMonitorInterval;
- }
-
- private int getHeartbeatMonitorInterval() {
- return this.heartbeatMonitorInterval;
- }
-
- /**
- * Publish a named config bag that may contain name-value pairs for app configurations such as hbase-site
- * @param name
- * @param description
- * @param entries
- */
- protected void publishComponentConfiguration(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
- PublishedConfiguration pubconf = new PublishedConfiguration();
- pubconf.description = description;
- pubconf.putValues(entries);
- log.info("publishing {}", pubconf);
- getAmState().getPublishedSliderConfigurations().put(name, pubconf);
- }
-
- /**
- * Get a list of all hosts for all role/container per role
- * @return
- */
- protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
- amState.refreshClusterStatus();
- return (Map<String, Map<String, ClusterNode>>)
- amState.getClusterStatus().status.get(
- ClusterDescriptionKeys.KEY_CLUSTER_LIVE);
- }
-
- private String getContainerLabel(Container container, String role) {
- return container.getId().toString() + LABEL_MAKER + role;
- }
-
- protected String getClusterInfoPropertyValue(String name) {
- StateAccessForProviders accessor = getAmState();
- assert accessor.isApplicationLive();
- ClusterDescription description = accessor.getClusterStatus();
- return description.getInfo(name);
- }
-
- /**
- * Lost heartbeat from the container - release it and ask for a replacement
- *
- * @param label
- *
- * @return if release is requested successfully
- */
- protected boolean releaseContainer(String label) {
- componentStatuses.remove(label);
- try {
- getAppMaster().refreshContainer(getContainerId(label), true);
- } catch (SliderException e) {
- log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
- return false;
- }
-
- return true;
- }
-
- /**
* Run this service
*
* @param instanceDefinition component description
@@ -421,16 +313,6 @@ public class AgentProviderService extends AbstractProviderService implements
return false;
}
- /**
- * Build the provider status, can be empty
- *
- * @return the provider status - map of entries to add to the info section
- */
- public Map<String, String> buildProviderStatus() {
- Map<String, String> stats = new HashMap<>();
- return stats;
- }
-
@Override
public boolean isSupportedRole(String role) {
return true;
@@ -489,7 +371,7 @@ public class AgentProviderService extends AbstractProviderService implements
// Otherwise, wait till the master that can publish is ready
if (isMaster &&
(canAnyMasterPublishConfig() == false || canPublishConfig(roleName))) {
- processReturnedStatus(heartBeat, componentStatus);
+ publishConfigAndExportGroups(heartBeat, componentStatus);
}
List<CommandReport> reports = heartBeat.getReports();
@@ -511,7 +393,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Component operation. Status: {}", result);
if (command == Command.INSTALL && report.getFolders() != null && report.getFolders().size() > 0) {
- processFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
+ publishLogFolderPaths(report.getFolders(), containerId, heartBeat.getFqdn());
}
}
@@ -558,13 +440,168 @@ public class AgentProviderService extends AbstractProviderService implements
return response;
}
+ @Override
+ public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
+ Map<String, String> details = super.buildMonitorDetails(clusterDesc);
+ buildRoleHostDetails(details);
+ return details;
+ }
+
+ @Override
+ public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
+ URL secureWebAPI,
+ ServiceInstanceData instanceData) throws IOException {
+ super.applyInitialRegistryDefinitions(unsecureWebAPI,
+ secureWebAPI,
+ instanceData
+ );
+
+ try {
+ instanceData.internalView.endpoints.put(
+ CustomRegistryConstants.AGENT_REST_API,
+ new RegisteredEndpoint(
+ new URL(secureWebAPI, SLIDER_PATH_AGENTS),
+ "Agent REST API"));
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ if(containerId != null) {
+ String containerIdStr = containerId.toString();
+ }
+ }
+
/**
- * Format the folder locations before publishing in the registry service
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void readAndSetHeartbeatMonitoringInterval(AggregateConf instanceDefinition) {
+ String hbMonitorInterval = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.HEARTBEAT_MONITOR_INTERVAL,
+ Integer.toString(DEFAULT_HEARTBEAT_MONITOR_INTERVAL));
+ try {
+ setHeartbeatMonitorInterval(Integer.parseInt(hbMonitorInterval));
+ }catch (NumberFormatException e) {
+ log.warn(
+ "Bad value {} for {}. Defaulting to ",
+ hbMonitorInterval,
+ HEARTBEAT_MONITOR_INTERVAL,
+ DEFAULT_HEARTBEAT_MONITOR_INTERVAL);
+ }
+ }
+
+ /**
+ * Reads and sets the heartbeat monitoring interval. If bad value is provided then log it and set to default.
+ * @param instanceDefinition
+ */
+ private void initializeAgentDebugCommands(AggregateConf instanceDefinition) {
+ String launchParameterStr = instanceDefinition.getAppConfOperations().
+ getGlobalOptions().getOption(AgentKeys.AGENT_INSTANCE_DEBUG_DATA, "");
+ agentLaunchParameter = new AgentLaunchParameter(launchParameterStr);
+ }
+
+ @VisibleForTesting
+ protected Metainfo getMetainfo() {
+ return this.metainfo;
+ }
+
+ @VisibleForTesting
+ protected Map<String, ComponentInstanceState> getComponentStatuses() {
+ return componentStatuses;
+ }
+
+ @VisibleForTesting
+ protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
+ String appDef) throws IOException {
+ return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
+ }
+
+ @VisibleForTesting
+ protected void setHeartbeatMonitorInterval(int heartbeatMonitorInterval) {
+ this.heartbeatMonitorInterval = heartbeatMonitorInterval;
+ }
+
+ private int getHeartbeatMonitorInterval() {
+ return this.heartbeatMonitorInterval;
+ }
+
+ /**
+ * Publish a named property bag that may contain name-value pairs for app configurations such as hbase-site
+ * @param name
+ * @param description
+ * @param entries
+ */
+ protected void publishComponentConfiguration(String name, String description,
+ Iterable<Map.Entry<String, String>> entries) {
+ PublishedConfiguration pubconf = new PublishedConfiguration();
+ pubconf.description = description;
+ pubconf.putValues(entries);
+ log.info("publishing {}", pubconf);
+ getAmState().getPublishedSliderConfigurations().put(name, pubconf);
+ }
+
+ /**
+ * Get a list of all hosts for all role/container per role
+ * @return
+ */
+ protected Map<String, Map<String, ClusterNode>> getRoleClusterNodeMapping() {
+ amState.refreshClusterStatus();
+ return (Map<String, Map<String, ClusterNode>>)
+ amState.getClusterStatus().status.get(
+ ClusterDescriptionKeys.KEY_CLUSTER_LIVE);
+ }
+
+ private String getContainerLabel(Container container, String role) {
+ return container.getId().toString() + LABEL_MAKER + role;
+ }
+
+ protected String getClusterInfoPropertyValue(String name) {
+ StateAccessForProviders accessor = getAmState();
+ assert accessor.isApplicationLive();
+ ClusterDescription description = accessor.getClusterStatus();
+ return description.getInfo(name);
+ }
+
+ /**
+ * Lost heartbeat from the container - release it and ask for a replacement
+ *
+ * @param label
+ *
+ * @return if release is requested successfully
+ */
+ protected boolean releaseContainer(String label) {
+ componentStatuses.remove(label);
+ try {
+ getAppMaster().refreshContainer(getContainerId(label), true);
+ } catch (SliderException e) {
+ log.info("Error while requesting container release for {}. Message: {}", label, e.getMessage());
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * Build the provider status, can be empty
+ *
+ * @return the provider status - map of entries to add to the info section
+ */
+ public Map<String, String> buildProviderStatus() {
+ Map<String, String> stats = new HashMap<>();
+ return stats;
+ }
+
+
+ /**
+ * Format the folder locations and publish in the registry service
* @param folders
* @param containerId
* @param hostFqdn
*/
- private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
+ private void publishLogFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
for (String key : folders.keySet()) {
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
@@ -578,7 +615,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @param heartBeat
* @param componentStatus
*/
- protected void processReturnedStatus(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
+ protected void publishConfigAndExportGroups(HeartBeat heartBeat, ComponentInstanceState componentStatus) {
List<ComponentStatus> statuses = heartBeat.getComponentStatus();
if (statuses != null && !statuses.isEmpty()) {
log.info("Processing {} status reports.", statuses.size());
@@ -658,11 +695,11 @@ public class AgentProviderService extends AbstractProviderService implements
boolean publishData = false;
String portValPattern = String.format(portVarFormat, portName);
if(templateToExport.contains(portValPattern)) {
- templateToExport.replace(portValPattern, ports.get(portName));
+ templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
publishData = true;
}
if(templateToExport.contains(hostNamePattern)) {
- templateToExport.replace(hostNamePattern, hostFqdn);
+ templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
publishData = true;
}
if(publishData) {
@@ -1032,9 +1069,13 @@ public class AgentProviderService extends AbstractProviderService implements
//apply any port updates
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
- String lookupKey = configName + "." + key;
- if (this.getAllocatedPorts().containsKey(lookupKey)) {
- config.put(key, getAllocatedPorts().get(lookupKey));
+ String value = config.get(key);
+ if(!value.contains(SHARED_PORT_TAG)) {
+ // If the config property is not shared then do not pass on the already allocated value
+ String lookupKey = configName + "." + key;
+ if (this.getAllocatedPorts().containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts().get(lookupKey));
+ }
}
}
}
@@ -1065,13 +1106,6 @@ public class AgentProviderService extends AbstractProviderService implements
config.put("app_install_dir", "${AGENT_WORK_ROOT}/app/install");
}
- @Override
- public Map<String, String> buildMonitorDetails(ClusterDescription clusterDesc) {
- Map<String, String> details = super.buildMonitorDetails(clusterDesc);
- buildRoleHostDetails(details);
- return details;
- }
-
private void buildRoleHostDetails(Map<String, String> details) {
for (Map.Entry<String, Map<String, ClusterNode>> entry :
getRoleClusterNodeMapping().entrySet()) {
@@ -1080,25 +1114,4 @@ public class AgentProviderService extends AbstractProviderService implements
"");
}
}
-
- @Override
- public void applyInitialRegistryDefinitions(URL unsecureWebAPI,
- URL secureWebAPI,
- ServiceInstanceData instanceData) throws IOException {
- super.applyInitialRegistryDefinitions(unsecureWebAPI,
- secureWebAPI,
- instanceData
- );
-
- try {
- instanceData.internalView.endpoints.put(
- CustomRegistryConstants.AGENT_REST_API,
- new RegisteredEndpoint(
- new URL(secureWebAPI, SLIDER_PATH_AGENTS),
- "Agent REST API"));
- } catch (URISyntaxException e) {
- throw new IOException(e);
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..a2dc36f 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -1061,6 +1061,8 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
}
+
+ getProviderService().notifyContainerCompleted(containerId);
}
// ask for more containers if any failed
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/dca4818c/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 4d63263..1f9cb41 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -355,6 +355,45 @@ public class TestAgentProviderService {
}
@Test
+ public void testComponentSpecificPublishes() throws Exception {
+ InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
+ Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doReturn(metainfo).when(mockAps).getMetainfo();
+
+ Map<String, String> ports = new HashMap<>();
+ ports.put("global.listen_port", "10010");
+ mockAps.processComponentSpecificPublishes(ports,
+ "cid1",
+ "host1",
+ "HBASE_REGIONSERVER");
+ ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
+ forClass(Collection.class);
+ ArgumentCaptor<String> publishNameCaptor = ArgumentCaptor.
+ forClass(String.class);
+ Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ anyString(),
+ publishNameCaptor.capture(),
+ entriesCaptor.capture());
+ assert entriesCaptor.getAllValues().size() == 1;
+ for (Collection coll : entriesCaptor.getAllValues()) {
+ Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
+ for (Map.Entry entry : entrySet) {
+ log.info("{}:{}", entry.getKey(), entry.getValue().toString());
+ if (entry.getKey().equals("PropertyA")) {
+ assert entry.getValue().toString().equals("host1:10010");
+ }
+ }
+ }
+ assert publishNameCaptor.getAllValues().size() == 1;
+ for (String coll : publishNameCaptor.getAllValues()) {
+ assert coll.equals("ComponentInstanceData");
+ }
+ }
+
+ @Test
public void testProcessConfig() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
@@ -387,16 +426,16 @@ public class TestAgentProviderService {
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
- mockAps.processReturnedStatus(hb, componentStatus);
+ mockAps.publishConfigAndExportGroups(hb, componentStatus);
assert componentStatus.getConfigReported() == true;
- ArgumentCaptor<Collection> commandCaptor = ArgumentCaptor.
+ ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
Mockito.verify(mockAps, Mockito.times(3)).publishComponentConfiguration(
anyString(),
anyString(),
- commandCaptor.capture());
- assert commandCaptor.getAllValues().size() == 3;
- for (Collection coll : commandCaptor.getAllValues()) {
+ entriesCaptor.capture());
+ assert entriesCaptor.getAllValues().size() == 3;
+ for (Collection coll : entriesCaptor.getAllValues()) {
Set<Map.Entry<String, String>> entrySet = (Set<Map.Entry<String, String>>) coll;
for (Map.Entry entry : entrySet) {
log.info("{}:{}", entry.getKey(), entry.getValue().toString());
[11/18] git commit: SLIDER-273. Add ability to specify a port as
shared across component instances or not
Posted by st...@apache.org.
SLIDER-273. Add ability to specify a port as shared across component instances or not
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/50dc76c9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/50dc76c9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/50dc76c9
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 50dc76c978ef4ff684263b93e8b56619cc947284
Parents: dca4818
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Aug 3 19:30:13 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
app-packages/memcached/appConfig.json | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 27 +++-
.../agent/TestCustomServiceOrchestrator.py | 18 ++-
.../providers/AbstractProviderService.java | 5 +
.../providers/agent/AgentProviderService.java | 126 ++++++++++++-------
.../model/mock/MockProviderService.groovy | 5 +
.../agent/TestAgentProviderService.java | 58 +++++++--
.../publisher/TestAgentProviderService.java | 4 +-
8 files changed, 185 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/app-packages/memcached/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/memcached/appConfig.json b/app-packages/memcached/appConfig.json
index 11fb863..78cb486 100644
--- a/app-packages/memcached/appConfig.json
+++ b/app-packages/memcached/appConfig.json
@@ -11,7 +11,7 @@
"site.global.xmx_val": "256m",
"site.global.xms_val": "128m",
"site.global.memory_val": "200M",
- "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}"
+ "site.global.listen_port": "${MEMCACHED.ALLOCATED_PORT}{DEFAULT_11211}{DO_NOT_PROPAGATE}"
},
"components": {
"slider-appmaster": {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index f6cdfb2..0f4acc6 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -233,12 +233,17 @@ class CustomServiceOrchestrator():
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
${AGENT_LOG_ROOT} -> AgentConfig.getLogPath()
+ ALLOCATED_PORT is a hint to allocate port. It works as follows:
+ Its of the form {component_name.ALLOCATED_PORT}[{DEFAULT_default_port}][{DO_NOT_PROPAGATE}]
+ Either a port gets allocated or if not then just set the value to "0"
"""
def finalize_command(self, command, store_config, allocated_ports):
component = command['componentName']
- allocated_port_format = "${{{0}.ALLOCATED_PORT}}"
- port_allocation_req = allocated_port_format.format(component)
+ allocated_for_this_component_format = "${{{0}.ALLOCATED_PORT}}"
+ allocated_for_any = ".ALLOCATED_PORT}"
+
+ port_allocation_req = allocated_for_this_component_format.format(component)
if 'configurations' in command:
for key in command['configurations']:
if len(command['configurations'][key]) > 0:
@@ -251,6 +256,10 @@ class CustomServiceOrchestrator():
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
allocated_ports[key + "." + k] = value
+ elif allocated_for_any in value:
+ ## All unallocated ports should be set to 0
+ logger.info("Assigning port 0 " + "for " + value)
+ value = "0"
command['configurations'][key][k] = value
pass
pass
@@ -264,8 +273,15 @@ class CustomServiceOrchestrator():
pass
+ """
+ Port allocation can asks for multiple dynamic ports
+ port_req_pattern is of type ${component_name.ALLOCATED_PORT}
+ append {DEFAULT_ and find the default value
+ append {DO_NOT_PROPAGATE} if it exists
+ """
def allocate_ports(self, value, port_req_pattern):
default_port_pattern = "{DEFAULT_"
+ do_not_propagate_pattern = "{DO_NOT_PROPAGATE}"
index = value.find(port_req_pattern)
while index != -1:
replaced_pattern = port_req_pattern
@@ -276,8 +292,15 @@ class CustomServiceOrchestrator():
end_index = value.find("}", start_index)
def_port_str = value[start_index:end_index]
def_port = int(def_port_str)
+ # default value of 0 means allocate any dynamic port
+ if def_port == 0:
+ def_port = None
+
replaced_pattern = replaced_pattern + def_port_str + "}"
pass
+ if index == value.find(replaced_pattern + do_not_propagate_pattern):
+ replaced_pattern = replaced_pattern + do_not_propagate_pattern
+ pass
port = self.allocate_port(def_port)
value = value.replace(replaced_pattern, str(port), 1)
logger.info("Allocated port " + str(port) + " for " + replaced_pattern)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3343a8b..c07f05f 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -284,8 +284,9 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret, 102)
+ @patch.object(CustomServiceOrchestrator, "is_port_available")
@patch.object(CustomServiceOrchestrator, "allocate_port")
- def test_allocate_port_combinations(self, allocate_port_mock):
+ def test_allocate_port_combinations(self, allocate_port_mock, is_port_available_mock):
tempdir = tempfile.gettempdir()
config = MagicMock()
config.get.return_value = "something"
@@ -296,13 +297,20 @@ class TestCustomServiceOrchestrator(TestCase):
dummy_controller = MagicMock()
orchestrator = CustomServiceOrchestrator(config, dummy_controller)
- allocate_port_mock.side_effect = [101, 102, 103]
+ is_port_available_mock.return_value = False
+ allocate_port_mock.side_effect = [101, 102, 103, 104, 105, 106]
ret = orchestrator.allocate_ports("1000", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1000")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "101")
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT},${A.ALLOCATED_PORT}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "102,103")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "104")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_0}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "105")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DO_NOT_PROPAGATE}", "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "106")
@patch.object(CustomServiceOrchestrator, "is_port_available")
@@ -320,10 +328,14 @@ class TestCustomServiceOrchestrator(TestCase):
is_port_available_mock.return_value = True
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}", "${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005")
+
ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1005}-${A.ALLOCATED_PORT}{DEFAULT_1006}",
"${A.ALLOCATED_PORT}")
self.assertEqual(ret, "1005-1006")
+ ret = orchestrator.allocate_ports("${A.ALLOCATED_PORT}{DEFAULT_1006}{DO_NOT_PROPAGATE}",
+ "${A.ALLOCATED_PORT}")
+ self.assertEqual(ret, "1006")
@patch("hostname.public_hostname")
@patch("os.path.isfile")
@@ -492,7 +504,7 @@ class TestCustomServiceOrchestrator(TestCase):
orchestrator.finalize_command(command, True, {})
self.assertEqual(command['configurations']['hbase-site']['log_root'], tempdir + "/log")
self.assertEqual(command['configurations']['hbase-site']['blog_root'], "/b/" + tempdir + "/log")
- self.assertEqual(command['configurations']['oozie-site']['b_port'], "${HBASE_REGIONSERVER.ALLOCATED_PORT}")
+ self.assertEqual(command['configurations']['oozie-site']['b_port'], "0")
self.assertEqual(orchestrator.applied_configs, command['configurations'])
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
index e35227c..7639625 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.slider.api.ClusterDescription;
import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.tools.ConfigHelper;
@@ -104,6 +105,10 @@ public abstract class AbstractProviderService
return restOps;
}
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ }
+
public void setAgentRestOperations(AgentRestOperations agentRestOperations) {
this.restOps = agentRestOperations;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bbe2001..28c874b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -111,11 +111,12 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
- private static final String SHARED_PORT_TAG = "{SHARED}";
+ private static final String SHARED_PORT_TAG = "SHARED";
+ private static final String DO_NOT_PROPAGATE_TAG = "{DO_NOT_PROPAGATE}";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
- private final Map<String, String> allocatedPorts = new ConcurrentHashMap<>();
+ private final Map<String, Map<String, String>> allocatedPorts = new ConcurrentHashMap<>();
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private Map<String, ComponentInstanceState> componentStatuses = new ConcurrentHashMap<>();
@@ -381,11 +382,12 @@ public class AgentProviderService extends AbstractProviderService implements
if (ports != null && !ports.isEmpty()) {
for (Map.Entry<String, String> port : ports.entrySet()) {
log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
- this.allocatedPorts.put(port.getKey(), port.getValue());
+ this.getAllocatedPorts().put(port.getKey(), port.getValue());
+ this.getAllocatedPorts(containerId).put(port.getKey(), port.getValue());
}
// component specific publishes
- processComponentSpecificPublishes(ports, containerId, heartBeat.getFqdn(), roleName);
+ processAndPublishComponentSpecificData(ports, containerId, heartBeat.getFqdn(), roleName);
}
CommandResult result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -469,8 +471,21 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void notifyContainerCompleted(ContainerId containerId) {
- if(containerId != null) {
+ if (containerId != null) {
String containerIdStr = containerId.toString();
+ if (this.componentInstanceData.containsKey(containerIdStr)) {
+ synchronized (this.componentInstanceData) {
+ this.componentInstanceData.remove(containerIdStr);
+ }
+ log.info("Removing container specific data for {}", containerIdStr);
+ publishComponentInstanceData();
+ }
+
+ if (this.allocatedPorts.containsKey(containerIdStr)) {
+ synchronized (this.allocatedPorts) {
+ this.allocatedPorts.remove(containerIdStr);
+ }
+ }
}
}
@@ -534,8 +549,8 @@ public class AgentProviderService extends AbstractProviderService implements
* @param description
* @param entries
*/
- protected void publishComponentConfiguration(String name, String description,
- Iterable<Map.Entry<String, String>> entries) {
+ protected void publishApplicationInstanceData(String name, String description,
+ Iterable<Map.Entry<String, String>> entries) {
PublishedConfiguration pubconf = new PublishedConfiguration();
pubconf.description = description;
pubconf.putValues(entries);
@@ -606,7 +621,7 @@ public class AgentProviderService extends AbstractProviderService implements
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
- publishComponentConfiguration(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
+ publishApplicationInstanceData(LOG_FOLDERS_TAG, LOG_FOLDERS_TAG, (new HashMap<>(this.workFolders)).entrySet());
}
@@ -627,7 +642,7 @@ public class AgentProviderService extends AbstractProviderService implements
if (status.getConfigs() != null) {
for (String key : status.getConfigs().keySet()) {
Map<String, String> configs = status.getConfigs().get(key);
- publishComponentConfiguration(key, key, configs.entrySet());
+ publishApplicationInstanceData(key, key, configs.entrySet());
}
if (hasExportGroups) {
@@ -665,7 +680,7 @@ public class AgentProviderService extends AbstractProviderService implements
map.put(export.getName(), value);
log.info("Preparing to publish. Key {} and Value {}", export.getName(), value);
}
- publishComponentConfiguration(exportGroupName, exportGroupName, map.entrySet());
+ publishApplicationInstanceData(exportGroupName, exportGroupName, map.entrySet());
}
}
}
@@ -676,10 +691,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
/** Publish component instance specific data if the component demands it */
- protected void processComponentSpecificPublishes(Map<String, String> ports,
- String containerId,
- String hostFqdn,
- String roleName) {
+ protected void processAndPublishComponentSpecificData(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String roleName) {
String portVarFormat = "${site.%s}";
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<>();
@@ -687,23 +702,25 @@ public class AgentProviderService extends AbstractProviderService implements
Application application = getMetainfo().getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
- if(component.getExports().size() > 0) {
+ if (component.getExports().size() > 0) {
- for(Export export : component.getExports()) {
+ for (Export export : component.getExports()) {
String templateToExport = export.getValue();
- for(String portName : ports.keySet()) {
+ for (String portName : ports.keySet()) {
boolean publishData = false;
String portValPattern = String.format(portVarFormat, portName);
- if(templateToExport.contains(portValPattern)) {
+ if (templateToExport.contains(portValPattern)) {
templateToExport = templateToExport.replace(portValPattern, ports.get(portName));
publishData = true;
}
- if(templateToExport.contains(hostNamePattern)) {
+ if (templateToExport.contains(hostNamePattern)) {
templateToExport = templateToExport.replace(hostNamePattern, hostFqdn);
publishData = true;
}
- if(publishData) {
+ if (publishData) {
toPublish.put(export.getName(), templateToExport);
+ log.info("Publishing {} for name {} and container {}",
+ templateToExport, export.getName(), containerId);
}
}
}
@@ -711,26 +728,30 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- if(toPublish.size() > 0) {
+ if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
- if(!this.componentInstanceData.containsKey(containerId)) {
+ if (!this.componentInstanceData.containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<>();
} else {
perContainerData = this.componentInstanceData.get(containerId);
}
perContainerData.putAll(toPublish);
this.componentInstanceData.put(containerId, perContainerData);
- Map<String, String> dataToPublish = new HashMap<>();
- synchronized (this.componentInstanceData) {
- for(String container : this.componentInstanceData.keySet()) {
- for(String prop : this.componentInstanceData.get(container).keySet()) {
- dataToPublish.put(
- container + "." + prop, this.componentInstanceData.get(container).get(prop));
- }
+ publishComponentInstanceData();
+ }
+ }
+
+ private void publishComponentInstanceData() {
+ Map<String, String> dataToPublish = new HashMap<>();
+ synchronized (this.componentInstanceData) {
+ for (String container : this.componentInstanceData.keySet()) {
+ for (String prop : this.componentInstanceData.get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, this.componentInstanceData.get(container).get(prop));
}
}
- publishComponentConfiguration(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
}
+ publishApplicationInstanceData(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
}
/**
@@ -864,7 +885,7 @@ public class AgentProviderService extends AbstractProviderService implements
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- setInstallCommandConfigurations(cmd);
+ setInstallCommandConfigurations(cmd, containerId);
cmd.setCommandParams(setCommandParameters(scriptPath, false));
@@ -914,9 +935,9 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd) throws SliderException {
+ private void setInstallCommandConfigurations(ExecutionCommand cmd, String containerId) throws SliderException {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
}
@@ -943,7 +964,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, false));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
@@ -998,17 +1019,33 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, true));
- Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
+ Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId);
cmd.setConfigurations(configurations);
response.addExecutionCommand(cmd);
}
protected Map<String, String> getAllocatedPorts() {
- return this.allocatedPorts;
+ return getAllocatedPorts(SHARED_PORT_TAG);
}
- private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf)
+ protected Map<String, Map<String, String>> getComponentInstanceData() {
+ return this.componentInstanceData;
+ }
+
+ protected Map<String, String> getAllocatedPorts(String containerId) {
+ if (!this.allocatedPorts.containsKey(containerId)) {
+ synchronized (this.allocatedPorts) {
+ if (!this.allocatedPorts.containsKey(containerId)) {
+ this.allocatedPorts.put(containerId, new ConcurrentHashMap());
+ }
+ }
+ }
+ return this.allocatedPorts.get(containerId);
+ }
+
+ private Map<String, Map<String, String>> buildCommandConfigurations(
+ ConfTreeOperations appConf, String containerId)
throws SliderException {
Map<String, Map<String, String>> configurations = new TreeMap<>();
@@ -1019,7 +1056,7 @@ public class AgentProviderService extends AbstractProviderService implements
//Add global
for (String configType : configs) {
addNamedConfiguration(configType, appConf.getGlobalOptions().options,
- configurations, tokens);
+ configurations, tokens, containerId);
}
return configurations;
@@ -1057,7 +1094,7 @@ public class AgentProviderService extends AbstractProviderService implements
private void addNamedConfiguration(String configName, Map<String, String> sourceConfig,
Map<String, Map<String, String>> configurations,
- Map<String, String> tokens) {
+ Map<String, String> tokens, String containerId) {
Map<String, String> config = new HashMap<>();
if (configName.equals(GLOBAL_CONFIG_TAG)) {
addDefaultGlobalConfig(config);
@@ -1070,12 +1107,17 @@ public class AgentProviderService extends AbstractProviderService implements
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
String value = config.get(key);
- if(!value.contains(SHARED_PORT_TAG)) {
- // If the config property is not shared then do not pass on the already allocated value
- String lookupKey = configName + "." + key;
+ String lookupKey = configName + "." + key;
+ if(!value.contains(DO_NOT_PROPAGATE_TAG)) {
+ // If the config property is shared then pass on the already allocated value
+ // from any container
if (this.getAllocatedPorts().containsKey(lookupKey)) {
config.put(key, getAllocatedPorts().get(lookupKey));
}
+ } else {
+ if (this.getAllocatedPorts(containerId).containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts(containerId).get(lookupKey));
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
index 7b73451..db28a22 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockProviderService.groovy
@@ -23,6 +23,7 @@ import org.apache.hadoop.fs.Path
import org.apache.hadoop.service.LifecycleEvent
import org.apache.hadoop.service.ServiceStateChangeListener
import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
import org.apache.slider.api.ClusterDescription
import org.apache.slider.common.tools.SliderFileSystem
import org.apache.slider.core.conf.AggregateConf
@@ -229,4 +230,8 @@ class MockProviderService implements ProviderService {
throws MalformedURLException, IOException {
}
+
+ @Override
+ public void notifyContainerCompleted(ContainerId containerId) {
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 1f9cb41..10b1eb7 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -360,20 +360,20 @@ public class TestAgentProviderService {
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
AgentProviderService aps = new AgentProviderService();
AgentProviderService mockAps = Mockito.spy(aps);
- doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
Map<String, String> ports = new HashMap<>();
ports.put("global.listen_port", "10010");
- mockAps.processComponentSpecificPublishes(ports,
- "cid1",
- "host1",
- "HBASE_REGIONSERVER");
+ mockAps.processAndPublishComponentSpecificData(ports,
+ "cid1",
+ "host1",
+ "HBASE_REGIONSERVER");
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
ArgumentCaptor<String> publishNameCaptor = ArgumentCaptor.
forClass(String.class);
- Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
anyString(),
publishNameCaptor.capture(),
entriesCaptor.capture());
@@ -422,7 +422,7 @@ public class TestAgentProviderService {
ComponentInstanceState componentStatus = new ComponentInstanceState("HBASE_MASTER", "aid", "cid");
AgentProviderService mockAps = Mockito.spy(aps);
- doNothing().when(mockAps).publishComponentConfiguration(anyString(), anyString(), anyCollection());
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
doReturn(metainfo).when(mockAps).getMetainfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
@@ -430,7 +430,7 @@ public class TestAgentProviderService {
assert componentStatus.getConfigReported() == true;
ArgumentCaptor<Collection> entriesCaptor = ArgumentCaptor.
forClass(Collection.class);
- Mockito.verify(mockAps, Mockito.times(3)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(3)).publishApplicationInstanceData(
anyString(),
anyString(),
entriesCaptor.capture());
@@ -627,7 +627,7 @@ public class TestAgentProviderService {
anyString(),
anyString(),
any(HeartBeatResponse.class));
- doNothing().when(mockAps).publishComponentConfiguration(
+ doNothing().when(mockAps).publishApplicationInstanceData(
anyString(),
anyString(),
anyCollection());
@@ -797,13 +797,44 @@ public class TestAgentProviderService {
log.warn(he.getMessage());
}
- Mockito.verify(mockAps, Mockito.times(1)).publishComponentConfiguration(
+ Mockito.verify(mockAps, Mockito.times(1)).publishApplicationInstanceData(
anyString(),
anyString(),
anyCollection());
}
@Test
+ public void testNotifyContainerCompleted() {
+ AgentProviderService aps = new AgentProviderService();
+ AgentProviderService mockAps = Mockito.spy(aps);
+ doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
+
+ ContainerId cid = new MyContainerId(1);
+ String id = cid.toString();
+ mockAps.getAllocatedPorts().put("a", "100");
+ mockAps.getAllocatedPorts(id).put("b", "101");
+ mockAps.getAllocatedPorts("cid2").put("c", "102");
+
+ mockAps.getComponentInstanceData().put("cid2", new HashMap<String, String>());
+ mockAps.getComponentInstanceData().put(id, new HashMap<String, String>());
+
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+ mockAps.notifyContainerCompleted(new MyContainerId(1));
+
+ Assert.assertNull(mockAps.getComponentInstanceData().get(id));
+ Assert.assertNotNull(mockAps.getComponentInstanceData().get("cid2"));
+
+ Assert.assertEquals(mockAps.getAllocatedPorts().size(), 1);
+ Assert.assertEquals(mockAps.getAllocatedPorts(id).size(), 0);
+ Assert.assertEquals(mockAps.getAllocatedPorts("cid2").size(), 1);
+ }
+
+ @Test
public void testAddInstallCommand() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
@@ -867,6 +898,8 @@ public class TestAgentProviderService {
treeOps.set("config_types", "hbase-site");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");
treeOps.getGlobalOptions().put("site.hbase-site.b.port", "${HBASE_MASTER.ALLOCATED_PORT}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random.port", "${HBASE_MASTER.ALLOCATED_PORT}{DO_NOT_PROPAGATE}");
+ treeOps.getGlobalOptions().put("site.hbase-site.random2.port", "${HBASE_MASTER.ALLOCATED_PORT}");
expect(access.getAppConfSnapshot()).andReturn(treeOps).anyTimes();
expect(access.getInternalsSnapshot()).andReturn(treeOps).anyTimes();
@@ -885,6 +918,9 @@ public class TestAgentProviderService {
allocatedPorts.put("hbase-site.a.port", "10023");
allocatedPorts.put("hbase-site.b.port", "10024");
doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
+ Map<String, String> allocatedPorts2 = new HashMap<>();
+ allocatedPorts2.put("hbase-site.random.port", "10025");
+ doReturn(allocatedPorts2).when(mockAps).getAllocatedPorts(anyString());
replay(access);
@@ -894,6 +930,8 @@ public class TestAgentProviderService {
Assert.assertTrue(hbaseSiteConf.containsKey("a.port"));
Assert.assertTrue(hbaseSiteConf.get("a.port").equals("10023"));
Assert.assertTrue(hbaseSiteConf.get("b.port").equals("10024"));
+ Assert.assertTrue(hbaseSiteConf.get("random.port").equals("10025"));
+ Assert.assertTrue(hbaseSiteConf.get("random2.port").equals("${HBASE_MASTER.ALLOCATED_PORT}"));
}
private static class MyContainer extends Container {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/50dc76c9/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
index 2427009..92e6491 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/publisher/TestAgentProviderService.java
@@ -46,8 +46,8 @@ public class TestAgentProviderService extends AgentProviderService{
dummyProps.put("prop1", "val1");
dummyProps.put("prop2", "val2");
log.info("publishing dummy-site.xml with values {}", dummyProps);
- publishComponentConfiguration("dummy-site", "dummy configuration",
- dummyProps.entrySet());
+ publishApplicationInstanceData("dummy-site", "dummy configuration",
+ dummyProps.entrySet());
}
[02/18] git commit: SLIDER-270: flex down twice not working. This
isn't showing up in a (new) mock test
Posted by st...@apache.org.
SLIDER-270: flex down twice not working. This isn't showing up in a (new) mock test
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/28babb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/28babb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/28babb19
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 28babb1923bbfd23cc9bc0148d12497b9e848f29
Parents: 0b52d4b
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 16:05:03 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 16:05:03 2014 +0100
----------------------------------------------------------------------
.../server/appmaster/SliderAppMaster.java | 16 +--
.../slider/server/appmaster/state/AppState.java | 43 +++----
.../appstate/TestAppStateDynamicRoles.groovy | 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++++++++++++++++++
.../model/appstate/TestMockRMOperations.groovy | 37 ------
.../model/mock/BaseMockAppStateTest.groovy | 57 ++++++++-
.../appmaster/model/mock/MockYarnEngine.groovy | 4 +-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
11 files changed, 219 insertions(+), 80 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
index 7e0ae5e..1ad7fa7 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/SliderAppMaster.java
@@ -671,12 +671,13 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
//build the instance
appState.buildInstance(instanceDefinition,
- providerConf,
- providerRoles,
- fs.getFileSystem(),
- historyDir,
- liveContainers,
- appInformation);
+ serviceConf,
+ providerConf,
+ providerRoles,
+ fs.getFileSystem(),
+ historyDir,
+ liveContainers,
+ appInformation);
// add the AM to the list of nodes in the cluster
@@ -1055,8 +1056,7 @@ public class SliderAppMaster extends AbstractSliderLaunchedService
// non complete containers should not be here
assert (status.getState() == ContainerState.COMPLETE);
- AppState.NodeCompletionResult result = appState.onCompletedNode(
- getConfig(), status);
+ AppState.NodeCompletionResult result = appState.onCompletedNode(status);
if (result.containerFailed) {
RoleInstance ri = result.roleInstance;
log.error("Role instance {} failed ", ri);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index cc238ff..9f8a39d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -54,7 +54,6 @@ import org.apache.slider.core.exceptions.ErrorStrings;
import org.apache.slider.core.exceptions.NoSuchNodeException;
import org.apache.slider.core.exceptions.SliderInternalStateException;
import org.apache.slider.core.exceptions.TriggerClusterTeardownException;
-import org.apache.slider.core.registry.docstore.PublishedConfigSet;
import org.apache.slider.providers.ProviderRole;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,6 +249,8 @@ public class AppState {
private long startTimeThreshold;
private int failureThreshold = 10;
+
+ private String logServerURL = "";
public AppState(AbstractRecordFactory recordFactory) {
this.recordFactory = recordFactory;
@@ -432,6 +433,7 @@ public class AppState {
/**
* Build up the application state
* @param instanceDefinition definition of the applicatin instance
+ * @param appmasterConfig
* @param publishedProviderConf any configuration info to be published by a provider
* @param providerRoles roles offered by a provider
* @param fs filesystem
@@ -440,12 +442,12 @@ public class AppState {
* @param applicationInfo
*/
public synchronized void buildInstance(AggregateConf instanceDefinition,
- Configuration publishedProviderConf,
- List<ProviderRole> providerRoles,
- FileSystem fs,
- Path historyDir,
- List<Container> liveContainers,
- Map<String, String> applicationInfo) throws
+ Configuration appmasterConfig, Configuration publishedProviderConf,
+ List<ProviderRole> providerRoles,
+ FileSystem fs,
+ Path historyDir,
+ List<Container> liveContainers,
+ Map<String, String> applicationInfo) throws
BadClusterStateException,
BadConfigException,
IOException {
@@ -514,6 +516,10 @@ public class AppState {
//rebuild any live containers
rebuildModelFromRestart(liveContainers);
+ // any am config options to pick up
+
+ logServerURL = appmasterConfig.get(YarnConfiguration.YARN_LOG_SERVER_URL,
+ "");
//mark as live
applicationLive = true;
}
@@ -1161,18 +1167,6 @@ public class AppState {
* @return NodeCompletionResult
*/
public synchronized NodeCompletionResult onCompletedNode(ContainerStatus status) {
- return onCompletedNode(null, status);
- }
-
- /**
- * handle completed node in the CD -move something from the live
- * server list to the completed server list
- * @param amConf YarnConfiguration
- * @param status the node that has just completed
- * @return NodeCompletionResult
- */
- public synchronized NodeCompletionResult onCompletedNode(Configuration amConf,
- ContainerStatus status) {
ContainerId containerId = status.getContainerId();
NodeCompletionResult result = new NodeCompletionResult();
RoleInstance roleInstance;
@@ -1219,11 +1213,8 @@ public class AppState {
}
String completedLogsUrl = null;
Container c = roleInstance.container;
- String url = null;
- if (amConf != null) {
- url = amConf.get(YarnConfiguration.YARN_LOG_SERVER_URL);
- }
- if (user != null && url != null) {
+ String url = logServerURL;
+ if (user != null && SliderUtils.isSet(url)) {
completedLogsUrl = url
+ "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user;
}
@@ -1255,12 +1246,12 @@ public class AppState {
completionOfUnknownContainerEvent.incrementAndGet();
}
}
-
+
if (result.surplusNode) {
//a surplus node
return result;
}
-
+
//record the complete node's details; this pulls it from the livenode set
//remove the node
ContainerId id = status.getContainerId();
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
index 6e387d8..6d70885 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateDynamicRoles.groovy
@@ -72,11 +72,13 @@ class TestAppStateDynamicRoles extends BaseMockAppStateTest
appState.buildInstance(
instance,
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
historyPath,
- null, null)
+ null,
+ null)
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
index 190e927..8d7ff4f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestAppStateRebuildOnAMRestart.groovy
@@ -71,11 +71,13 @@ class TestAppStateRebuildOnAMRestart extends BaseMockAppStateTest
//and rebuild
appState.buildInstance(
factory.newInstanceDefinition(r0, r1, r2),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
historyPath,
- containers, null)
+ containers,
+ null)
assert appState.getStartedCountainerCount() == clusterSize
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
index 1693365..911880a 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestFlexDynamicRoles.groovy
@@ -73,6 +73,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest
appState.buildInstance(instance,
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
@@ -176,6 +177,7 @@ class TestFlexDynamicRoles extends BaseMockAppStateTest
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
new file mode 100644
index 0000000..d2dc028
--- /dev/null
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockFlexing.groovy
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.server.appmaster.model.appstate
+
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.Container
+import org.apache.hadoop.yarn.api.records.ContainerId
+import org.apache.hadoop.yarn.client.api.AMRMClient
+import org.apache.slider.server.appmaster.model.mock.BaseMockAppStateTest
+import org.apache.slider.server.appmaster.model.mock.MockFactory
+import org.apache.slider.server.appmaster.model.mock.MockRMOperationHandler
+import org.apache.slider.server.appmaster.model.mock.MockRoles
+import org.apache.slider.server.appmaster.state.AbstractRMOperation
+import org.apache.slider.server.appmaster.state.AppState
+import org.apache.slider.server.appmaster.state.ContainerAssignment
+import org.apache.slider.server.appmaster.state.ContainerReleaseOperation
+import org.apache.slider.server.appmaster.state.ContainerRequestOperation
+import org.apache.slider.server.appmaster.state.RMOperationHandler
+import org.apache.slider.server.appmaster.state.RoleInstance
+import org.junit.Test
+
+import static org.apache.slider.server.appmaster.state.ContainerPriority.buildPriority
+import static org.apache.slider.server.appmaster.state.ContainerPriority.extractRole
+
+@Slf4j
+class TestMockFlexing extends BaseMockAppStateTest implements MockRoles {
+
+ @Override
+ String getTestName() {
+ return "TestMockFlexing"
+ }
+
+
+
+ @Test
+ public void testFlexDuringLaunchPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ List<Container> allocations = engine.execute(ops)
+ List<ContainerAssignment> assignments = [];
+ List<AbstractRMOperation> releases = []
+ appState.onContainersAllocated(allocations, assignments, releases)
+ assert assignments.size() == 1
+ ContainerAssignment assigned = assignments[0]
+ Container target = assigned.container
+ RoleInstance ri = roleInstance(assigned)
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ //now this is the start point.
+ appState.containerStartSubmitted(target, ri);
+
+ ops = appState.reviewRequestAndReleaseNodes()
+ assert ops.empty
+
+ RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
+ }
+
+ @Test
+ public void testFlexBeforeAllocationPhase() throws Throwable {
+ role0Status.desired = 1
+
+ List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
+ assert !ops.empty
+ List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
+ assert ops2.empty
+ }
+
+
+ @Test
+ public void testFlexDownTwice() throws Throwable {
+ int r0 = 6
+ int r1 = 0
+ int r2 = 0
+ role0Status.desired = r0
+ role1Status.desired = r1
+ role2Status.desired = r2
+ List<RoleInstance> instances = createAndStartNodes()
+
+ int clusterSize = r0 + r1 + r2
+ assert instances.size() == clusterSize
+ log.info("shrinking cluster")
+ r0 = 4
+ role0Status.desired = r0
+ List<AppState.NodeCompletionResult> completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 2
+
+ // no-op review
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 0
+
+
+ // now shrink again
+ role0Status.desired = r0 = 1
+ completionResults = []
+ instances = createStartAndStopNodes(completionResults)
+ assert instances.size() == 0
+ // assert two nodes were released
+ assert completionResults.size() == 3
+
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
index 168ac9f..e27827b 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/appstate/TestMockRMOperations.groovy
@@ -173,41 +173,4 @@ class TestMockRMOperations extends BaseMockAppStateTest implements MockRoles {
assert ri3 == null
}
- @Test
- public void testFlexDuringLaunchPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocations = engine.execute(
- ops)
- List<ContainerAssignment> assignments = [];
- List<AbstractRMOperation> releases = []
- appState.onContainersAllocated(allocations, assignments, releases)
- assert assignments.size() == 1
- ContainerAssignment assigned = assignments[0]
- Container target = assigned.container
- RoleInstance ri = roleInstance(assigned)
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- //now this is the start point.
- appState.containerStartSubmitted(target, ri);
-
- ops = appState.reviewRequestAndReleaseNodes()
- assert ops.empty
-
- RoleInstance ri2 = appState.innerOnNodeManagerContainerStarted(target.id)
- }
-
- @Test
- public void testFlexBeforeAllocationPhase() throws Throwable {
- role0Status.desired = 1
-
- List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- assert !ops.empty
- List<AbstractRMOperation> ops2 = appState.reviewRequestAndReleaseNodes()
- assert ops2.empty
- }
-
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
index 628c729..4a34e7f 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/BaseMockAppStateTest.groovy
@@ -82,6 +82,7 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES)
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
@@ -166,20 +167,72 @@ abstract class BaseMockAppStateTest extends SliderTestBase implements MockRoles
* @return a list of roles
*/
protected List<RoleInstance> createAndStartNodes() {
- List<RoleInstance> instances = createAndSubmitNodes()
+ return createStartAndStopNodes([])
+ }
+
+ /**
+ * Create, Start and stop nodes
+ * @param completionResults List filled in with the status on all completed nodes
+ * @return the nodes
+ */
+ public List<RoleInstance> createStartAndStopNodes(
+ List<AppState.NodeCompletionResult> completionResults) {
+ List<ContainerId> released = []
+ List<RoleInstance> instances = createAndSubmitNodes(released)
for (RoleInstance instance : instances) {
assert appState.onNodeManagerContainerStarted(instance.containerId)
}
+ releaseContainers(completionResults,
+ released,
+ ContainerState.COMPLETE,
+ "released",
+ 0
+ )
return instances
}
/**
+ * Release a list of containers, updating the completion results
+ * @param completionResults
+ * @param containerIds
+ * @param containerState
+ * @param exitText
+ * @param containerExitCode
+ * @return
+ */
+ public def releaseContainers(
+ List<AppState.NodeCompletionResult> completionResults,
+ List<ContainerId> containerIds,
+ ContainerState containerState,
+ String exitText,
+ int containerExitCode) {
+ containerIds.each { ContainerId id ->
+ ContainerStatus status = ContainerStatus.newInstance(id,
+ containerState,
+ exitText,
+ containerExitCode)
+ completionResults << appState.onCompletedNode(status)
+
+ }
+ }
+
+ /**
* Create nodes and submit them
* @return a list of roles
*/
public List<RoleInstance> createAndSubmitNodes() {
+ return createAndSubmitNodes([])
+ }
+
+ /**
+ * Create nodes and submit them
+ * @param released a list that is built up of all released nodes
+ * @return a list of roles allocated
+ */
+ public List<RoleInstance> createAndSubmitNodes(
+ List<ContainerId> released) {
List<AbstractRMOperation> ops = appState.reviewRequestAndReleaseNodes()
- List<Container> allocatedContainers = engine.execute(ops)
+ List<Container> allocatedContainers = engine.execute(ops, released)
List<ContainerAssignment> assignments = [];
List<AbstractRMOperation> operations = []
appState.onContainersAllocated(allocatedContainers, assignments, operations)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
index 7ebdf52..763f653 100644
--- a/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/server/appmaster/model/mock/MockYarnEngine.groovy
@@ -86,9 +86,7 @@ class MockYarnEngine {
* @param ops
* @return
*/
- List<Container> execute(
- List<AbstractRMOperation> ops
- ) {
+ List<Container> execute(List<AbstractRMOperation> ops) {
return execute(ops, [])
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
index 17fbe2b..96a4510 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/agent/TestAMAgentWebServices.java
@@ -24,7 +24,6 @@ import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.api.json.JSONConfiguration;
-import com.sun.jersey.test.framework.JerseyTest;
import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -43,7 +42,6 @@ import org.apache.slider.server.appmaster.web.WebAppApiImpl;
import org.apache.slider.server.appmaster.web.rest.RestPaths;
import org.apache.slider.server.services.security.CertificateManager;
import org.apache.slider.server.services.security.SecurityUtils;
-import org.apache.slider.test.SliderTestBase;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -125,6 +123,7 @@ public class TestAMAgentWebServices {
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/28babb19/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
index 134f0bc..96e196f 100644
--- a/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
+++ b/slider-core/src/test/java/org/apache/slider/server/appmaster/web/rest/management/TestAMManagementWebServices.java
@@ -170,6 +170,7 @@ public class TestAMManagementWebServices extends JerseyTest {
appState.setContainerLimits(RM_MAX_RAM, RM_MAX_CORES);
appState.buildInstance(
factory.newInstanceDefinition(0, 0, 0),
+ new Configuration(),
new Configuration(false),
factory.ROLES,
fs,
[14/18] git commit: SLIDER-219. Multiple dynamic ports may be asked
as part of the same config value
Posted by st...@apache.org.
SLIDER-219. Multiple dynamic ports may be asked as part of the same config value
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ad2c798d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ad2c798d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ad2c798d
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: ad2c798d309aded2ef18dea7aa1f90acdd367b14
Parents: 1894a0c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jul 30 23:44:22 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Aug 5 18:51:58 2014 -0700
----------------------------------------------------------------------
.../memcached/package/scripts/memcached.py | 2 +-
.../memcached/package/scripts/params.py | 1 +
.../python/agent/CustomServiceOrchestrator.py | 2 +-
.../agent/TestCustomServiceOrchestrator.py | 6 +-
slider-agent/src/test/python/agent/TestMain.py | 4 +-
.../providers/agent/AgentProviderService.java | 70 ++++++++++++++++++--
.../agent/TestAgentProviderService.java | 4 +-
7 files changed, 76 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/memcached.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/memcached.py b/app-packages/memcached/package/scripts/memcached.py
index b2df89a..cf04cf1 100644
--- a/app-packages/memcached/package/scripts/memcached.py
+++ b/app-packages/memcached/package/scripts/memcached.py
@@ -33,7 +33,7 @@ class Memcached(Script):
import params
env.set_params(params)
self.configure(env)
- process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val}")
+ process_cmd = format("{java64_home}/bin/java -Xmx{xmx_val} -Xms{xms_val} -classpath {app_root}/*:{additional_cp} com.thimbleware.jmemcached.Main --memory={memory_val} --port={port}")
Execute(process_cmd,
user=params.app_user,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/app-packages/memcached/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/app-packages/memcached/package/scripts/params.py b/app-packages/memcached/package/scripts/params.py
index a2fba8b..f8e64da 100644
--- a/app-packages/memcached/package/scripts/params.py
+++ b/app-packages/memcached/package/scripts/params.py
@@ -30,3 +30,4 @@ additional_cp = config['configurations']['global']['additional_cp']
xmx_val = config['configurations']['global']['xmx_val']
xms_val = config['configurations']['global']['xms_val']
memory_val = config['configurations']['global']['memory_val']
+port = config['configurations']['global']['listen_port']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 7d740c2..f6cdfb2 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -250,7 +250,7 @@ class CustomServiceOrchestrator():
self.config.getLogPath())
if port_allocation_req in value:
value = self.allocate_ports(value, port_allocation_req)
- allocated_ports[k] = value
+ allocated_ports[key + "." + k] = value
command['configurations'][key][k] = value
pass
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 3b41bd7..3343a8b 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -226,7 +226,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
self.assertEqual(ret['exitcode'], 0)
- self.assertEqual(ret['allocated_ports'], {'a.port': '10233'})
+ self.assertEqual(ret['allocated_ports'], {'a.a.port': '10233'})
self.assertTrue(run_file_mock.called)
self.assertEqual(run_file_mock.call_count, 1)
@@ -480,8 +480,8 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(command['configurations']['oozie-site']['a_port'], "10023")
self.assertEqual(orchestrator.applied_configs, {})
self.assertEqual(len(allocated_ports), 1)
- self.assertTrue('a_port' in allocated_ports)
- self.assertEqual(allocated_ports['a_port'], '10023')
+ self.assertTrue('oozie-site.a_port' in allocated_ports)
+ self.assertEqual(allocated_ports['oozie-site.a_port'], '10023')
command['configurations']['hbase-site']['work_root'] = "${AGENT_WORK_ROOT}"
command['configurations']['hbase-site']['log_root'] = "${AGENT_LOG_ROOT}/log"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index bc8d2a3..4ad7a8f 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -20,10 +20,10 @@ limitations under the License.
import StringIO
import sys
-from agent import NetUtil, security
+import NetUtil, security
from mock.mock import MagicMock, patch, ANY
import unittest
-from agent import ProcessHelper, main
+import ProcessHelper, main
import logging
import signal
from AgentConfig import AgentConfig
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f878b36..b39601a 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -109,7 +109,7 @@ public class AgentProviderService extends AbstractProviderService implements
private static final String CONTAINER_ID = "container_id";
private static final String GLOBAL_CONFIG_TAG = "global";
private static final String LOG_FOLDERS_TAG = "LogFolders";
- private static final String COMPONENT_DATA_TAG = "ComponentData";
+ private static final String COMPONENT_DATA_TAG = "ComponentInstanceData";
private static final int MAX_LOG_ENTRIES = 20;
private static final int DEFAULT_HEARTBEAT_MONITOR_INTERVAL = 60 * 1000;
private final Object syncLock = new Object();
@@ -127,7 +127,7 @@ public class AgentProviderService extends AbstractProviderService implements
return size() > MAX_LOG_ENTRIES;
}
});
- private Map<String, Map<String, String>> componentData = new ConcurrentHashMap();
+ private Map<String, Map<String, String>> componentInstanceData = new ConcurrentHashMap();
private Boolean canAnyMasterPublish = null;
private AgentLaunchParameter agentLaunchParameter = null;
@@ -501,6 +501,9 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Recording allocated port for {} as {}", port.getKey(), port.getValue());
this.allocatedPorts.put(port.getKey(), port.getValue());
}
+
+ // component specific publishes
+ processComponentSpecificPublishes(ports, containerId, heartBeat.getFqdn(), roleName);
}
CommandResult result = CommandResult.getCommandResult(report.getStatus());
Command command = Command.getCommand(report.getRoleCommand());
@@ -635,6 +638,64 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ /** Publish component instance specific data if the component demands it */
+ protected void processComponentSpecificPublishes(Map<String, String> ports,
+ String containerId,
+ String hostFqdn,
+ String roleName) {
+ String portVarFormat = "${site.%s}";
+ String hostNamePattern = "${THIS_HOST}";
+ Map<String, String> toPublish = new HashMap<>();
+
+ Application application = getMetainfo().getApplication();
+ for (Component component : application.getComponents()) {
+ if (component.getName().equals(roleName)) {
+ if(component.getExports().size() > 0) {
+
+ for(Export export : component.getExports()) {
+ String templateToExport = export.getValue();
+ for(String portName : ports.keySet()) {
+ boolean publishData = false;
+ String portValPattern = String.format(portVarFormat, portName);
+ if(templateToExport.contains(portValPattern)) {
+ templateToExport.replace(portValPattern, ports.get(portName));
+ publishData = true;
+ }
+ if(templateToExport.contains(hostNamePattern)) {
+ templateToExport.replace(hostNamePattern, hostFqdn);
+ publishData = true;
+ }
+ if(publishData) {
+ toPublish.put(export.getName(), templateToExport);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if(toPublish.size() > 0) {
+ Map<String, String> perContainerData = null;
+ if(!this.componentInstanceData.containsKey(containerId)) {
+ perContainerData = new ConcurrentHashMap<>();
+ } else {
+ perContainerData = this.componentInstanceData.get(containerId);
+ }
+ perContainerData.putAll(toPublish);
+ this.componentInstanceData.put(containerId, perContainerData);
+ Map<String, String> dataToPublish = new HashMap<>();
+ synchronized (this.componentInstanceData) {
+ for(String container : this.componentInstanceData.keySet()) {
+ for(String prop : this.componentInstanceData.get(container).keySet()) {
+ dataToPublish.put(
+ container + "." + prop, this.componentInstanceData.get(container).get(prop));
+ }
+ }
+ }
+ publishComponentConfiguration(COMPONENT_DATA_TAG, COMPONENT_DATA_TAG, dataToPublish.entrySet());
+ }
+ }
+
/**
* Extract script path from the application metainfo
*
@@ -971,8 +1032,9 @@ public class AgentProviderService extends AbstractProviderService implements
//apply any port updates
if (!this.getAllocatedPorts().isEmpty()) {
for (String key : config.keySet()) {
- if (this.getAllocatedPorts().containsKey(key)) {
- config.put(key, getAllocatedPorts().get(key));
+ String lookupKey = configName + "." + key;
+ if (this.getAllocatedPorts().containsKey(lookupKey)) {
+ config.put(key, getAllocatedPorts().get(lookupKey));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ad2c798d/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index c34d0ea..4d63263 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -843,8 +843,8 @@ public class TestAgentProviderService {
roleClusterNodeMap.put("HBASE_MASTER", container);
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
Map<String, String> allocatedPorts = new HashMap<>();
- allocatedPorts.put("a.port", "10023");
- allocatedPorts.put("b.port", "10024");
+ allocatedPorts.put("hbase-site.a.port", "10023");
+ allocatedPorts.put("hbase-site.b.port", "10024");
doReturn(allocatedPorts).when(mockAps).getAllocatedPorts();
replay(access);
[04/18] git commit: SLIDER-270 test to replicate the failure
condition in the slider-hbase-provider
Posted by st...@apache.org.
SLIDER-270 test to replicate the failure condition in the slider-hbase-provider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/34c00e86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/34c00e86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/34c00e86
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 34c00e86ddbf1c4ec8c800ff68be15a10752c14a
Parents: 28babb1
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 18:10:59 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 18:10:59 2014 +0100
----------------------------------------------------------------------
.../minicluster/HBaseMiniClusterTestBase.groovy | 149 ++++++++++++-------
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 ++++++++
3 files changed, 151 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
index 3f49771..c9f0a4f 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/HBaseMiniClusterTestBase.groovy
@@ -26,7 +26,6 @@ import org.apache.hadoop.hbase.client.HConnection
import org.apache.slider.api.ClusterDescription
import org.apache.slider.api.ClusterNode
import org.apache.slider.api.ResourceKeys
-import org.apache.slider.providers.hbase.HBaseKeys
import org.apache.slider.client.SliderClient
import org.apache.slider.core.main.ServiceLauncher
import org.apache.slider.providers.hbase.HBaseTestUtils
@@ -214,9 +213,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
(ROLE_MASTER): masters,
(ROLE_WORKER): workers,
];
- extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY << YRAM
- extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ hbaseArgs(extraArgs)
return createCluster(clustername,
roles,
@@ -226,6 +223,15 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
[:])
}
+ public List<String> hbaseArgs(List<String> extraArgs) {
+ extraArgs << ARG_RES_COMP_OPT << ROLE_MASTER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_RES_COMP_OPT << ROLE_WORKER << ResourceKeys.YARN_MEMORY <<
+ YRAM
+ extraArgs << ARG_PROVIDER << PROVIDER_HBASE;
+ return extraArgs;
+ }
+
/**
* Create an AM without a master
* @param clustername AM name
@@ -241,9 +247,7 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
];
return createCluster(clustername,
roles,
- [
- ARG_PROVIDER, PROVIDER_HBASE
- ],
+ hbaseArgs([]),
deleteExistingData,
blockUntilRunning,
[:])
@@ -278,62 +282,93 @@ public abstract class HBaseMiniClusterTestBase extends YarnZKMiniClusterTestBase
int masters,
int masterFlexTarget,
int workers,
- int flexTarget,
+ int workerFlexTarget,
boolean testHBaseAfter) {
- createMiniCluster(clustername, configuration,
- 1,
- true);
+ SliderClient sliderClient = startHBaseCluster(clustername, masters, workers)
+
+ //now flex
+ return flexCluster(
+ sliderClient,
+ clustername,
+ masterFlexTarget,
+ workerFlexTarget,
+ testHBaseAfter)
+
+ }
+
+ public SliderClient startHBaseCluster(
+ String clustername,
+ int masters,
+ int workers) {
+ clustername = createMiniCluster(clustername, configuration,
+ 1,
+ true);
//now launch the cluster
SliderClient sliderClient;
ServiceLauncher<SliderClient> launcher = createCluster(clustername,
- [
- (ROLE_MASTER): masters,
- (ROLE_WORKER): workers,
- ],
- [
- ARG_RES_COMP_OPT , ROLE_MASTER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_RES_COMP_OPT , ROLE_WORKER, ResourceKeys.YARN_MEMORY, YRAM,
- ARG_PROVIDER , PROVIDER_HBASE
- ],
- true,
- true,
- [:]);
+ [
+ (ROLE_MASTER): masters,
+ (ROLE_WORKER): workers,
+ ],
+ hbaseArgs([]),
+ true,
+ true,
+ [:]);
sliderClient = launcher.service;
- try {
- basicHBaseClusterStartupSequence(sliderClient);
-
- describe("Waiting for initial worker count of $workers");
-
- //verify the #of roles is as expected
- //get the hbase status
- waitForWorkerInstanceCount(sliderClient, workers, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masters, hbaseClusterStartupToLiveTime);
-
- log.info("Slider worker count at $workers, waiting for region servers to match");
- waitForHBaseRegionServerCount(sliderClient, clustername, workers, hbaseClusterStartupToLiveTime);
-
- //now flex
- describe("Flexing masters:$masters -> $masterFlexTarget ; workers $workers -> $flexTarget");
- boolean flexed;
- flexed = 0 == sliderClient.flex(clustername,
- [
- (ROLE_WORKER): flexTarget,
- (ROLE_MASTER): masterFlexTarget
- ]
- );
- waitForWorkerInstanceCount(sliderClient, flexTarget, hbaseClusterStartupToLiveTime);
- waitForSliderMasterCount(sliderClient, masterFlexTarget,
- hbaseClusterStartupToLiveTime);
-
- if (testHBaseAfter) {
- waitForHBaseRegionServerCount(sliderClient, clustername, flexTarget,
- hbaseClusterStartupToLiveTime);
- }
- return flexed;
- } finally {
- maybeStopCluster(sliderClient, null, "end of flex test run");
- }
+ basicHBaseClusterStartupSequence(sliderClient);
+
+ describe("Waiting for initial worker count of $workers");
+
+ //verify the #of roles is as expected
+ //get the hbase status
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(
+ sliderClient,
+ masters,
+ hbaseClusterStartupToLiveTime);
+
+ log.info(
+ "Slider worker count at $workers, waiting for region servers to match");
+ waitForHBaseRegionServerCount(
+ sliderClient,
+ clustername,
+ workers,
+ hbaseClusterStartupToLiveTime);
+ sliderClient
+ }
+
+ public boolean flexCluster(
+ SliderClient sliderClient,
+ String clustername,
+ int masterFlexTarget,
+ int workerFlexTarget,
+ boolean testHBaseAfter) {
+ int flexTarget
+ describe(
+ "Flexing masters -> $masterFlexTarget ; workers -> ${workerFlexTarget}");
+ boolean flexed;
+ flexed = 0 == sliderClient.flex(clustername,
+ [
+ (ROLE_WORKER): workerFlexTarget,
+ (ROLE_MASTER): masterFlexTarget
+ ]
+ );
+ waitForWorkerInstanceCount(
+ sliderClient,
+ workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ waitForSliderMasterCount(sliderClient, masterFlexTarget,
+ hbaseClusterStartupToLiveTime);
+
+ if (testHBaseAfter) {
+ waitForHBaseRegionServerCount(sliderClient, clustername, workerFlexTarget,
+ hbaseClusterStartupToLiveTime);
+ }
+ flexed
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
index 680286c..683c02a 100644
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlex2To5.groovy
@@ -31,7 +31,7 @@ import org.junit.Test
class TestClusterFlex2To5 extends HBaseMiniClusterTestBase {
@Test
- public void testClusterFlex1To5() throws Throwable {
+ public void testClusterFlex2To5() throws Throwable {
assert flexHBaseClusterTestRun(
"",
1,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/34c00e86/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
new file mode 100644
index 0000000..43ade73
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexDownMultiple.groovy
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.client.SliderClient
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexDownMultiple extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterFlexDownMultiple() throws Throwable {
+ def clusterName = createClusterName();
+ SliderClient sliderClient = startHBaseCluster(clusterName, 1, 4)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 3,
+ true)
+
+ assert flexCluster(
+ sliderClient,
+ clusterName,
+ 1,
+ 1,
+ true)
+
+
+
+ }
+
+
+}
[06/18] git commit: SLIDER-270 improving diags and failure log
generation in AppState release operations
Posted by st...@apache.org.
SLIDER-270 improving diags and failure log generation in AppState release operations
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/268253c1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/268253c1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/268253c1
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 268253c1bb1582d4d1879c4798064e59bb447da0
Parents: ae4d073
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:07:23 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:08:05 2014 +0100
----------------------------------------------------------------------
.../slider/server/appmaster/state/AppState.java | 69 +++++++++++++-------
.../server/appmaster/state/RoleStatus.java | 4 +-
2 files changed, 46 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
index 9f8a39d..74e91a0 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/AppState.java
@@ -696,7 +696,7 @@ public class AppState {
roleStatusMap.get(priority));
}
roleStatusMap.put(priority,
- new RoleStatus(providerRole));
+ new RoleStatus(providerRole));
roles.put(providerRole.name, providerRole);
}
@@ -1175,10 +1175,14 @@ public class AppState {
log.info("Container was queued for release");
Container container = containersBeingReleased.remove(containerId);
RoleStatus roleStatus = lookupRoleStatus(container);
- log.info("decrementing role count for role {}", roleStatus.getName());
- roleStatus.decReleasing();
- roleStatus.decActual();
- roleStatus.incCompleted();
+ int releasing = roleStatus.decReleasing();
+ int actual = roleStatus.decActual();
+ int completedCount = roleStatus.incCompleted();
+ log.info("decrementing role count for role {} to {}; releasing={}, completed={}",
+ roleStatus.getName(),
+ actual,
+ releasing,
+ completedCount);
roleHistory.onReleaseCompleted(container);
} else if (surplusNodes.remove(containerId)) {
@@ -1205,25 +1209,17 @@ public class AppState {
roleStatus.decActual();
boolean shortLived = isShortLived(roleInstance);
String message;
- if (roleInstance.container != null) {
- String user = null;
- try {
- user = SliderUtils.getCurrentUser().getShortUserName();
- } catch (IOException ignored) {
- }
- String completedLogsUrl = null;
- Container c = roleInstance.container;
- String url = logServerURL;
- if (user != null && SliderUtils.isSet(url)) {
- completedLogsUrl = url
- + "/" + c.getNodeId() + "/" + roleInstance.getContainerId() + "/ctx/" + user;
- }
- message = String.format("Failure %s on host %s" +
- (completedLogsUrl != null ? ", see %s" : ""), roleInstance.getContainerId(),
- c.getNodeId().getHost(), completedLogsUrl);
+ Container failedContainer = roleInstance.container;
+
+ //build the failure message
+ if (failedContainer != null) {
+ String completedLogsUrl = getLogsURLForContainer(failedContainer);
+ message = String.format("Failure %s on host %s: %s" +
+ roleInstance.getContainerId(),
+ failedContainer.getNodeId().getHost(),
+ completedLogsUrl);
} else {
- message = String.format("Failure %s",
- containerId.toString());
+ message = String.format("Failure %s", containerId);
}
roleStatus.noteFailed(message);
//have a look to see if it short lived
@@ -1231,8 +1227,8 @@ public class AppState {
roleStatus.incStartFailed();
}
- if (roleInstance.container != null) {
- roleHistory.onFailedContainer(roleInstance.container, shortLived);
+ if (failedContainer!= null) {
+ roleHistory.onFailedContainer(failedContainer, shortLived);
}
} catch (YarnRuntimeException e1) {
@@ -1270,6 +1266,29 @@ public class AppState {
return result;
}
+ /**
+ * Get the URL log for a container
+ * @param c container
+ * @return the URL or "" if it cannot be determined
+ */
+ protected String getLogsURLForContainer(Container c) {
+ if (c==null) {
+ return null;
+ }
+ String user = null;
+ try {
+ user = SliderUtils.getCurrentUser().getShortUserName();
+ } catch (IOException ignored) {
+ }
+ String completedLogsUrl = "";
+ String url = logServerURL;
+ if (user != null && SliderUtils.isSet(url)) {
+ completedLogsUrl = url
+ + "/" + c.getNodeId() + "/" + c.getId() + "/ctx/" + user;
+ }
+ return completedLogsUrl;
+ }
+
/**
* Return the percentage done that Slider is to have YARN display in its
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/268253c1/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
index 04d8b37..b3dddf6 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/state/RoleStatus.java
@@ -175,8 +175,8 @@ public final class RoleStatus implements Cloneable {
this.completed = completed;
}
- public void incCompleted() {
- completed ++;
+ public int incCompleted() {
+ return completed ++;
}
public int getStarted() {
return started;
[07/18] git commit: SLIDER-284 switch branch-2 Hadoop version to
2.6.0-SNAPSHOT
Posted by st...@apache.org.
SLIDER-284 switch branch-2 Hadoop version to 2.6.0-SNAPSHOT
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/040ef798
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/040ef798
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/040ef798
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: 040ef79802d0f9c3a77a4b64a5fbea0d05f67fb0
Parents: 268253c
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:08:38 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:08:38 2014 +0100
----------------------------------------------------------------------
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/040ef798/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 7cca66c..253d40f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1264,7 +1264,7 @@
<!-- hadoop branch-2 builds -->
<id>branch-2</id>
<properties>
- <hadoop.version>2.5.0-SNAPSHOT</hadoop.version>
+ <hadoop.version>2.6.0-SNAPSHOT</hadoop.version>
</properties>
</profile>
[18/18] git commit: Merge branch 'develop' into
feature/SLIDER-284-hadoop2.5
Posted by st...@apache.org.
Merge branch 'develop' into feature/SLIDER-284-hadoop2.5
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ef22cb29
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ef22cb29
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ef22cb29
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: ef22cb2965a2216b98a8bdfa2db98cceb2598ee2
Parents: 248b6dd d68be5a
Author: Steve Loughran <st...@apache.org>
Authored: Wed Aug 6 09:35:10 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Aug 6 09:35:10 2014 +0100
----------------------------------------------------------------------
app-packages/hbase/appConfig.json | 2 +-
app-packages/hbase/metainfo.xml | 15 +
.../hbase/package/scripts/hbase_service.py | 2 +
.../hbase/package/scripts/hbase_thrift.py | 62 +++
app-packages/hbase/package/scripts/params.py | 1 +
app-packages/hbase/resources.json | 5 +
app-packages/memcached-win/appConfig.json | 6 +-
app-packages/memcached-win/metainfo.xml | 8 +-
.../memcached-win/package/scripts/memcached.py | 2 +-
.../memcached-win/package/scripts/params.py | 1 +
app-packages/memcached/appConfig.json | 7 +-
app-packages/memcached/metainfo.xml | 8 +-
.../memcached/package/scripts/memcached.py | 2 +-
.../memcached/package/scripts/params.py | 1 +
pom.xml | 2 +-
.../python/agent/CustomServiceOrchestrator.py | 86 +++-
.../core/providers/windows/system.py | 54 +-
.../agent/TestCustomServiceOrchestrator.py | 127 ++++-
slider-agent/src/test/python/agent/TestGrep.py | 2 +-
slider-agent/src/test/python/agent/TestMain.py | 8 +-
.../src/test/python/agent/TestRegistration.py | 4 +-
.../org/apache/slider/client/SliderClient.java | 10 +-
.../common/params/AbstractActionArgs.java | 14 +-
.../org/apache/slider/common/params/ArgOps.java | 7 +-
.../apache/slider/common/params/ClientArgs.java | 5 +-
.../apache/slider/common/params/CommonArgs.java | 11 +-
.../providers/AbstractProviderService.java | 5 +
.../slider/providers/ProviderService.java | 7 +
.../providers/agent/AgentProviderService.java | 510 ++++++++++++-------
.../agent/application/metadata/Application.java | 9 +
.../agent/application/metadata/Component.java | 13 +
.../application/metadata/MetainfoParser.java | 5 +
.../server/appmaster/SliderAppMaster.java | 22 +-
.../slider/server/appmaster/state/AppState.java | 142 +++---
.../server/appmaster/state/RoleHistory.java | 6 +-
.../server/appmaster/state/RoleStatus.java | 4 +-
.../appstate/TestAppStateDynamicRoles.groovy | 4 +-
.../TestAppStateRebuildOnAMRestart.groovy | 4 +-
.../model/appstate/TestFlexDynamicRoles.groovy | 2 +
.../model/appstate/TestMockFlexing.groovy | 128 +++++
.../model/appstate/TestMockRMOperations.groovy | 37 --
.../TestRoleHistoryContainerEvents.groovy | 2 +-
.../model/mock/BaseMockAppStateTest.groovy | 57 ++-
.../model/mock/MockProviderService.groovy | 5 +
.../appmaster/model/mock/MockYarnEngine.groovy | 6 +-
.../slider/test/YarnMiniClusterTestBase.groovy | 7 +-
.../agent/TestAgentProviderService.java | 137 ++++-
.../web/rest/agent/TestAMAgentWebServices.java | 3 +-
.../management/TestAMManagementWebServices.java | 1 +
.../publisher/TestAgentProviderService.java | 4 +-
.../providers/hbase/HBaseClientProvider.java | 2 +-
.../slider/providers/hbase/HBaseKeys.java | 5 +
.../providers/hbase/HBaseProviderService.java | 5 +
.../slider/providers/hbase/HBaseRoles.java | 3 +
.../minicluster/HBaseMiniClusterTestBase.groovy | 150 +++---
.../flexing/TestClusterFlex2To5.groovy | 2 +-
.../flexing/TestClusterFlexDownMultiple.groovy | 58 +++
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++
.../flexing/TestHMasterFlex1To2.groovy | 45 --
.../src/test/resources/log4j.properties | 1 +
60 files changed, 1392 insertions(+), 496 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ef22cb29/pom.xml
----------------------------------------------------------------------
[05/18] git commit: SLIDER-270 rename test so that TestClusterFlex\*
is a wildcard for all flex tests
Posted by st...@apache.org.
SLIDER-270 rename test so that TestClusterFlex\* is a wildcard for all flex tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/ae4d0734
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/ae4d0734
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/ae4d0734
Branch: refs/heads/feature/SLIDER-284-hadoop2.5
Commit: ae4d07340b0c0fe54b3dc75e58744393d5949d31
Parents: 34c00e8
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 5 21:06:54 2014 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 5 21:06:54 2014 +0100
----------------------------------------------------------------------
.../TestClusterFlexHMasterFlex1To2.groovy | 45 ++++++++++++++++++++
.../flexing/TestHMasterFlex1To2.groovy | 45 --------------------
2 files changed, 45 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
new file mode 100644
index 0000000..298f40f
--- /dev/null
+++ b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestClusterFlexHMasterFlex1To2.groovy
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.hbase.minicluster.flexing
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
+import org.junit.Test
+
+/**
+ * Create a master against the File:// fs
+ */
+@CompileStatic
+@Slf4j
+class TestClusterFlexHMasterFlex1To2 extends HBaseMiniClusterTestBase {
+
+ @Test
+ public void testClusterMasterFlex1To2() throws Throwable {
+ assert flexHBaseClusterTestRun(
+ "",
+ 1,
+ 2,
+ 1,
+ 1,
+ true)
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/ae4d0734/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
----------------------------------------------------------------------
diff --git a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy b/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
deleted file mode 100644
index ca26572..0000000
--- a/slider-providers/hbase/slider-hbase-provider/src/test/groovy/org/apache/slider/providers/hbase/minicluster/flexing/TestHMasterFlex1To2.groovy
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.slider.providers.hbase.minicluster.flexing
-
-import groovy.transform.CompileStatic
-import groovy.util.logging.Slf4j
-import org.apache.slider.providers.hbase.minicluster.HBaseMiniClusterTestBase
-import org.junit.Test
-
-/**
- * Create a master against the File:// fs
- */
-@CompileStatic
-@Slf4j
-class TestHMasterFlex1To2 extends HBaseMiniClusterTestBase {
-
- @Test
- public void testClusterFlex1To2() throws Throwable {
- assert flexHBaseClusterTestRun(
- "",
- 1,
- 2,
- 1,
- 1,
- true)
- }
-
-
-}