You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2015/03/18 05:10:26 UTC
[1/5] incubator-slider git commit: SLIDER-815 Binary files (dlls and
exes) in git source and Slider source release (both .zip and .tar.gz)
Repository: incubator-slider
Updated Branches:
refs/heads/feature/package_simplification_II 38fed9935 -> df3522598
SLIDER-815 Binary files (dlls and exes) in git source and Slider source release (both .zip and .tar.gz)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/601ac135
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/601ac135
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/601ac135
Branch: refs/heads/feature/package_simplification_II
Commit: 601ac135215aa183bfea9861ae7a62ec9688980d
Parents: 38fed99
Author: Gour Saha <go...@apache.org>
Authored: Mon Mar 16 11:02:56 2015 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Mon Mar 16 11:02:56 2015 -0700
----------------------------------------------------------------------
.../hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll | Bin 92672 -> 0 bytes
bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll | Bin 57344 -> 0 bytes
.../hadoop-2.6.0-SNAPSHOT/bin/winutils.exe | Bin 108032 -> 0 bytes
bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md | 22 -------------------
bin/windows/hadoop-2.6.0/bin/hadoop.dll | Bin 92672 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hadoop.pdb | Bin 502784 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hdfs.dll | Bin 57344 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hdfs.pdb | Bin 347136 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/winutils.exe | Bin 108032 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/winutils.pdb | Bin 896000 -> 0 bytes
10 files changed, 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll b/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll
deleted file mode 100644
index 8da86d2..0000000
Binary files a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll b/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll
deleted file mode 100644
index d96248c..0000000
Binary files a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/winutils.exe
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/winutils.exe b/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/winutils.exe
deleted file mode 100644
index a4edc85..0000000
Binary files a/bin/windows/hadoop-2.6.0-SNAPSHOT/bin/winutils.exe and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md b/bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md
deleted file mode 100644
index 9e4dd52..0000000
--- a/bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md
+++ /dev/null
@@ -1,22 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-# Windows Binaries
-
-This directory contains the native libraries needed for running Hadoop on
-windows. It's here so that jenkins builds on windows do not
-need to rely on a pre-installed version of hadoop.
-
-To ise
-
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/hadoop.dll
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/hadoop.dll b/bin/windows/hadoop-2.6.0/bin/hadoop.dll
deleted file mode 100644
index 78ad8a8..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/hadoop.dll and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/hadoop.pdb
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/hadoop.pdb b/bin/windows/hadoop-2.6.0/bin/hadoop.pdb
deleted file mode 100644
index 3701ec8..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/hadoop.pdb and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/hdfs.dll
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/hdfs.dll b/bin/windows/hadoop-2.6.0/bin/hdfs.dll
deleted file mode 100644
index 9ce72c8..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/hdfs.dll and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/hdfs.pdb
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/hdfs.pdb b/bin/windows/hadoop-2.6.0/bin/hdfs.pdb
deleted file mode 100644
index 8d4c06c..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/hdfs.pdb and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/winutils.exe
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/winutils.exe b/bin/windows/hadoop-2.6.0/bin/winutils.exe
deleted file mode 100644
index 6fbbd20..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/winutils.exe and /dev/null differ
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/601ac135/bin/windows/hadoop-2.6.0/bin/winutils.pdb
----------------------------------------------------------------------
diff --git a/bin/windows/hadoop-2.6.0/bin/winutils.pdb b/bin/windows/hadoop-2.6.0/bin/winutils.pdb
deleted file mode 100644
index bab211f..0000000
Binary files a/bin/windows/hadoop-2.6.0/bin/winutils.pdb and /dev/null differ
[2/5] incubator-slider git commit: SLIDER-812 Making component
configurations in appConfig available on the SliderAgent side (Thomas Liu via
gourksaha)
Posted by sm...@apache.org.
SLIDER-812 Making component configurations in appConfig available on the SliderAgent side (Thomas Liu via gourksaha)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/96177ed0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/96177ed0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/96177ed0
Branch: refs/heads/feature/package_simplification_II
Commit: 96177ed0c3822a96944d5c2296788937fdbcf51b
Parents: 601ac13
Author: Gour Saha <go...@apache.org>
Authored: Tue Mar 17 15:34:59 2015 -0700
Committer: Gour Saha <go...@apache.org>
Committed: Tue Mar 17 15:34:59 2015 -0700
----------------------------------------------------------------------
.../providers/agent/AgentProviderService.java | 17 ++++++++++++++++-
.../appmaster/web/rest/agent/ExecutionCommand.java | 10 ++++++++++
2 files changed, 26 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96177ed0/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bdf168e..8f6f3bf 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -20,6 +20,7 @@ package org.apache.slider.providers.agent;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -1659,10 +1660,20 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
+ Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
+ cmd.setComponentConfigurations(componentConfigurations);
+
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
response.addExecutionCommand(cmd);
+
+ log.debug("command looks like: " + cmd.toString());
+ }
+
+ private Map<String, Map<String, String>> buildComponentConfigurations(
+ ConfTreeOperations appConf) {
+ return appConf.getComponents();
}
protected static String getPackageListFromApplication(Application application) {
@@ -1795,11 +1806,15 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true));
+ Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
+ cmd.setComponentConfigurations(componentConfigurations);
+
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
response.addExecutionCommand(cmd);
-
+
+ log.debug("command looks like: " + cmd.toString());
// With start command, the corresponding command for graceful stop needs to
// be sent. This will be used when a particular container is lost as per RM,
// but then the agent is still running and heart-beating to the Slider AM.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/96177ed0/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index 5fb3b5e..6144227 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -43,6 +43,7 @@ public class ExecutionCommand {
private Map<String, String> roleParams = null;
private String roleCommand;
private Map<String, Map<String, String>> configurations;
+ private Map<String, Map<String, String>> componentConfigurations;
private Map<String, String> commandParams;
private String serviceName;
private String componentName;
@@ -181,6 +182,15 @@ public class ExecutionCommand {
this.componentName = componentName;
}
+ public Map<String, Map<String, String>> getComponentConfigurations() {
+ return componentConfigurations;
+ }
+
+ public void setComponentConfigurations(
+ Map<String, Map<String, String>> componentConfigurations) {
+ this.componentConfigurations = componentConfigurations;
+ }
+
@Override
public String toString() {
StringBuilder builder = new StringBuilder();
[5/5] incubator-slider git commit: Merge branch 'develop' into
feature/package_simplification_II
Posted by sm...@apache.org.
Merge branch 'develop' into feature/package_simplification_II
Conflicts:
slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/df352259
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/df352259
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/df352259
Branch: refs/heads/feature/package_simplification_II
Commit: df35225987ca7c9e521f64af49c1b3ed3c03af43
Parents: 704e813 96177ed
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Mar 17 21:08:32 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Mar 17 21:08:32 2015 -0700
----------------------------------------------------------------------
.../hadoop-2.6.0-SNAPSHOT/bin/hadoop.dll | Bin 92672 -> 0 bytes
bin/windows/hadoop-2.6.0-SNAPSHOT/bin/hdfs.dll | Bin 57344 -> 0 bytes
.../hadoop-2.6.0-SNAPSHOT/bin/winutils.exe | Bin 108032 -> 0 bytes
bin/windows/hadoop-2.6.0-SNAPSHOT/readme.md | 22 -------------------
bin/windows/hadoop-2.6.0/bin/hadoop.dll | Bin 92672 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hadoop.pdb | Bin 502784 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hdfs.dll | Bin 57344 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/hdfs.pdb | Bin 347136 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/winutils.exe | Bin 108032 -> 0 bytes
bin/windows/hadoop-2.6.0/bin/winutils.pdb | Bin 896000 -> 0 bytes
.../providers/agent/AgentProviderService.java | 11 +++++++++-
.../web/rest/agent/ExecutionCommand.java | 10 +++++++++
12 files changed, 20 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df352259/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --cc slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 4eadf13,8f6f3bf..d2298f0
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@@ -1698,21 -1660,20 +1699,28 @@@ public class AgentProviderService exten
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
- Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
+ } else {
+ // assume it to be default shell command
+ ComponentCommand effectiveCommand = compCmd;
+ if(effectiveCommand == null) {
+ effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
+ }
+ cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
+ configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
+ }
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
+
response.addExecutionCommand(cmd);
+
+ log.debug("command looks like: " + cmd.toString());
+ }
+
+ private Map<String, Map<String, String>> buildComponentConfigurations(
+ ConfTreeOperations appConf) {
+ return appConf.getComponents();
}
protected static String getPackageListFromApplication(Application application) {
@@@ -1864,21 -1804,17 +1872,22 @@@
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true));
-
- Map<String, Map<String, String>> componentConfigurations = buildComponentConfigurations(appConf);
- cmd.setComponentConfigurations(componentConfigurations);
-
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
-
cmd.setConfigurations(configurations);
+
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
+ } else {
+ if(startCommand == null) {
+ throw new SliderException("Expected START command not found for component " + componentName);
+ }
+ cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
+ configurations.get("global").put("exec_cmd", startCommand.getExec());
+ }
+
response.addExecutionCommand(cmd);
-
+
+ log.debug("command looks like: " + cmd.toString());
// With start command, the corresponding command for graceful stop needs to
// be sent. This will be used when a particular container is lost as per RM,
// but then the agent is still running and heart-beating to the Slider AM.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/df352259/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
[4/5] incubator-slider git commit: SLIDER-663. Make it easy to
develop and deploy application packages that are essentially shell commands
(part-I)
Posted by sm...@apache.org.
SLIDER-663. Make it easy to develop and deploy application packages that are essentially shell commands (part-I)
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/704e8136
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/704e8136
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/704e8136
Branch: refs/heads/feature/package_simplification_II
Commit: 704e81363a72444f0d27b6fe3ab43d46b7791231
Parents: 38fed99
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Tue Mar 17 17:49:49 2015 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Tue Mar 17 17:49:49 2015 -0700
----------------------------------------------------------------------
.../python/agent/CustomServiceOrchestrator.py | 56 ++++-
.../python/scripts/shell_cmd/basic_installer.py | 55 +++++
.../src/main/python/scripts/shell_cmd/params.py | 29 +++
.../agent/TestCustomServiceOrchestrator.py | 104 ++++++++-
.../org/apache/slider/api/InternalKeys.java | 8 +
.../org/apache/slider/client/SliderClient.java | 22 +-
.../org/apache/slider/common/SliderKeys.java | 12 +-
.../AbstractClusterBuildingActionArgs.java | 31 ++-
.../slider/common/params/AddonArgsDelegate.java | 54 +++++
.../apache/slider/common/params/Arguments.java | 3 +
.../slider/common/tools/CoreFileSystem.java | 27 ++-
.../apache/slider/common/tools/SliderUtils.java | 56 ++++-
.../slider/core/build/InstanceBuilder.java | 6 +
.../core/persist/AppDefinitionPersister.java | 174 +++++++++++++++
.../slider/core/persist/InstancePaths.java | 4 +
.../providers/agent/AgentClientProvider.java | 57 +++--
.../slider/providers/agent/AgentKeys.java | 2 +
.../providers/agent/AgentProviderService.java | 211 ++++++++++++++-----
.../slider/providers/agent/AgentUtils.java | 30 ++-
.../providers/agent/ComponentCommandOrder.java | 4 +-
.../agent/application/metadata/Application.java | 94 +++++++--
.../application/metadata/CommandOrder.java | 9 +-
.../application/metadata/CommandScript.java | 11 +-
.../agent/application/metadata/Component.java | 83 ++++++--
.../application/metadata/ComponentCommand.java | 85 ++++++++
.../agent/application/metadata/ConfigFile.java | 8 +-
.../agent/application/metadata/Export.java | 9 +-
.../agent/application/metadata/ExportGroup.java | 11 +-
.../agent/application/metadata/Metainfo.java | 24 ++-
.../application/metadata/MetainfoParser.java | 62 +++++-
.../agent/application/metadata/OSPackage.java | 9 +-
.../agent/application/metadata/OSSpecific.java | 11 +-
.../agent/application/metadata/Package.java | 60 ++++++
.../agent/application/metadata/Validate.java | 27 +++
.../web/rest/agent/ExecutionCommand.java | 18 +-
.../test_min_pkg/nc_ping_cmd/appConfig.json | 8 +
.../test_min_pkg/nc_ping_cmd/metainfo.json | 28 +++
.../test_min_pkg/nc_ping_cmd/resources.json | 17 ++
.../test_min_pkg/sleep_cmd/metainfo.json | 16 ++
.../test_min_pkg/sleep_cmd/resources.json | 17 ++
.../agent/TestAgentClientProvider2.java | 12 +-
.../agent/TestAgentProviderService.java | 129 +++++++-----
.../metadata/TestMetainfoParser.java | 98 ++++++++-
.../funtest/framework/CommandTestBase.groovy | 80 +++++++
.../funtest/lifecycle/AgentMinSleepIT.groovy | 110 ++++++++++
.../funtest/lifecycle/AgentPingSocketIT.groovy | 142 +++++++++++++
46 files changed, 1907 insertions(+), 216 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 119c926..87ce621 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -76,23 +76,37 @@ class CustomServiceOrchestrator():
override_output_files=True, store_command=False):
allocated_ports = {}
try:
+ py_file_list = []
+ json_path = None
+
script_type = command['commandParams']['script_type']
- script = command['commandParams']['script']
- timeout = int(command['commandParams']['command_timeout'])
task_id = command['taskId']
command_name = command['roleCommand']
- script_path = self.resolve_script_path(self.base_dir, script, script_type)
- script_tuple = (script_path, self.base_dir)
-
tmpstrucoutfile = os.path.realpath(posixpath.join(self.tmp_dir,
"structured-out-{0}.json".format(task_id)))
- if script_type.upper() != self.SCRIPT_TYPE_PYTHON:
- # We don't support anything else yet
+ if script_type.upper() == self.SCRIPT_TYPE_PYTHON:
+ script = command['commandParams']['script']
+ timeout = int(command['commandParams']['command_timeout'])
+ script_path = self.resolve_script_path(self.base_dir, script, script_type)
+ script_tuple = (script_path, self.base_dir)
+ py_file_list = [script_tuple]
+
+ json_path = self.dump_command_to_json(command, allocated_ports, store_command)
+ elif script_type.upper() == "SHELL":
+ timeout = int(command['commandParams']['command_timeout'])
+
+ json_path = self.dump_command_to_json(command, allocated_ports, store_command)
+ script_path = os.path.realpath(posixpath.join(self.config.getWorkRootPath(),
+ "infra", "agent", "slider-agent", "scripts",
+ "shell_cmd", "basic_installer.py"))
+ script_tuple = (script_path, self.base_dir)
+ py_file_list = [script_tuple]
+ else:
+ # We don't support anything else yet
message = "Unknown script type {0}".format(script_type)
raise AgentException(message)
- json_path = self.dump_command_to_json(command, allocated_ports, store_command)
- py_file_list = [script_tuple]
+
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
logger_level = logging.getLevelName(logger.level)
@@ -239,6 +253,7 @@ class CustomServiceOrchestrator():
os.unlink(file_path)
self.finalize_command(command, store_command, allocated_ports)
+ self.finalize_exec_command(command)
with os.fdopen(os.open(file_path, os.O_WRONLY | os.O_CREAT,
0644), 'w') as f:
@@ -246,6 +261,7 @@ class CustomServiceOrchestrator():
f.write(content)
return file_path
+
"""
patch content
${AGENT_WORK_ROOT} -> AgentConfig.getWorkRootPath()
@@ -291,6 +307,28 @@ class CustomServiceOrchestrator():
pass
"""
+ configurations/global/exec_cmd should be resolved based on the rest of the config
+ {$conf:@//site/global/xmx_val} ==> command['configurations']['global']['xmx_val']
+ """
+ def finalize_exec_command(self, command):
+ variable_format = "{{$conf:@//site/{0}/{1}}}"
+ if 'configurations' in command:
+ if 'global' in command['configurations'] and 'exec_cmd' in command['configurations']['global']:
+ exec_cmd = command['configurations']['global']['exec_cmd']
+ for key in command['configurations']:
+ if len(command['configurations'][key]) > 0:
+ for k, value in command['configurations'][key].items():
+ replaced_key = variable_format.format(key, k)
+ exec_cmd = exec_cmd.replace(replaced_key, value)
+ pass
+ pass
+ pass
+ command['configurations']['global']['exec_cmd'] = exec_cmd
+ pass
+ pass
+
+
+ """
All unallocated ports should be set to 0
Look for "${SOME_COMPONENT_NAME.ALLOCATED_PORT}"
or "${component.ALLOCATED_PORT}{DEFAULT_port}"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
new file mode 100644
index 0000000..561fd6c
--- /dev/null
+++ b/slider-agent/src/main/python/scripts/shell_cmd/basic_installer.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+class BasicInstaller(Script):
+ def install(self, env):
+ self.install_packages(env)
+ pass
+
+ def configure(self, env):
+ pass
+
+ def start(self, env):
+ import params
+ env.set_params(params)
+ self.configure(env)
+ process_cmd = format("{cmd}")
+
+ Execute(process_cmd,
+ logoutput=False,
+ wait_for_finish=False,
+ pid_file=params.pid_file,
+ poll_after = 5
+ )
+
+ def stop(self, env):
+ import params
+ env.set_params(params)
+
+ def status(self, env):
+ import params
+ env.set_params(params)
+ check_process_status(params.pid_file)
+
+if __name__ == "__main__":
+ BasicInstaller().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/main/python/scripts/shell_cmd/params.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/scripts/shell_cmd/params.py b/slider-agent/src/main/python/scripts/shell_cmd/params.py
new file mode 100644
index 0000000..9051f43
--- /dev/null
+++ b/slider-agent/src/main/python/scripts/shell_cmd/params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+
+app_root = config['configurations']['global']['app_root']
+pid_file = config['configurations']['global']['pid_file']
+exec_cmd = config['configurations']['global']['exec_cmd']
+cmd = format(exec_cmd)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 30c8d7a..aaef3f9 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -23,15 +23,13 @@ import pprint
from unittest import TestCase
import unittest
-import threading
import tempfile
-import time
+import posixpath
import logging
-from threading import Thread
from PythonExecutor import PythonExecutor
from CustomServiceOrchestrator import CustomServiceOrchestrator
-from mock.mock import MagicMock, patch
+from mock.mock import MagicMock, patch, call
import StringIO
import sys
from socket import socket
@@ -438,7 +436,6 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual.__self__.maxDiff = None
self.assertEqual(ret['exitcode'], 0)
self.assertTrue(run_file_mock.called)
- self.assertEqual(orchestrator.stored_command, command)
ret = orchestrator.requestComponentStatus(command_get)
self.assertEqual(ret['configurations'], expected)
@@ -447,6 +444,103 @@ class TestCustomServiceOrchestrator(TestCase):
self.assertEqual(ret['configurations'], expected_specific)
pass
+ @patch("hostname.public_hostname")
+ @patch("os.path.isfile")
+ @patch("os.unlink")
+ @patch.object(PythonExecutor, "run_file")
+ def test_runCommand_with_shell_config(self,
+ run_file_mock,
+ unlink_mock,
+ isfile_mock,
+ hostname_mock):
+ hostname_mock.return_value = "test.hst"
+ isfile_mock.return_value = True
+ command = {
+ 'role': 'MEMCACHED',
+ 'componentName': 'MEMCACHED',
+ 'hostLevelParams': {
+ 'jdk_location': 'some_location'
+ },
+ 'commandParams': {
+ 'script_type': 'SHELL',
+ 'command_timeout': '600'
+ },
+ 'configurations': {
+ "memcached-site": {
+ "memcached.log": "${AGENT_LOG_ROOT}",
+ "memcached.number": "10485760"},
+ "memcached-log4j": {"a": "b"}
+ },
+ 'taskId': '3',
+ 'roleCommand': 'INSTALL',
+ 'commandType': 'EXECUTION_COMMAND',
+ 'commandId': '1-1'
+ }
+
+ command_get = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND'
+ }
+
+ command_get_specific = {
+ 'roleCommand': 'GET_CONFIG',
+ 'commandType': 'STATUS_COMMAND',
+ 'commandParams': {
+ 'config_type': 'memcached-site'
+ }
+ }
+
+ tempdir = tempfile.gettempdir()
+ config = MagicMock()
+ config.get.return_value = "something"
+ config.getResolvedPath.return_value = tempdir
+ config.getWorkRootPath.return_value = tempdir
+ config.getLogPath.return_value = tempdir
+
+ dummy_controller = MagicMock()
+ orchestrator = CustomServiceOrchestrator(config, dummy_controller, self.agentToggleLogger)
+ # normal run case
+ run_file_mock.return_value = {
+ 'stdout': 'sss',
+ 'stderr': 'eee',
+ 'exitcode': 0,
+ }
+
+ expected = {
+ 'memcached-site': {
+ 'memcached.log': tempdir, 'memcached.number': '10485760'},
+ 'memcached-log4j': {'a': 'b'}}
+
+ expected_specific = {
+ 'memcached-site': {
+ 'memcached.log': tempdir, 'memcached.number': '10485760'},
+ }
+
+ ret = orchestrator.runCommand(command, "out.txt", "err.txt", True, True)
+ self.assertEqual.__self__.maxDiff = None
+ self.assertEqual(ret['exitcode'], 0)
+ self.assertTrue(run_file_mock.called)
+
+ ret = orchestrator.requestComponentStatus(command_get)
+ self.assertEqual(ret['configurations'], expected)
+
+ ret = orchestrator.requestComponentStatus(command_get_specific)
+ self.assertEqual(ret['configurations'], expected_specific)
+
+ script_path = os.path.realpath(posixpath.join(tempdir,
+ "infra", "agent", "slider-agent", "scripts",
+ "shell_cmd", "basic_installer.py"))
+ run_file_mock.assert_has_calls([call(
+ script_path,
+ ['INSTALL', os.path.realpath(posixpath.join(tempdir, 'command-3.json')),
+ os.path.realpath(posixpath.join(tempdir, 'package'))],
+ 'out.txt', 'err.txt', 600,
+ os.path.realpath(posixpath.join(tempdir, 'structured-out-3.json')),
+ 'INFO', True,
+ [('PYTHONPATH', ":".join([os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent', 'jinja2')),
+ os.path.realpath(posixpath.join(tempdir, 'infra', 'agent', 'slider-agent'))]))])])
+ pass
+
@patch.object(CustomServiceOrchestrator, "runCommand")
def test_requestComponentStatus(self, runCommand_mock):
status_command = {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
index b360fbe..fd17ed1 100644
--- a/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/api/InternalKeys.java
@@ -69,6 +69,14 @@ public interface InternalKeys {
*/
String INTERNAL_DATA_DIR_PATH = "internal.data.dir.path";
/**
+ * where the app def is stored
+ */
+ String INTERNAL_APPDEF_DIR_PATH = "internal.appdef.dir.path";
+ /**
+ * where addons for the app are stored
+ */
+ String INTERNAL_ADDONS_DIR_PATH = "internal.addons.dir.path";
+ /**
* Time in milliseconds to wait after forking any in-AM
* process before attempting to start up the containers: {@value}
*
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 84ed5a4..c23177a 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -20,6 +20,7 @@ package org.apache.slider.client;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
@@ -133,6 +134,7 @@ import org.apache.slider.core.launch.LaunchedApplication;
import org.apache.slider.core.launch.RunningApplication;
import org.apache.slider.core.launch.SerializedApplicationReport;
import org.apache.slider.core.main.RunService;
+import org.apache.slider.core.persist.AppDefinitionPersister;
import org.apache.slider.core.persist.ApplicationReportSerDeser;
import org.apache.slider.core.persist.ConfPersister;
import org.apache.slider.core.persist.LockAcquireFailedException;
@@ -179,7 +181,6 @@ import java.net.InetSocketAddress;
import java.net.URISyntaxException;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
-import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -1052,7 +1053,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
JSONObject config = null;
if(clientInfo.clientConfig != null) {
try {
- byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath());
+ byte[] encoded = Files.toByteArray(clientInfo.clientConfig);
config = new JSONObject(new String(encoded, Charset.defaultCharset()));
}catch(JSONException jsonEx) {
log.error("Unable to read supplied config", jsonEx);
@@ -1304,6 +1305,9 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
}
+ AppDefinitionPersister appDefinitionPersister = new AppDefinitionPersister(sliderFileSystem);
+ appDefinitionPersister.processSuppliedDefinitions(clustername, buildInfo, appConf);
+
//get the command line options
ConfTree cmdLineAppOptions = buildInfo.buildAppOptionsConfTree();
ConfTree cmdLineResourceOptions = buildInfo.buildResourceOptionsConfTree();
@@ -1395,19 +1399,25 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// make any substitutions needed at this stage
replaceTokens(appConf.getConfTree(), getUsername(), clustername);
- // providers to validate what there is
- AggregateConf instanceDescription = builder.getInstanceDescription();
- validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
- validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
+ // TODO: Refactor the validation code and persistence code
try {
persistInstanceDefinition(overwrite, appconfdir, builder);
+ appDefinitionPersister.persistPackages();
+
} catch (LockAcquireFailedException e) {
log.warn("Failed to get a Lock on {} : {}", builder, e);
throw new BadClusterStateException("Failed to save " + clustername
+ ": " + e);
}
+
+ // providers to validate what there is
+ // TODO: Validation should be done before persistence
+ AggregateConf instanceDescription = builder.getInstanceDescription();
+ validateInstanceDefinition(sliderAM, instanceDescription, sliderFileSystem);
+ validateInstanceDefinition(provider, instanceDescription, sliderFileSystem);
}
+
protected void persistInstanceDefinition(boolean overwrite,
Path appconfdir,
InstanceBuilder builder)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
index 5cf7022..30ff258 100644
--- a/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/common/SliderKeys.java
@@ -168,11 +168,21 @@ public interface SliderKeys extends SliderXmlConfKeys {
String TMP_LOGDIR_PREFIX = "/tmp/slider-";
String TMP_DIR_PREFIX = "tmp";
String AM_DIR_PREFIX = "appmaster";
-
+
+ /**
+ * Store the default app definition, e.g. metainfo file or content of a folder
+ */
+ String APP_DEF_DIR = "appdef";
+ /**
+ * Store additional app defs - co-processors
+ */
+ String ADDONS_DIR = "addons";
+
String SLIDER_JAR = "slider.jar";
String JCOMMANDER_JAR = "jcommander.jar";
String GSON_JAR = "gson.jar";
String AGENT_TAR = "slider-agent.tar.gz";
+ String DEFAULT_APP_PKG = "appPkg.zip";
String DEFAULT_JVM_HEAP = "256M";
int DEFAULT_YARN_MEMORY = 256;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
index 56e01c3..f2e3c61 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AbstractClusterBuildingActionArgs.java
@@ -74,13 +74,6 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Provider of the specific cluster application")
public String provider = SliderProviderFactory.DEFAULT_CLUSTER_TYPE;
- /*
-
- @Parameter(names = {ARG_PACKAGE},
- description = "URI to a slider package",
- converter = URIArgumentConverter.class )
- public URI packageURI;
- */
@Parameter(names = {ARG_PACKAGE},
description = "URI to a slider package")
public String packageURI;
@@ -93,6 +86,14 @@ public abstract class AbstractClusterBuildingActionArgs extends
description = "Template application configuration")
public File template;
+ @Parameter(names = {ARG_METAINFO},
+ description = "Application meta info")
+ public File appMetaInfo;
+
+ @Parameter(names = {ARG_APPDEF},
+ description = "Application def (folder or a zip package)")
+ public File appDef;
+
@Parameter(names = {ARG_QUEUE},
description = "Queue to submit the application")
public String queue;
@@ -100,6 +101,9 @@ public abstract class AbstractClusterBuildingActionArgs extends
@ParametersDelegate
public ComponentArgsDelegate componentDelegate = new ComponentArgsDelegate();
+ @ParametersDelegate
+ public AddonArgsDelegate addonDelegate = new AddonArgsDelegate();
+
@ParametersDelegate
public AppAndResouceOptionArgsDelegate optionsDelegate =
@@ -152,6 +156,19 @@ public abstract class AbstractClusterBuildingActionArgs extends
return componentDelegate.getComponentMap();
}
+ @VisibleForTesting
+ public List<String> getAddonTuples() {
+ return addonDelegate.getAddonTuples();
+ }
+
+ /**
+ * Get the list of addons (may be empty, but never null)
+ */
+ public Map<String, String> getAddonMap() throws
+ BadCommandArgumentsException {
+ return addonDelegate.getAddonMap();
+ }
+
public Path getConfdir() {
return confdir;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
new file mode 100644
index 0000000..65ebc4b
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/AddonArgsDelegate.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class AddonArgsDelegate extends AbstractArgsDelegate {
+
+ /**
+ * This is a listing of addon packages
+ */
+ @Parameter(names = {ARG_ADDON},
+ arity = 2,
+ description = "--addon <name> <folder or package>",
+ splitter = DontSplitArguments.class)
+ public List<String> addonTuples = new ArrayList<String>(0);
+
+
+ /**
+ * Get the list of addons (may be empty, but never null)
+ *
+ * @return map of named addons
+ *
+ * @throws BadCommandArgumentsException parse problem
+ */
+ public Map<String, String> getAddonMap() throws BadCommandArgumentsException {
+ return convertTupleListToMap("addon", addonTuples);
+ }
+
+ public List<String> getAddonTuples() {
+ return addonTuples;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 9164edc..ea393d5 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -113,6 +113,9 @@ public interface Arguments {
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
String ARG_CONFIG = "--config";
+ String ARG_METAINFO = "--metainfo";
+ String ARG_ADDON = "--addon";
+ String ARG_APPDEF = "--appdef";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
index e835312..3df73df 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/CoreFileSystem.java
@@ -122,6 +122,31 @@ public class CoreFileSystem {
}
/**
+ * Build up the path string for app def folder -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildAppDefDirPath(String clustername) {
+ Path path = buildClusterDirPath(clustername);
+ return new Path(path, SliderKeys.APP_DEF_DIR);
+ }
+
+ /**
+ * Build up the path string for addon folder -no attempt to
+ * create the directory is made
+ *
+ * @param clustername name of the cluster
+ * @return the path for persistent data
+ */
+ public Path buildAddonDirPath(String clustername, String addonId) {
+ Preconditions.checkNotNull(addonId);
+ Path path = buildClusterDirPath(clustername);
+ return new Path(path, SliderKeys.ADDONS_DIR + "/" + addonId);
+ }
+
+ /**
* Build up the path string for package install location -no attempt to
* create the directory is made
*
@@ -568,7 +593,7 @@ public class CoreFileSystem {
/**
* List all application instances persisted for this user, giving the
- * patha. The instance name is the last element in the path
+ * path. The instance name is the last element in the path
* @return a possibly empty map of application instance names to paths
*/
public Map<String, Path> listPersistentInstances() throws IOException {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
index ce52b89..1767d9c 100644
--- a/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/common/tools/SliderUtils.java
@@ -69,6 +69,7 @@ import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
import java.io.File;
+import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
@@ -107,6 +108,8 @@ import java.util.TreeMap;
import java.util.TreeSet;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
/**
* These are slider-specific Util methods
@@ -1732,7 +1735,58 @@ public final class SliderUtils {
}
/**
- * This wrapps ApplicationReports and generates a string version
+ * Given a source folder create zipped file
+ *
+ * @param srcFolder
+ * @param zipFile
+ *
+ * @throws IOException
+ */
+ public static void zipFolder(File srcFolder, File zipFile) throws IOException {
+ log.info("Zipping folder {} to {}", srcFolder.getAbsolutePath(), zipFile.getAbsolutePath());
+ List<String> files = new ArrayList<>();
+ generateFileList(files, srcFolder, srcFolder, true);
+
+ byte[] buffer = new byte[1024];
+
+ try (FileOutputStream fos = new FileOutputStream(zipFile)) {
+ try (ZipOutputStream zos = new ZipOutputStream(fos)) {
+
+ for (String file : files) {
+ ZipEntry ze = new ZipEntry(file);
+ zos.putNextEntry(ze);
+ try (FileInputStream in = new FileInputStream(srcFolder + File.separator + file)) {
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ zos.write(buffer, 0, len);
+ }
+ }
+ }
+ }
+ }
+ }
+
+
+ private static void generateFileList(List<String> fileList, File node, File rootFolder, Boolean relative) {
+ if (node.isFile()) {
+ String fileFullPath = node.toString();
+ if (relative) {
+ fileList.add(fileFullPath.substring(rootFolder.toString().length() + 1, fileFullPath.length()));
+ } else {
+ fileList.add(fileFullPath);
+ }
+ }
+
+ if (node.isDirectory()) {
+ String[] subNode = node.list();
+ for (String filename : subNode) {
+ generateFileList(fileList, new File(node, filename), rootFolder, relative);
+ }
+ }
+ }
+
+ /**
+ * This wraps ApplicationReports and generates a string version
* iff the toString() operator is invoked
*/
public static class OnDemandReportStringifier {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
index 6812613..0a8dcdb 100644
--- a/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
+++ b/slider-core/src/main/java/org/apache/slider/core/build/InstanceBuilder.java
@@ -49,6 +49,8 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Map;
+import static org.apache.slider.api.InternalKeys.INTERNAL_ADDONS_DIR_PATH;
+import static org.apache.slider.api.InternalKeys.INTERNAL_APPDEF_DIR_PATH;
import static org.apache.slider.api.InternalKeys.INTERNAL_QUEUE;
import static org.apache.slider.api.OptionKeys.INTERNAL_AM_TMP_DIR;
import static org.apache.slider.api.OptionKeys.INTERNAL_TMP_DIR;
@@ -142,6 +144,10 @@ public class InstanceBuilder {
instancePaths.generatedConfPath.toUri());
internalOps.set(INTERNAL_DATA_DIR_PATH,
instancePaths.dataPath.toUri());
+ internalOps.set(INTERNAL_APPDEF_DIR_PATH,
+ instancePaths.appDefPath.toUri());
+ internalOps.set(INTERNAL_ADDONS_DIR_PATH,
+ instancePaths.addonsPath.toUri());
internalOps.set(InternalKeys.INTERNAL_PROVIDER_NAME, provider);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
new file mode 100644
index 0000000..8f0f7b0
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/AppDefinitionPersister.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.core.persist;
+
+import com.google.common.io.Files;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.fs.Path;
+import org.apache.slider.common.SliderKeys;
+import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.tools.SliderFileSystem;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.conf.ConfTreeOperations;
+import org.apache.slider.core.exceptions.BadClusterStateException;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.BadConfigException;
+import org.apache.slider.providers.agent.AgentKeys;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Class to prepare and persist app and add-on definitions.
+ *
+ * In this case, the app definition and add-on definitions are auto-inferred from the user input rather than explicit
+ * inclusion of application package in the config.
+ *
+ * Processing an app definition involves one or more of the following: - modify appConfig - package definition into a
+ * temporary folder - upload to HDFS
+ *
+ * This class keeps track of all the required operations and allows them to be invoked by build operation
+ */
+public class AppDefinitionPersister {
+ private static final Logger log =
+ LoggerFactory.getLogger(AppDefinitionPersister.class);
+
+ private final SliderFileSystem sliderFileSystem;
+ private List<AppDefinition> appDefinitions;
+
+ public AppDefinitionPersister(SliderFileSystem sliderFileSystem) {
+ this.sliderFileSystem = sliderFileSystem;
+ appDefinitions = new ArrayList<>();
+ }
+
+
+ /**
+ * Process the application package or folder by copying it to the cluster path
+ *
+ * @param appDefinition details of application package
+ *
+ * @throws BadConfigException
+ * @throws IOException
+ */
+ private void persistDefinitionPackageOrFolder(AppDefinition appDefinition)
+ throws BadConfigException, IOException {
+ if (!appDefinition.appDefPkgOrFolder.canRead()) {
+ throw new BadConfigException("Pkg/Folder cannot be accessed - "
+ + appDefinition.appDefPkgOrFolder.getAbsolutePath());
+ }
+
+ File src = appDefinition.appDefPkgOrFolder;
+ String targetName = appDefinition.appDefPkgOrFolder.getName();
+
+ if (appDefinition.appDefPkgOrFolder.isDirectory()) {
+ log.info("Processing app package/folder {} for {}",
+ appDefinition.appDefPkgOrFolder.getAbsolutePath(),
+ appDefinition.pkgName);
+ File tmpDir = Files.createTempDir();
+ File zipFile = new File(tmpDir.getCanonicalPath(), File.separator + appDefinition.pkgName);
+ SliderUtils.zipFolder(appDefinition.appDefPkgOrFolder, zipFile);
+
+ src = zipFile;
+ targetName = appDefinition.pkgName;
+ }
+
+ sliderFileSystem.getFileSystem().copyFromLocalFile(
+ false,
+ false,
+ new Path(src.toURI()),
+ new Path(appDefinition.targetFolderInFs, targetName));
+ }
+
+ public void persistPackages() throws BadConfigException, IOException {
+ for (AppDefinition appDefinition : appDefinitions) {
+ persistDefinitionPackageOrFolder(appDefinition);
+ }
+ }
+
+ public void processSuppliedDefinitions(String clustername,
+ AbstractClusterBuildingActionArgs buildInfo,
+ ConfTreeOperations appConf)
+ throws BadConfigException, IOException, BadCommandArgumentsException {
+ // if metainfo is provided add to the app instance
+ if (buildInfo.appMetaInfo != null) {
+ File tempDir = Files.createTempDir();
+ File pkgSrcDir = new File(tempDir, "default");
+ pkgSrcDir.mkdirs();
+ Files.copy(buildInfo.appMetaInfo, new File(pkgSrcDir, "metainfo.json"));
+
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
+ log.info("Using default app def path {}", appDirPath.toString());
+
+ appDefinitions.add(new AppDefinition(appDirPath, pkgSrcDir, SliderKeys.DEFAULT_APP_PKG));
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
+ log.info("Setting app package to {}.", appDefPath);
+ }
+
+ if (buildInfo.appDef != null) {
+ Path appDirPath = sliderFileSystem.buildAppDefDirPath(clustername);
+ appDefinitions.add(new AppDefinition(appDirPath, buildInfo.appDef, SliderKeys.DEFAULT_APP_PKG));
+ Path appDefPath = new Path(appDirPath, SliderKeys.DEFAULT_APP_PKG);
+ appConf.getGlobalOptions().set(AgentKeys.APP_DEF, appDefPath);
+ log.info("Setting app package to {}.", appDefPath);
+ }
+
+ if (buildInfo.addonDelegate.getAddonMap().size() > 0) {
+ List<String> addons = new ArrayList<String>();
+ Map<String, String> addonMap = buildInfo.addonDelegate.getAddonMap();
+ for (String key : addonMap.keySet()) {
+ Path addonPath = sliderFileSystem.buildAddonDirPath(clustername, key);
+ String addonPkgName = "addon_" + key + ".zip";
+ appDefinitions.add(new AppDefinition(addonPath, buildInfo.appDef, addonPkgName));
+ String addOnKey = AgentKeys.ADDON_PREFIX + key;
+ Path addonPkgPath = new Path(addonPath, addonPkgName);
+ log.info("Setting addon package {} to {}.", addOnKey, addonPkgPath);
+ appConf.getGlobalOptions().set(addOnKey, addonPkgPath);
+ }
+
+ String existingList = appConf.getGlobalOptions().get(AgentKeys.ADDONS);
+ if (SliderUtils.isUnset(existingList)) {
+ existingList = "";
+ }
+ appConf.getGlobalOptions().set(AgentKeys.ADDONS, existingList + StringUtils.join(addons, ","));
+ }
+ }
+
+
+ // Helper class to hold details for the app and addon packages
+ class AppDefinition {
+ // The target folder where the package will be stored
+ Path targetFolderInFs;
+ // The on disk location of the app def package or folder
+ File appDefPkgOrFolder;
+ // Package name
+ String pkgName;
+
+ public AppDefinition(Path targetFolderInFs, File appDefPkgOrFolder, String pkgName) {
+ this.targetFolderInFs = targetFolderInFs;
+ this.appDefPkgOrFolder = appDefPkgOrFolder;
+ this.pkgName = pkgName;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
index df1206d..3505ac3 100644
--- a/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
+++ b/slider-core/src/main/java/org/apache/slider/core/persist/InstancePaths.java
@@ -34,6 +34,8 @@ public class InstancePaths {
public final Path dataPath;
public final Path tmpPath;
public final Path tmpPathAM;
+ public final Path appDefPath;
+ public final Path addonsPath;
public InstancePaths(Path instanceDir) {
this.instanceDir = instanceDir;
@@ -45,6 +47,8 @@ public class InstancePaths {
dataPath = new Path(instanceDir, SliderKeys.DATA_DIR_NAME);
tmpPath = new Path(instanceDir, SliderKeys.TMP_DIR_PREFIX);
tmpPathAM = new Path(tmpPath, SliderKeys.AM_DIR_PREFIX);
+ appDefPath = new Path(tmpPath, SliderKeys.APP_DEF_DIR);
+ addonsPath = new Path(tmpPath, SliderKeys.ADDONS_DIR);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 9da269d..2e981c6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -152,6 +152,7 @@ public class AgentClientProvider extends AbstractClientProvider
} catch (BadConfigException bce) {
throw new BadConfigException("Application definition must be provided. " + bce.getMessage());
}
+
String appDef = instanceDefinition.getAppConfOperations().
getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
log.info("Validating app definition {}", appDef);
@@ -170,7 +171,7 @@ public class AgentClientProvider extends AbstractClientProvider
metaInfo = AgentUtils.getApplicationMetainfo(fs, appDef);
} catch (IOException ioe) {
// Ignore missing metainfo file for now
- log.info("Missing metainfo.xml {}", ioe.getMessage());
+ log.info("Missing metainfo {}", ioe.getMessage());
}
}
@@ -215,19 +216,31 @@ public class AgentClientProvider extends AbstractClientProvider
"Component %s is not a member of application.", name);
}
- MapOperations componentConfig = resources.getMandatoryComponent(name);
- int count =
- componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
- int definedMinCount = componentDef.getMinInstanceCountInt();
- int definedMaxCount = componentDef.getMaxInstanceCountInt();
- if (count < definedMinCount || count > definedMaxCount) {
- throw new BadConfigException("Component %s, %s value %d out of range. "
- + "Expected minimum is %d and maximum is %d",
- name,
- ResourceKeys.COMPONENT_INSTANCES,
- count,
- definedMinCount,
- definedMaxCount);
+ // ensure that intance count is 0 for client components
+ if ("CLIENT".equals(componentDef.getCategory())) {
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ if (count > 0) {
+ throw new BadConfigException("Component %s is of type CLIENT and cannot be instantiated."
+ + " Use \"slider client install ...\" command instead.",
+ name);
+ }
+ } else {
+ MapOperations componentConfig = resources.getMandatoryComponent(name);
+ int count =
+ componentConfig.getMandatoryOptionInt(ResourceKeys.COMPONENT_INSTANCES);
+ int definedMinCount = componentDef.getMinInstanceCountInt();
+ int definedMaxCount = componentDef.getMaxInstanceCountInt();
+ if (count < definedMinCount || count > definedMaxCount) {
+ throw new BadConfigException("Component %s, %s value %d out of range. "
+ + "Expected minimum is %d and maximum is %d",
+ name,
+ ResourceKeys.COMPONENT_INSTANCES,
+ count,
+ definedMinCount,
+ definedMaxCount);
+ }
}
}
}
@@ -327,7 +340,19 @@ public class AgentClientProvider extends AbstractClientProvider
while (offset < size) {
offset += zipInputStream.read(content, offset, size - offset);
}
- metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content));
+ metaInfo = new MetainfoParser().fromXmlStream(new ByteArrayInputStream(content));
+ }
+ } else if ("metainfo.json".equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zipInputStream.read(content, offset, size - offset);
+ }
+ metaInfo = new MetainfoParser().fromJsonStream(new ByteArrayInputStream(content));
}
} else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
int size = (int) zipEntry.getSize();
@@ -362,7 +387,7 @@ public class AgentClientProvider extends AbstractClientProvider
}
if (metaInfo == null) {
- throw new SliderException("Not a valid app package. Could not read metainfo.xml.");
+ throw new SliderException("Not a valid app package. Could not read metainfo.");
}
expandAgentTar(agentPkgDir);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
index 2323f97..963dadc 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentKeys.java
@@ -67,6 +67,8 @@ public interface AgentKeys {
String AGENT_MAIN_SCRIPT = "agent/main.py";
String APP_DEF = "application.def";
+ String ADDON_PREFIX = "application.addon.";
+ String ADDONS = "application.addons";
String AGENT_VERSION = "agent.version";
String AGENT_CONF = "agent.conf";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index bdf168e..4eadf13 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.registry.client.types.Endpoint;
import org.apache.hadoop.registry.client.types.ProtocolTypes;
import org.apache.hadoop.registry.client.types.ServiceRecord;
-import org.apache.hadoop.security.alias.CredentialProviderFactory;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.records.Container;
@@ -38,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.slider.api.ClusterDescription;
-import org.apache.slider.api.ClusterDescriptionKeys;
import org.apache.slider.api.ClusterNode;
import org.apache.slider.api.InternalKeys;
import org.apache.slider.api.OptionKeys;
@@ -69,6 +67,7 @@ import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
@@ -77,6 +76,7 @@ import org.apache.slider.providers.agent.application.metadata.ExportGroup;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
import org.apache.slider.providers.agent.application.metadata.OSPackage;
import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.Package;
import org.apache.slider.providers.agent.application.metadata.PropertyInfo;
import org.apache.slider.server.appmaster.actions.ProviderReportedContainerLoss;
import org.apache.slider.server.appmaster.actions.RegisterComponentInstance;
@@ -157,7 +157,7 @@ public class AgentProviderService extends AbstractProviderService implements
private int heartbeatMonitorInterval = 0;
private AgentClientProvider clientProvider;
private AtomicInteger taskId = new AtomicInteger(0);
- private volatile Metainfo metainfo = null;
+ private volatile Metainfo metaInfo = null;
private Map<String, DefaultConfig> defaultConfigs = null;
private ComponentCommandOrder commandOrder = null;
private HeartbeatMonitor monitor;
@@ -233,7 +233,7 @@ public class AgentProviderService extends AbstractProviderService implements
Set<String> names = resources.getComponentNames();
names.remove(SliderKeys.COMPONENT_AM);
for (String name : names) {
- Component componentDef = getMetainfo().getApplicationComponent(name);
+ Component componentDef = getMetaInfo().getApplicationComponent(name);
if (componentDef == null) {
throw new BadConfigException(
"Component %s is not a member of application.", name);
@@ -262,21 +262,20 @@ public class AgentProviderService extends AbstractProviderService implements
String appDef = instanceDefinition.getAppConfOperations()
.getGlobalOptions().getMandatoryOption(AgentKeys.APP_DEF);
- if (metainfo == null) {
+ if (metaInfo == null) {
synchronized (syncLock) {
- if (metainfo == null) {
+ if (metaInfo == null) {
readAndSetHeartbeatMonitoringInterval(instanceDefinition);
initializeAgentDebugCommands(instanceDefinition);
- metainfo = getApplicationMetainfo(fileSystem, appDef);
- if (metainfo == null || metainfo.getApplication() == null) {
+ metaInfo = getApplicationMetainfo(fileSystem, appDef);
+ if (metaInfo == null || metaInfo.getApplication() == null) {
log.error("metainfo.xml is unavailable or malformed at {}.", appDef);
throw new SliderException(
"metainfo.xml is required in app package.");
}
- commandOrder = new ComponentCommandOrder(metainfo.getApplication()
- .getCommandOrder());
- defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metainfo);
+ commandOrder = new ComponentCommandOrder(metaInfo.getApplication().getCommandOrders());
+ defaultConfigs = initializeDefaultConfigs(fileSystem, appDef, metaInfo);
monitor = new HeartbeatMonitor(this, getHeartbeatMonitorInterval());
monitor.start();
}
@@ -717,15 +716,20 @@ public class AgentProviderService extends AbstractProviderService implements
StateAccessForProviders accessor = getAmState();
CommandScript cmdScript = getScriptPathFromMetainfo(roleName);
+ List<ComponentCommand> commands = getMetaInfo().getApplicationComponent(roleName).getCommands();
- if (cmdScript == null || cmdScript.getScript() == null) {
+ if ((cmdScript == null || cmdScript.getScript() == null) && commands.size() == 0) {
log.error("role.script is unavailable for {}. Commands will not be sent.",
roleName);
return response;
}
- String scriptPath = cmdScript.getScript();
- long timeout = cmdScript.getTimeout();
+ String scriptPath = null;
+ long timeout = 600L;
+ if(cmdScript != null) {
+ scriptPath = cmdScript.getScript();
+ timeout = cmdScript.getTimeout();
+ }
if (timeout == 0L) {
timeout = 600L;
@@ -778,15 +782,50 @@ public class AgentProviderService extends AbstractProviderService implements
if (Command.NOP != command) {
if (command == Command.INSTALL) {
log.info("Installing {} on {}.", roleName, containerId);
- addInstallCommand(roleName, containerId, response, scriptPath, timeout);
+ if (scriptPath != null) {
+ addInstallCommand(roleName, containerId, response, scriptPath, null, timeout);
+ } else {
+ // commands
+ ComponentCommand installCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("INSTALL")) {
+ installCmd = compCmd;
+ }
+ }
+ addInstallCommand(roleName, containerId, response, null, installCmd, timeout);
+ }
componentStatus.commandIssued(command);
} else if (command == Command.START) {
// check against dependencies
boolean canExecute = commandOrder.canExecute(roleName, command, getComponentStatuses().values());
if (canExecute) {
log.info("Starting {} on {}.", roleName, containerId);
- addStartCommand(roleName, containerId, response, scriptPath, timeout, isMarkedAutoRestart(roleName));
- componentStatus.commandIssued(command);
+ if (scriptPath != null) {
+ addStartCommand(roleName,
+ containerId,
+ response,
+ scriptPath,
+ null,
+ null,
+ timeout,
+ isMarkedAutoRestart(roleName));
+ componentStatus.commandIssued(command);
+ } else {
+ ComponentCommand startCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("START")) {
+ startCmd = compCmd;
+ }
+ }
+ ComponentCommand stopCmd = null;
+ for (ComponentCommand compCmd : commands) {
+ if (compCmd.getName().equals("STOP")) {
+ stopCmd = compCmd;
+ }
+ }
+ addStartCommand(roleName, containerId, response, null, startCmd, stopCmd, timeout, false);
+ componentStatus.commandIssued(command);
+ }
} else {
log.info("Start of {} on {} delayed as dependencies have not started.", roleName, containerId);
}
@@ -858,7 +897,6 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- // component specific publishes
processAndPublishComponentSpecificData(ports, containerId, fqdn, roleName);
processAndPublishComponentSpecificExports(ports, containerId, fqdn, roleName);
@@ -1012,8 +1050,8 @@ public class AgentProviderService extends AbstractProviderService implements
}
@VisibleForTesting
- protected Metainfo getMetainfo() {
- return this.metainfo;
+ protected Metainfo getMetaInfo() {
+ return this.metaInfo;
}
@VisibleForTesting
@@ -1023,7 +1061,7 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
+ String appDef) throws IOException, BadConfigException {
return AgentUtils.getApplicationMetainfo(fileSystem, appDef);
}
@@ -1045,7 +1083,7 @@ public class AgentProviderService extends AbstractProviderService implements
*/
protected Map<String, DefaultConfig> initializeDefaultConfigs(SliderFileSystem fileSystem,
String appDef, Metainfo metainfo) throws IOException {
- Map<String, DefaultConfig> defaultConfigMap = new HashMap<String, DefaultConfig>();
+ Map<String, DefaultConfig> defaultConfigMap = new HashMap<>();
if (SliderUtils.isNotEmpty(metainfo.getApplication().getConfigFiles())) {
for (ConfigFile configFile : metainfo.getApplication().getConfigFiles()) {
DefaultConfig config = null;
@@ -1226,7 +1264,7 @@ public class AgentProviderService extends AbstractProviderService implements
log.info("Status report: {}", status.toString());
if (status.getConfigs() != null) {
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
if (canAnyMasterPublishConfig() == false || canPublishConfig(componentName)) {
// If no Master can explicitly publish then publish if its a master
@@ -1370,10 +1408,10 @@ public class AgentProviderService extends AbstractProviderService implements
String hostNamePattern = "${THIS_HOST}";
Map<String, String> toPublish = new HashMap<String, String>();
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
for (Component component : application.getComponents()) {
if (component.getName().equals(componentName)) {
- if (!component.getComponentExports().isEmpty()) {
+ if (component.getComponentExports().size() > 0) {
for (ComponentExport export : component.getComponentExports()) {
String templateToExport = export.getValue();
@@ -1399,7 +1437,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
- if (!toPublish.isEmpty()) {
+ if (toPublish.size() > 0) {
Map<String, String> perContainerData = null;
if (!getComponentInstanceData().containsKey(containerId)) {
perContainerData = new ConcurrentHashMap<String, String>();
@@ -1420,8 +1458,8 @@ public class AgentProviderService extends AbstractProviderService implements
String portVarFormat = "${site.%s}";
String hostNamePattern = "${" + compName + "_HOST}";
- List<ExportGroup> appExportGroups = getMetainfo().getApplication().getExportGroups();
- Component component = getMetainfo().getApplicationComponent(compName);
+ List<ExportGroup> appExportGroups = getMetaInfo().getApplication().getExportGroups();
+ Component component = getMetaInfo().getApplicationComponent(compName);
if (component != null && SliderUtils.isSet(component.getCompExports())
&& SliderUtils.isNotEmpty(appExportGroups)) {
@@ -1523,7 +1561,7 @@ public class AgentProviderService extends AbstractProviderService implements
* @return the component entry or null for no match
*/
protected Component getApplicationComponent(String roleName) {
- return getMetainfo().getApplicationComponent(roleName);
+ return getMetaInfo().getApplicationComponent(roleName);
}
/**
@@ -1583,7 +1621,7 @@ public class AgentProviderService extends AbstractProviderService implements
protected boolean isMarkedAutoRestart(String roleName) {
Component component = getApplicationComponent(roleName);
if (component != null) {
- return component.getRequiresAutoRestart();
+ return component.getAutoStartOnFailureBoolean();
}
return false;
}
@@ -1595,7 +1633,7 @@ public class AgentProviderService extends AbstractProviderService implements
*/
protected boolean canAnyMasterPublishConfig() {
if (canAnyMasterPublish == null) {
- Application application = getMetainfo().getApplication();
+ Application application = getMetaInfo().getApplication();
if (application == null) {
log.error("Malformed app definition: Expect application as root element in the metainfo.xml");
} else {
@@ -1637,6 +1675,7 @@ public class AgentProviderService extends AbstractProviderService implements
String containerId,
HeartBeatResponse response,
String scriptPath,
+ ComponentCommand compCmd,
long timeout)
throws SliderException {
assert getAmState().isApplicationLive();
@@ -1651,7 +1690,7 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setComponentName(componentName);
cmd.setRole(componentName);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(PACKAGE_LIST, getPackageList());
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
@@ -1659,9 +1698,20 @@ public class AgentProviderService extends AbstractProviderService implements
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
cmd.setConfigurations(configurations);
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
+ } else {
+ // assume it to be default shell command
+ ComponentCommand effectiveCommand = compCmd;
+ if(effectiveCommand == null) {
+ effectiveCommand = ComponentCommand.getDefaultComponentCommand("INSTALL");
+ }
+ cmd.setCommandParams(commandParametersSet(effectiveCommand, timeout, false));
+ configurations.get("global").put("exec_cmd", effectiveCommand.getExec());
+ }
cmd.setHostname(getClusterInfoPropertyValue(StatusKeys.INFO_AM_HOSTNAME));
+
response.addExecutionCommand(cmd);
}
@@ -1670,12 +1720,19 @@ public class AgentProviderService extends AbstractProviderService implements
String pkgListFormatString = "[%s]";
List<String> packages = new ArrayList();
if (application != null) {
- List<OSSpecific> osSpecifics = application.getOSSpecifics();
- if (osSpecifics != null && osSpecifics.size() > 0) {
- for (OSSpecific osSpecific : osSpecifics) {
- if (osSpecific.getOsType().equals("any")) {
- for (OSPackage osPackage : osSpecific.getPackages()) {
- packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
+ if (application.getPackages().size() > 0) {
+ List<Package> appPackages = application.getPackages();
+ for (Package appPackage : appPackages) {
+ packages.add(String.format(pkgFormatString, appPackage.getType(), appPackage.getName()));
+ }
+ } else {
+ List<OSSpecific> osSpecifics = application.getOSSpecifics();
+ if (osSpecifics != null && osSpecifics.size() > 0) {
+ for (OSSpecific osSpecific : osSpecifics) {
+ if (osSpecific.getOsType().equals("any")) {
+ for (OSPackage osPackage : osSpecific.getPackages()) {
+ packages.add(String.format(pkgFormatString, osPackage.getType(), osPackage.getName()));
+ }
}
}
}
@@ -1690,7 +1747,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private String getPackageList() {
- return getPackageListFromApplication(getMetainfo().getApplication());
+ return getPackageListFromApplication(getMetaInfo().getApplication());
}
private void prepareExecutionCommand(ExecutionCommand cmd) {
@@ -1698,8 +1755,8 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setCommandId(cmd.getTaskId() + "-1");
}
- private Map<String, String> setCommandParameters(String scriptPath, long timeout, boolean recordConfig) {
- Map<String, String> cmdParams = new TreeMap<String, String>();
+ private Map<String, String> commandParametersSet(String scriptPath, long timeout, boolean recordConfig) {
+ Map<String, String> cmdParams = new TreeMap<>();
cmdParams.put("service_package_folder",
"${AGENT_WORK_ROOT}/work/app/definition/package");
cmdParams.put("script", scriptPath);
@@ -1710,6 +1767,18 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
+ private Map<String, String> commandParametersSet(ComponentCommand compCmd, long timeout, boolean recordConfig) {
+ Map<String, String> cmdParams = new TreeMap<>();
+ cmdParams.put("service_package_folder",
+ "${AGENT_WORK_ROOT}/work/app/definition/package");
+ cmdParams.put("command", compCmd.getExec());
+ cmdParams.put("schema_version", "2.0");
+ cmdParams.put("command_timeout", Long.toString(timeout));
+ cmdParams.put("script_type", compCmd.getType());
+ cmdParams.put("record_config", Boolean.toString(recordConfig));
+ return cmdParams;
+ }
+
@VisibleForTesting
protected void addStatusCommand(String componentName,
String containerId,
@@ -1730,11 +1799,11 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setRoleCommand(StatusCommand.STATUS_COMMAND);
Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, false));
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, false));
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
@@ -1767,7 +1836,9 @@ public class AgentProviderService extends AbstractProviderService implements
@VisibleForTesting
protected void addStartCommand(String componentName, String containerId, HeartBeatResponse response,
- String scriptPath, long timeout, boolean isMarkedAutoRestart)
+ String scriptPath, ComponentCommand startCommand,
+ ComponentCommand stopCommand,
+ long timeout, boolean isMarkedAutoRestart)
throws
SliderException {
assert getAmState().isApplicationLive();
@@ -1784,20 +1855,28 @@ public class AgentProviderService extends AbstractProviderService implements
cmd.setServiceName(clusterName);
cmd.setComponentName(componentName);
cmd.setRole(componentName);
- Map<String, String> hostLevelParams = new TreeMap<String, String>();
- hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getMandatoryOption(JAVA_HOME));
+ Map<String, String> hostLevelParams = new TreeMap<>();
+ hostLevelParams.put(JAVA_HOME, appConf.getGlobalOptions().getOption(JAVA_HOME, getJDKDir()));
hostLevelParams.put(CONTAINER_ID, containerId);
cmd.setHostLevelParams(hostLevelParams);
- Map<String, String> roleParams = new TreeMap<String, String>();
+ Map<String, String> roleParams = new TreeMap<>();
cmd.setRoleParams(roleParams);
cmd.getRoleParams().put("auto_restart", Boolean.toString(isMarkedAutoRestart));
- cmd.setCommandParams(setCommandParameters(scriptPath, timeout, true));
-
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf, containerId, componentName);
-
cmd.setConfigurations(configurations);
+
+ if(SliderUtils.isSet(scriptPath)) {
+ cmd.setCommandParams(commandParametersSet(scriptPath, timeout, true));
+ } else {
+ if(startCommand == null) {
+ throw new SliderException("Expected START command not found for component " + componentName);
+ }
+ cmd.setCommandParams(commandParametersSet(startCommand, timeout, true));
+ configurations.get("global").put("exec_cmd", startCommand.getExec());
+ }
+
response.addExecutionCommand(cmd);
// With start command, the corresponding command for graceful stop needs to
@@ -1815,16 +1894,25 @@ public class AgentProviderService extends AbstractProviderService implements
cmdStop.setRole(componentName);
Map<String, String> hostLevelParamsStop = new TreeMap<String, String>();
hostLevelParamsStop.put(JAVA_HOME, appConf.getGlobalOptions()
- .getMandatoryOption(JAVA_HOME));
+ .getOption(JAVA_HOME, ""));
hostLevelParamsStop.put(CONTAINER_ID, containerId);
cmdStop.setHostLevelParams(hostLevelParamsStop);
Map<String, String> roleParamsStop = new TreeMap<String, String>();
cmdStop.setRoleParams(roleParamsStop);
cmdStop.getRoleParams().put("auto_restart",
- Boolean.toString(isMarkedAutoRestart));
+ Boolean.toString(isMarkedAutoRestart));
+
+ if(SliderUtils.isSet(scriptPath)) {
+ cmdStop.setCommandParams(commandParametersSet(scriptPath, timeout, true));
+ } else {
+ if(stopCommand == null) {
+ stopCommand = ComponentCommand.getDefaultComponentCommand("STOP");
+ }
+ cmd.setCommandParams(commandParametersSet(stopCommand, timeout, true));
+ configurations.get("global").put("exec_cmd", startCommand.getExec());
+ }
- cmdStop.setCommandParams(setCommandParameters(scriptPath, timeout, true));
Map<String, Map<String, String>> configurationsStop = buildCommandConfigurations(
appConf, containerId, componentName);
@@ -1832,6 +1920,19 @@ public class AgentProviderService extends AbstractProviderService implements
response.addExecutionCommand(cmdStop);
}
+ protected static String getJDKDir() {
+ File javaHome = new File(System.getProperty("java.home")).getParentFile();
+ File jdkDirectory = null;
+ if (javaHome.getName().contains("jdk")) {
+ jdkDirectory = javaHome;
+ }
+ if (jdkDirectory != null) {
+ return jdkDirectory.getAbsolutePath();
+ } else {
+ return "";
+ }
+ }
+
protected Map<String, String> getAllocatedPorts() {
return getAllocatedPorts(SHARED_PORT_TAG);
}
@@ -1938,7 +2039,7 @@ public class AgentProviderService extends AbstractProviderService implements
List<String> configList = new ArrayList<String>();
configList.add(GLOBAL_CONFIG_TAG);
- List<ConfigFile> configFiles = getMetainfo().getApplication().getConfigFiles();
+ List<ConfigFile> configFiles = getMetaInfo().getApplication().getConfigFiles();
for (ConfigFile configFile : configFiles) {
log.info("Expecting config type {}.", configFile.getDictionaryName());
configList.add(configFile.getDictionaryName());
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
index 1d61c15..6fe1161 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentUtils.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
import org.apache.slider.providers.agent.application.metadata.DefaultConfigParser;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
@@ -28,7 +29,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
-import java.io.File;
import java.io.IOException;
import java.io.InputStream;
@@ -39,20 +39,30 @@ public class AgentUtils {
private static final Logger log = LoggerFactory.getLogger(AgentUtils.class);
public static Metainfo getApplicationMetainfo(SliderFileSystem fileSystem,
- String appDef) throws IOException {
+ String appDef) throws IOException, BadConfigException {
log.info("Reading metainfo at {}", appDef);
FileSystem fs = fileSystem.getFileSystem();
Path appPath = new Path(appDef);
- InputStream metainfoStream = SliderUtils.getApplicationResourceInputStream(
- fs, appPath, "metainfo.xml");
- if (metainfoStream == null) {
- log.error("metainfo.xml is unavailable at {}.", appDef);
- throw new FileNotFoundException("metainfo.xml is required in app package. " +
- appPath);
- }
- Metainfo metainfo = new MetainfoParser().parse(metainfoStream);
+ Metainfo metainfo = null;
+ MetainfoParser metainfoParser = new MetainfoParser();
+ InputStream metainfoJsonStream = SliderUtils.getApplicationResourceInputStream(
+ fs, appPath, "metainfo.json");
+ if (metainfoJsonStream == null) {
+ InputStream metainfoXMLStream = SliderUtils.getApplicationResourceInputStream(
+ fs, appPath, "metainfo.xml");
+ if (metainfoXMLStream != null) {
+ metainfo = metainfoParser.fromXmlStream(metainfoXMLStream);
+ }
+ } else {
+ metainfo = metainfoParser.fromJsonStream(metainfoJsonStream);
+ }
+ if (metainfo == null) {
+ log.error("metainfo is unavailable at {}.", appDef);
+ throw new FileNotFoundException("metainfo.xml/json is required in app package. " +
+ appPath);
+ }
return metainfo;
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
index f4ace5f..194d6ff 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/ComponentCommandOrder.java
@@ -49,13 +49,13 @@ public class ComponentCommandOrder {
if (requiredStates.size() > 0) {
Map<String, List<ComponentState>> compDep = dependencies.get(componentCmd.command);
if (compDep == null) {
- compDep = new HashMap<String, List<ComponentState>>();
+ compDep = new HashMap<>();
dependencies.put(componentCmd.command, compDep);
}
List<ComponentState> requirements = compDep.get(componentCmd.componentName);
if (requirements == null) {
- requirements = new ArrayList<ComponentState>();
+ requirements = new ArrayList<>();
compDep.put(componentCmd.componentName, requirements);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
index bc43d4b..cd5555f 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Application.java
@@ -16,29 +16,31 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
+
import java.util.ArrayList;
import java.util.List;
/**
* Application type defined in the metainfo
*/
-public class Application {
+public class Application implements Validate {
String name;
String comment;
String version;
String exportedConfigs;
- List<Component> components;
- List<ExportGroup> exportGroups;
- List<OSSpecific> osSpecifics;
- List<CommandOrder> commandOrders;
- List<ConfigFile> configFiles;
+ List<Component> components = new ArrayList<>();
+ List<ExportGroup> exportGroups = new ArrayList<>();
+ List<OSSpecific> osSpecifics = new ArrayList<>();
+ List<CommandOrder> commandOrders = new ArrayList<>();
+ List<ConfigFile> configFiles = new ArrayList<>();
+ List<Package> packages = new ArrayList<>();
public Application() {
- exportGroups = new ArrayList<ExportGroup>();
- components = new ArrayList<Component>();
- osSpecifics = new ArrayList<OSSpecific>();
- commandOrders = new ArrayList<CommandOrder>();
- configFiles = new ArrayList<ConfigFile>();
}
public String getName() {
@@ -73,18 +75,20 @@ public class Application {
this.exportedConfigs = exportedConfigs;
}
- public List<ConfigFile> getConfigFiles() {
- return configFiles;
- }
-
public void addConfigFile(ConfigFile configFile) {
this.configFiles.add(configFile);
}
+ @JsonProperty("configFiles")
+ public List<ConfigFile> getConfigFiles() {
+ return configFiles;
+ }
+
public void addComponent(Component component) {
components.add(component);
}
+ @JsonProperty("components")
public List<Component> getComponents() {
return components;
}
@@ -93,6 +97,7 @@ public class Application {
exportGroups.add(exportGroup);
}
+ @JsonProperty("exportGroups")
public List<ExportGroup> getExportGroups() {
return exportGroups;
}
@@ -101,6 +106,7 @@ public class Application {
osSpecifics.add(osSpecific);
}
+ @JsonIgnore
public List<OSSpecific> getOSSpecifics() {
return osSpecifics;
}
@@ -109,10 +115,16 @@ public class Application {
commandOrders.add(commandOrder);
}
- public List<CommandOrder> getCommandOrder() {
+ @JsonProperty("commandOrders")
+ public List<CommandOrder> getCommandOrders() {
return commandOrders;
}
+ @JsonProperty("packages")
+ public List<Package> getPackages() {
+ return packages;
+ }
+
@Override
public String toString() {
final StringBuilder sb =
@@ -128,4 +140,54 @@ public class Application {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ if(SliderUtils.isUnset(version)) {
+ throw new BadCommandArgumentsException("schema version cannot be null");
+ }
+
+ Metainfo.checkNonNull(getName(), "name", "application");
+
+ Metainfo.checkNonNull(getVersion(), "version", "application");
+
+ if(getComponents().size() == 0) {
+ throw new SliderException("application must contain at least one component");
+ }
+
+ if(version.equals(Metainfo.VERSION_TWO_ZERO)) {
+ if(getPackages().size() > 0) {
+ throw new SliderException("packages is not supported in version " + version);
+ }
+ }
+
+ if(version.equals(Metainfo.VERSION_TWO_ONE)) {
+ if(getOSSpecifics().size() > 0) {
+ throw new SliderException("osSpecifics is not supported in version " + version);
+ }
+ }
+
+ for(CommandOrder co : getCommandOrders()) {
+ co.validate(version);
+ }
+
+ for(Component comp : getComponents()) {
+ comp.validate(version);
+ }
+
+ for(ConfigFile cf : getConfigFiles()) {
+ cf.validate(version);
+ }
+
+ for(ExportGroup eg : getExportGroups()) {
+ eg.validate(version);
+ }
+
+ for(Package pkg : getPackages()) {
+ pkg.validate(version);
+ }
+
+ for(OSSpecific os : getOSSpecifics()) {
+ os.validate(version);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
index 825a104..40d8cc6 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandOrder.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
*
*/
-public class CommandOrder {
+public class CommandOrder implements Validate {
String command;
String requires;
@@ -51,4 +53,9 @@ public class CommandOrder {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getCommand(), "command", "package");
+ Metainfo.checkNonNull(getRequires(), "requires", "package");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
index 612322f..9915ba1 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/CommandScript.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
- *
+ * CommandScript that implements all component commands
*/
-public class CommandScript {
+public class CommandScript implements Validate {
String script;
String scriptType;
long timeout;
@@ -62,4 +64,9 @@ public class CommandScript {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getScript(), "script", "commandScript");
+ Metainfo.checkNonNull(getScriptType(), "scriptType", "commandScript");
+ }
}
[3/5] incubator-slider git commit: SLIDER-663. Make it easy to
develop and deploy application packages that are essentially shell commands
(part-I)
Posted by sm...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
index 418868c..3f1b7fe 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Component.java
@@ -19,6 +19,7 @@ package org.apache.slider.providers.agent.application.metadata;
import org.apache.slider.common.tools.SliderUtils;
import org.apache.slider.core.exceptions.BadConfigException;
import org.apache.slider.core.exceptions.SliderException;
+import org.codehaus.jackson.annotate.JsonProperty;
import java.util.ArrayList;
import java.util.List;
@@ -26,21 +27,28 @@ import java.util.List;
/**
*
*/
-public class Component {
+public class Component implements Validate {
+
+ public static String TYPE_STANDARD = "STANDARD";
+ public static String TYPE_DOCKER = "DOCKER";
+ public static String CATEGORY_MASTER = "MASTER";
+ public static String CATEGORY_SLAVE = "SLAVE";
+ public static String CATEGORY_CLIENT = "CLIENT";
+
String name;
- String category;
- String publishConfig;
- String minInstanceCount;
+ String category = CATEGORY_MASTER;
+ String publishConfig = Boolean.FALSE.toString();
+ String minInstanceCount = "0";
String maxInstanceCount;
- String autoStartOnFailure;
+ String autoStartOnFailure = Boolean.FALSE.toString();
String appExports;
String compExports;
CommandScript commandScript;
- List<ComponentExport> componentExports;
+ String type = TYPE_STANDARD;
+ List<ComponentExport> componentExports = new ArrayList<>();
+ List<ComponentCommand> commands = new ArrayList<>();
public Component() {
- publishConfig = Boolean.FALSE.toString();
- componentExports = new ArrayList<ComponentExport>();
}
public String getName() {
@@ -51,6 +59,14 @@ public class Component {
this.name = name;
}
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
public String getCategory() {
return category;
}
@@ -95,6 +111,14 @@ public class Component {
return minInstanceCount;
}
+ public Boolean getAutoStartOnFailureBoolean() {
+ if (SliderUtils.isUnset(getAutoStartOnFailure())) {
+ return Boolean.FALSE;
+ }
+
+ return Boolean.parseBoolean(getAutoStartOnFailure());
+ }
+
public int getMinInstanceCountInt() throws BadConfigException {
if (SliderUtils.isUnset(minInstanceCount)) {
return 0;
@@ -151,6 +175,11 @@ public class Component {
return Boolean.parseBoolean(this.autoStartOnFailure);
}
+ @JsonProperty("commands")
+ public List<ComponentCommand> getCommands() {
+ return this.commands;
+ }
+
@Override
public String toString() {
final StringBuilder sb =
@@ -162,9 +191,39 @@ public class Component {
return sb.toString();
}
- class AutoRestartSettings {
- private boolean requiresAutoRestart;
- private int maxFailures;
- private int inThisManyMinutes;
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "component");
+ Metainfo.checkNonNull(getCategory(), "category", "component");
+ if (!getCategory().equals(CATEGORY_MASTER)
+ && !getCategory().equals(CATEGORY_SLAVE)
+ && !getCategory().equals(CATEGORY_CLIENT)) {
+ throw new SliderException("Invalid category for the component " + getCategory());
+ }
+
+ Metainfo.checkNonNull(getType(), "type", "component");
+ if (!getType().equals(TYPE_DOCKER)
+ && !getType().equals(TYPE_STANDARD)) {
+ throw new SliderException("Invalid type for the component " + getType());
+ }
+
+ if (version.equals(Metainfo.VERSION_TWO_ZERO)) {
+ if (getType().equals(TYPE_DOCKER)) {
+ throw new SliderException(TYPE_DOCKER + " is not supported in version " + Metainfo.VERSION_TWO_ZERO);
+ }
+
+ if (getCommands().size() > 0) {
+ throw new SliderException("commands are not supported in version " + Metainfo.VERSION_TWO_ZERO);
+ }
+ }
+
+ if (commandScript != null) {
+ commandScript.validate(version);
+ }
+
+ if (version.equals(Metainfo.VERSION_TWO_ONE)) {
+ for (ComponentCommand cc : getCommands()) {
+ cc.validate(version);
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
new file mode 100644
index 0000000..52117c5
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ComponentCommand.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents the metadata associated with the application.
+ */
+public class ComponentCommand implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(ComponentCommand.class);
+
+
+ private String exec;
+ private String name = "START";
+ private String type = "SHELL";
+
+ /**
+ * Creator.
+ */
+ public ComponentCommand() {
+ }
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public void setExec(String exec) {
+ this.exec = exec;
+ }
+
+ public String getExec() {
+ return exec;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "componentCommand");
+
+ Metainfo.checkNonNull(getType(), "version", "application");
+ }
+
+ public static ComponentCommand getDefaultComponentCommand() {
+ ComponentCommand cc = new ComponentCommand();
+ cc.setExec("DEFAULT");
+ return cc;
+ }
+
+ public static ComponentCommand getDefaultComponentCommand(String commandName) {
+ ComponentCommand cc = new ComponentCommand();
+ cc.setExec("DEFAULT");
+ cc.setName(commandName);
+ return cc;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
index b9dfb4e..cb47512 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ConfigFile.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
*
*/
-public class ConfigFile {
+public class ConfigFile implements Validate {
String type;
String fileName;
String dictionaryName;
@@ -50,4 +52,8 @@ public class ConfigFile {
public void setDictionaryName(String dictionaryName) {
this.dictionaryName = dictionaryName;
}
+
+ public void validate(String version) throws SliderException {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
index 17326a3..5e0fb24 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Export.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
*
*/
-public class Export {
+public class Export implements Validate {
String name;
String value;
@@ -51,4 +53,9 @@ public class Export {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "export");
+ Metainfo.checkNonNull(getValue(), "value", "export");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
index d2e20a4..3d9f34c 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/ExportGroup.java
@@ -16,13 +16,15 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
import java.util.ArrayList;
import java.util.List;
/**
*
*/
-public class ExportGroup {
+public class ExportGroup implements Validate {
String name;
List<Export> exports;
@@ -59,4 +61,11 @@ public class ExportGroup {
sb.append('}');
return sb.toString();
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "exportGroup");
+ for(Export exp : getExports()) {
+ exp.validate(version);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
index b34cba1..f89846e 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Metainfo.java
@@ -16,6 +16,8 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.common.tools.SliderUtils;
+import org.apache.slider.core.exceptions.SliderException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -25,13 +27,12 @@ import org.slf4j.LoggerFactory;
public class Metainfo {
protected static final Logger log =
LoggerFactory.getLogger(Metainfo.class);
+ public static String VERSION_TWO_ZERO = "2.0";
+ public static String VERSION_TWO_ONE = "2.1";
String schemaVersion;
Application application;
- public Metainfo() {
- }
-
public String getSchemaVersion() {
return schemaVersion;
}
@@ -50,7 +51,7 @@ public class Metainfo {
public Component getApplicationComponent(String roleName) {
if (application == null) {
- log.error("Malformed app definition: Expect application as the top level element for metainfo.xml");
+ log.error("Malformed app definition: Expect application as the top level element for metainfo");
} else {
for (Component component : application.getComponents()) {
if (component.getName().equals(roleName)) {
@@ -60,4 +61,19 @@ public class Metainfo {
}
return null;
}
+
+ public void validate() throws SliderException {
+ if (!VERSION_TWO_ONE.equals(schemaVersion) ||
+ !VERSION_TWO_ZERO.equals(schemaVersion)) {
+ throw new SliderException("Unsupported version " + getSchemaVersion());
+ }
+
+ application.validate(schemaVersion);
+ }
+
+ public static void checkNonNull(String value, String field, String type) throws SliderException {
+ if (SliderUtils.isUnset(value)) {
+ throw new SliderException(type + "." + field + " cannot be null");
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
index 1d8403f..12af586 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/MetainfoParser.java
@@ -16,18 +16,78 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
import org.apache.commons.digester.Digester;
+import org.apache.commons.io.IOUtils;
import org.xml.sax.SAXException;
import java.io.IOException;
import java.io.InputStream;
+import java.io.StringWriter;
/**
*
*/
public class MetainfoParser {
+ private final GsonBuilder gsonBuilder = new GsonBuilder();
+ private final Gson gson;
- public Metainfo parse(InputStream metainfoStream) throws IOException {
+ public MetainfoParser() {
+ gson = gsonBuilder.create();
+ }
+
+ /**
+ * Convert to a JSON string
+ *
+ * @return a JSON string description
+ *
+ * @throws IOException Problems mapping/writing the object
+ */
+ public String toJsonString(Metainfo metaInfo) throws IOException {
+ return gson.toJson(metaInfo);
+ }
+
+ /**
+ * Convert from JSON
+ *
+ * @param json input
+ *
+ * @return the parsed JSON
+ *
+ * @throws IOException IO
+ */
+ public Metainfo fromJsonString(String json)
+ throws IOException {
+ return gson.fromJson(json, Metainfo.class);
+ }
+
+ /**
+ * Parse metainfo from an IOStream
+ *
+ * @param is
+ *
+ * @return
+ *
+ * @throws IOException
+ */
+ public Metainfo fromJsonStream(InputStream is) throws IOException {
+ StringWriter writer = new StringWriter();
+ IOUtils.copy(is, writer);
+ return fromJsonString(writer.toString());
+ }
+
+
+ /**
+ * Parse metainfo from an XML formatted IOStream
+ *
+ * @param metainfoStream
+ *
+ * @return
+ *
+ * @throws IOException
+ */
+ public Metainfo fromXmlStream(InputStream metainfoStream) throws IOException {
Digester digester = new Digester();
digester.setValidating(false);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
index 334f96b..32b4890 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSPackage.java
@@ -16,10 +16,12 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
/**
*
*/
-public class OSPackage {
+public class OSPackage implements Validate {
String type;
String name;
@@ -41,4 +43,9 @@ public class OSPackage {
public void setName(String name) {
this.name = name;
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "osPackage");
+ Metainfo.checkNonNull(getType(), "type", "osPackage");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
index 7c36e8e..c06d498 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/OSSpecific.java
@@ -16,13 +16,15 @@
*/
package org.apache.slider.providers.agent.application.metadata;
+import org.apache.slider.core.exceptions.SliderException;
+
import java.util.ArrayList;
import java.util.List;
/**
*
*/
-public class OSSpecific {
+public class OSSpecific implements Validate {
String osType;
List<OSPackage> packages;
@@ -45,4 +47,11 @@ public class OSSpecific {
public List<OSPackage> getPackages() {
return packages;
}
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getOsType(), "osType", "osSpecific");
+ for (OSPackage opkg : getPackages()) {
+ opkg.validate(version);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
new file mode 100644
index 0000000..b88f77d
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Package.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represents package description.
+ */
+public class Package implements Validate {
+ protected static final Logger
+ log = LoggerFactory.getLogger(Package.class);
+
+
+ private String name;
+ private String type;
+
+ public Package() {
+ }
+
+
+ public void setName(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public void setType(String type) {
+ this.type = type;
+ }
+
+ public void validate(String version) throws SliderException {
+ Metainfo.checkNonNull(getName(), "name", "package");
+ Metainfo.checkNonNull(getType(), "type", "package");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
new file mode 100644
index 0000000..ef03dcd
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/application/metadata/Validate.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.slider.providers.agent.application.metadata;
+
+import org.apache.slider.core.exceptions.SliderException;
+
+/**
+ * Implementer provides a validate method
+ */
+public interface Validate {
+
+ public void validate(String version) throws SliderException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
index 5fb3b5e..eca5e7d 100644
--- a/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
+++ b/slider-core/src/main/java/org/apache/slider/server/appmaster/web/rest/agent/ExecutionCommand.java
@@ -16,11 +16,11 @@
*/
package org.apache.slider.server.appmaster.web.rest.agent;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.annotate.JsonProperty;
import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.HashMap;
import java.util.Map;
@@ -31,7 +31,8 @@ import java.util.Map;
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
public class ExecutionCommand {
- private static Log LOG = LogFactory.getLog(ExecutionCommand.class);
+ protected static final Logger log =
+ LoggerFactory.getLogger(ExecutionCommand.class);
private AgentCommandType commandType = AgentCommandType.EXECUTION_COMMAND;
private String clusterName;
private long taskId;
@@ -46,6 +47,7 @@ public class ExecutionCommand {
private Map<String, String> commandParams;
private String serviceName;
private String componentName;
+ private String componentType;
public ExecutionCommand(AgentCommandType commandType) {
this.commandType = commandType;
@@ -121,6 +123,16 @@ public class ExecutionCommand {
this.clusterName = clusterName;
}
+ @JsonProperty("componentType")
+ public String getComponentType() {
+ return componentType;
+ }
+
+ @JsonProperty("componentType")
+ public void setComponentType(String componentType) {
+ this.componentType = componentType;
+ }
+
@JsonProperty("hostname")
public String getHostname() {
return hostname;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json
new file mode 100644
index 0000000..051a247
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json
@@ -0,0 +1,8 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "metadata": {
+ },
+ "global": {
+ "site.global.port": "${LISTEN.ALLOCATED_PORT}{PER_CONTAINER}"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json
new file mode 100644
index 0000000..3c7391c
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json
@@ -0,0 +1,28 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "LISTENER",
+ "exportGroups": [
+ {
+ "name": "Servers",
+ "exports": [
+ {
+ "name": "host_port",
+ "value": "${LISTEN_HOST}:${site.global.port}"
+ }
+ ]
+ }
+ ],
+ "components": [
+ {
+ "name": "LISTEN",
+ "compExports": "Servers-host_port",
+ "commands": [
+ {
+ "exec": "nc -l {$conf:@//site/global/port}"
+ }
+ ]
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json
new file mode 100644
index 0000000..b3ee0cf
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json
@@ -0,0 +1,17 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "256"
+ },
+ "LISTEN": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
new file mode 100644
index 0000000..c8ada2c
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json
@@ -0,0 +1,16 @@
+{
+ "schemaVersion": "2.1",
+ "application": {
+ "name": "SLEEPER",
+ "components": [
+ {
+ "name": "SLEEP_100",
+ "commands": [
+ {
+ "exec": "sleep 180"
+ }
+ ]
+ }
+ ]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
new file mode 100644
index 0000000..be7b962
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json
@@ -0,0 +1,17 @@
+{
+ "schema" : "http://example.org/specification/v2.0.0",
+ "metadata" : {
+ },
+ "global" : {
+ },
+ "components": {
+ "slider-appmaster": {
+ "yarn.memory": "256"
+ },
+ "SLEEP_100": {
+ "yarn.role.priority": "1",
+ "yarn.component.instances": "1",
+ "yarn.memory": "256"
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
index 3bc7ce2..4bd9842 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -33,8 +33,7 @@ import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
-import org.apache.slider.providers.agent.application.metadata.OSPackage;
-import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.apache.slider.providers.agent.application.metadata.Package;
import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -126,13 +125,10 @@ public class TestAgentClientProvider2 {
Metainfo metainfo = new Metainfo();
Application app = new Application();
metainfo.setApplication(app);
- OSSpecific osSpecific = new OSSpecific();
- osSpecific.setOsType("any");
- app.addOSSpecific(osSpecific);
- OSPackage pkg = new OSPackage();
- osSpecific.addOSPackage(pkg);
+ Package pkg = new Package();
pkg.setName("app.tar");
pkg.setType("tarball");
+ app.getPackages().add(pkg);
File clientInstallPath = new File("/tmp/file1");
String appName = "name";
@@ -267,7 +263,7 @@ public class TestAgentClientProvider2 {
client.actionClient(args);
}catch(SliderException e) {
log.info(e.getMessage());
- Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo.xml"));
+ Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo"));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index 38e7db1..f1d2c10 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -52,6 +52,7 @@ import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.CommandOrder;
import org.apache.slider.providers.agent.application.metadata.CommandScript;
import org.apache.slider.providers.agent.application.metadata.Component;
+import org.apache.slider.providers.agent.application.metadata.ComponentCommand;
import org.apache.slider.providers.agent.application.metadata.ComponentExport;
import org.apache.slider.providers.agent.application.metadata.ConfigFile;
import org.apache.slider.providers.agent.application.metadata.DefaultConfig;
@@ -264,7 +265,7 @@ public class TestAgentProviderService {
+ "</metainfo>";
@Test
- public void testRegistration() throws IOException {
+ public void testRegistration() throws Exception {
ConfTree tree = new ConfTree();
tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
@@ -305,6 +306,11 @@ public class TestAgentProviderService {
doReturn(cs).when(mockAps).getScriptPathFromMetainfo(anyString());
Metainfo metainfo = new Metainfo();
metainfo.setApplication(new Application());
+
+ Component hm = new Component();
+ hm.setName("HBASE_MASTER");
+ metainfo.getApplication().addComponent(hm);
+
doReturn(metainfo).when(mockAps).getApplicationMetainfo(any(SliderFileSystem.class), anyString());
Configuration conf = new Configuration();
@@ -318,6 +324,7 @@ public class TestAgentProviderService {
eq("mockcontainer_1"),
any(HeartBeatResponse.class),
eq("scripts/hbase_master.py"),
+ eq((ComponentCommand)null),
eq(600L));
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
@@ -422,7 +429,7 @@ public class TestAgentProviderService {
}
private AgentProviderService prepareProviderServiceForAgentStateTests()
- throws IOException {
+ throws Exception {
ContainerLaunchContext ctx = createNiceMock(ContainerLaunchContext.class);
Container container = createNiceMock(Container.class);
String role = "HBASE_MASTER";
@@ -455,7 +462,7 @@ public class TestAgentProviderService {
metainfo.setApplication(application);
doReturn(metainfo).when(mockAps).getApplicationMetainfo(
any(SliderFileSystem.class), anyString());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
@@ -463,13 +470,14 @@ public class TestAgentProviderService {
doReturn(true).when(mockAps).isMaster(anyString());
doNothing().when(mockAps).addInstallCommand(eq("HBASE_MASTER"),
eq("mockcontainer_1"), any(HeartBeatResponse.class),
- eq("scripts/hbase_master.py"), eq(600L));
+ eq("scripts/hbase_master.py"), eq((ComponentCommand)null),
+ eq(600L));
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
}
doNothing().when(mockAps).processAllocatedPorts(anyString(), anyString(),
- anyString(), anyMap());
+ anyString(), anyMap());
expect(access.isApplicationLive()).andReturn(true).anyTimes();
ClusterDescription desc = new ClusterDescription();
desc.setOption(OptionKeys.ZOOKEEPER_QUORUM, "host1:2181");
@@ -495,14 +503,14 @@ public class TestAgentProviderService {
ProviderRole providerRole = new ProviderRole(role, 1);
providerRoleMap.put(1, providerRole);
mockAps.rebuildContainerDetails(containers, "mockcontainer_1",
- providerRoleMap);
+ providerRoleMap);
return mockAps;
}
@Test
public void testThreeInstallFailures() throws IOException, SliderException {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
ConfTree tree = new ConfTree();
tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
@@ -553,6 +561,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
Mockito.anyLong());
doReturn(conf).when(mockAps).getConfig();
} catch (SliderException e) {
@@ -621,6 +630,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
Mockito.anyLong());
} catch (SliderException he) {
log.warn(he.getMessage());
@@ -630,14 +640,23 @@ public class TestAgentProviderService {
}
@Test
- public void testAgentStateStarted() throws IOException, SliderException {
+ public void testAgentStateStarted() throws Exception {
AggregateConf instanceDefinition = prepareConfForAgentStateTests();
AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
+
+ Metainfo metainfo = new Metainfo();
+ Application application = new Application();
+ Component hbaseMaster = new Component();
+ hbaseMaster.setName("HBASE_MASTER");
+ application.addComponent(hbaseMaster);
+ metainfo.setApplication(application);
+ doReturn(metainfo).when(mockAps).getMetaInfo();
+
Register reg = new Register();
reg.setResponseId(0);
reg.setLabel("mockcontainer_1___HBASE_MASTER");
Map<String,String> ports = new HashMap<String,String>();
- ports.put("a","100");
+ ports.put("a", "100");
reg.setAllocatedPorts(ports);
// Simulating agent in STARTED state
@@ -683,7 +702,7 @@ public class TestAgentProviderService {
}
@Test
- public void testAgentStateInstalled() throws IOException, SliderException {
+ public void testAgentStateInstalled() throws Exception, SliderException {
AggregateConf instanceDefinition = prepareConfForAgentStateTests();
AgentProviderService mockAps = prepareProviderServiceForAgentStateTests();
@@ -692,11 +711,14 @@ public class TestAgentProviderService {
CommandOrder cmdOrder = new CommandOrder();
cmdOrder.setCommand("HBASE_MASTER-START");
cmdOrder.setRequires("HBASE_MASTER-INSTALLED");
- application.addCommandOrder(cmdOrder);
+ application.getCommandOrders().add(cmdOrder);
+ Component hbaseMaster = new Component();
+ hbaseMaster.setName("HBASE_MASTER");
+ application.addComponent(hbaseMaster);
metainfo.setApplication(application);
doReturn(metainfo).when(mockAps).getApplicationMetainfo(
any(SliderFileSystem.class), anyString());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
doNothing().when(mockAps).addRoleRelatedTokens(anyMap());
Register reg = new Register();
@@ -810,11 +832,11 @@ public class TestAgentProviderService {
@Test
public void testComponentSpecificPublishes() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
Map<String, String> ports = new HashMap<String, String>();
ports.put("global.listen_port", "10010");
@@ -846,15 +868,14 @@ public class TestAgentProviderService {
}
}
-
@Test
public void testComponentSpecificPublishes2() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
doReturn(access).when(mockAps).getAmState();
PublishedExportsSet pubExpSet = new PublishedExportsSet();
@@ -935,7 +956,7 @@ public class TestAgentProviderService {
@Test
public void testProcessConfig() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
Assert.assertNotNull(metainfo.getApplication());
AgentProviderService aps = createAgentProviderService(new Configuration());
HeartBeat hb = new HeartBeat();
@@ -964,7 +985,7 @@ public class TestAgentProviderService {
new MockContainerId(1), "cid");
AgentProviderService mockAps = Mockito.spy(aps);
doNothing().when(mockAps).publishApplicationInstanceData(anyString(), anyString(), anyCollection());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
doReturn(roleClusterNodeMap).when(mockAps).getRoleClusterNodeMapping();
StateAccessForProviders access = createNiceMock(StateAccessForProviders.class);
doReturn(access).when(mockAps).getAmState();
@@ -1018,9 +1039,9 @@ public class TestAgentProviderService {
}
@Test
- public void testMetainfoParsing() throws Exception {
+ public void testMetaInfoParsing() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
Assert.assertNotNull(metainfo.getApplication());
Application application = metainfo.getApplication();
log.info("Service: " + application.toString());
@@ -1031,32 +1052,23 @@ public class TestAgentProviderService {
int found = 0;
for (Component component : components) {
if (component.getName().equals("HBASE_MASTER")) {
- Assert.assertEquals(component.getAutoStartOnFailure(), "true");
- Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.TRUE);
- Assert.assertEquals(component.getMinInstanceCount(), "1");
- Assert.assertEquals(component.getMaxInstanceCount(), "2");
+ Assert.assertEquals("true", component.getAutoStartOnFailure());
+ Assert.assertEquals(Boolean.TRUE, component.getAutoStartOnFailureBoolean());
+ Assert.assertEquals(component.getMinInstanceCountInt(), 1);
+ Assert.assertEquals(component.getMaxInstanceCountInt(), 2);
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_master.py");
Assert.assertEquals(component.getCategory(), "MASTER");
- Assert.assertEquals(component.getComponentExports().size(), 0);
Assert.assertEquals(component.getAppExports(), "QuickLinks-JMX_Endpoint,QuickLinks-Master_Status");
Assert.assertEquals(component.getCompExports(), "QuickLinks-Comp_Endpoint");
found++;
}
if (component.getName().equals("HBASE_REGIONSERVER")) {
- Assert.assertEquals(component.getAutoStartOnFailure(), "Falsee");
- Assert.assertEquals(component.getRequiresAutoRestart(), Boolean.FALSE);
+ Assert.assertEquals("Falsee", component.getAutoStartOnFailure());
+ Assert.assertEquals(Boolean.FALSE, component.getAutoStartOnFailureBoolean());
Assert.assertEquals(component.getMinInstanceCount(), "1");
Assert.assertNull(component.getMaxInstanceCount());
Assert.assertEquals(component.getCommandScript().getScript(), "scripts/hbase_regionserver.py");
Assert.assertEquals(component.getCategory(), "SLAVE");
- Assert.assertEquals(component.getComponentExports().size(), 2);
- List<ComponentExport> es = component.getComponentExports();
- ComponentExport e = es.get(0);
- Assert.assertEquals(e.getName(), "PropertyA");
- Assert.assertEquals(e.getValue(), "${THIS_HOST}:${site.global.listen_port}");
- e = es.get(1);
- Assert.assertEquals(e.getName(), "PropertyB");
- Assert.assertEquals(e.getValue(), "AConstant");
found++;
}
}
@@ -1083,10 +1095,10 @@ public class TestAgentProviderService {
}
Assert.assertEquals(found, 2);
- List<CommandOrder> cmdOrders = application.getCommandOrder();
+ List<CommandOrder> cmdOrders = application.getCommandOrders();
Assert.assertEquals(cmdOrders.size(), 2);
found = 0;
- for (CommandOrder co : application.getCommandOrder()) {
+ for (CommandOrder co : application.getCommandOrders()) {
if (co.getCommand().equals("HBASE_REGIONSERVER-START")) {
Assert.assertTrue(co.getRequires().equals("HBASE_MASTER-STARTED"));
found++;
@@ -1117,7 +1129,7 @@ public class TestAgentProviderService {
AgentProviderService aps = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps);
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
CommandScript script = mockAps.getScriptPathFromMetainfo("HBASE_MASTER");
Assert.assertEquals(script.getScript(), "scripts/hbase_master.py");
@@ -1131,25 +1143,25 @@ public class TestAgentProviderService {
+ " </comment>\n";
metainfo_1 = new ByteArrayInputStream(metainfo_1_str_bad.getBytes());
- metainfo = new MetainfoParser().parse(metainfo_1);
+ metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
Assert.assertNull(metainfo);
}
@Test
public void testMetaInfoRelatedOperations() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
InputStream metainfo_2 = new ByteArrayInputStream(metainfo_2_str.getBytes());
- Metainfo metainfo2 = new MetainfoParser().parse(metainfo_2);
+ Metainfo metainfo2 = new MetainfoParser().fromXmlStream(metainfo_2);
String role_hm = "HBASE_MASTER";
String role_hrs = "HBASE_REGIONSERVER";
AgentProviderService aps1 = createAgentProviderService(new Configuration());
AgentProviderService mockAps = Mockito.spy(aps1);
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
AgentProviderService mockAps2 = Mockito.spy(aps1);
- doReturn(metainfo2).when(mockAps2).getMetainfo();
+ doReturn(metainfo2).when(mockAps2).getMetaInfo();
Assert.assertTrue(mockAps.isMaster(role_hm));
Assert.assertFalse(mockAps.isMaster(role_hrs));
@@ -1165,11 +1177,11 @@ public class TestAgentProviderService {
}
@Test
- public void testOrchestratedAppStart() throws IOException {
+ public void testOrchestratedAppStart() throws Exception {
// App has two components HBASE_MASTER and HBASE_REGIONSERVER
// Start of HBASE_RS depends on the start of HBASE_MASTER
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
ConfTree tree = new ConfTree();
tree.global.put(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH, ".");
@@ -1222,12 +1234,15 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
Mockito.anyLong());
doNothing().when(mockAps).addStartCommand(
anyString(),
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
doNothing().when(mockAps).addGetConfigCommand(
@@ -1308,6 +1323,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
Mockito.anyLong());
hb = new HeartBeat();
@@ -1319,6 +1335,7 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
Mockito.anyLong());
// RS succeeds install but does not start
hb = new HeartBeat();
@@ -1338,6 +1355,8 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
// RS still does not start
@@ -1350,6 +1369,8 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
@@ -1372,6 +1393,8 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
Map<String, String> allocatedPorts = mockAps.getAllocatedPorts();
@@ -1389,6 +1412,8 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
// MASTER succeeds start
@@ -1415,6 +1440,8 @@ public class TestAgentProviderService {
anyString(),
any(HeartBeatResponse.class),
anyString(),
+ eq((ComponentCommand)null),
+ eq((ComponentCommand)null),
Mockito.anyLong(),
Matchers.anyBoolean());
// JDK7
@@ -1542,7 +1569,7 @@ public class TestAgentProviderService {
@Test
public void testAddInstallCommand() throws Exception {
InputStream metainfo_1 = new ByteArrayInputStream(metainfo_1_str.getBytes());
- Metainfo metainfo = new MetainfoParser().parse(metainfo_1);
+ Metainfo metainfo = new MetainfoParser().fromXmlStream(metainfo_1);
AgentProviderService aps = createAgentProviderService(new Configuration());
HeartBeatResponse hbr = new HeartBeatResponse();
@@ -1563,7 +1590,7 @@ public class TestAgentProviderService {
expect(access.isApplicationLive()).andReturn(true).anyTimes();
doReturn("HOST1").when(mockAps).getClusterInfoPropertyValue(anyString());
- doReturn(metainfo).when(mockAps).getMetainfo();
+ doReturn(metainfo).when(mockAps).getMetaInfo();
doReturn(new HashMap<String, DefaultConfig>()).when(mockAps).getDefaultConfigs();
Map<String, Map<String, ClusterNode>> roleClusterNodeMap = new HashMap<String, Map<String, ClusterNode>>();
@@ -1576,7 +1603,7 @@ public class TestAgentProviderService {
replay(access);
- mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "", 0);
+ mockAps.addInstallCommand("HBASE_MASTER", "cid1", hbr, "", null, 0);
ExecutionCommand cmd = hbr.getExecutionCommands().get(0);
String pkgs = cmd.getHostLevelParams().get(AgentKeys.PACKAGE_LIST);
Assert.assertEquals("[{\"type\":\"tarball\",\"name\":\"files/hbase-0.96.1-hadoop2-bin.tar.gz\"}]", pkgs);
@@ -1656,7 +1683,9 @@ public class TestAgentProviderService {
replay(access);
- mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", 0, Boolean.FALSE);
+ ComponentCommand startCmd = ComponentCommand.getDefaultComponentCommand();
+ ComponentCommand stopCmd = ComponentCommand.getDefaultComponentCommand("STOP");
+ mockAps.addStartCommand("HBASE_MASTER", "cid1", hbr, "", startCmd, stopCmd, 0, Boolean.FALSE);
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("hbase-site"));
Assert.assertTrue(hbr.getExecutionCommands().get(0).getConfigurations().containsKey("core-site"));
Map<String, String> hbaseSiteConf = hbr.getExecutionCommands().get(0).getConfigurations().get("hbase-site");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
index 1177e9d..30283d1 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/application/metadata/TestMetainfoParser.java
@@ -45,7 +45,7 @@ public class TestMetainfoParser {
InputStream resStream = this.getClass().getResourceAsStream(
METAINFO_XML);
MetainfoParser parser = new MetainfoParser();
- Metainfo metainfo = parser.parse(resStream);
+ Metainfo metainfo = parser.fromXmlStream(resStream);
Assert.assertNotNull(metainfo);
Assert.assertNotNull(metainfo.getApplication());
Application application = metainfo.getApplication();
@@ -67,4 +67,100 @@ public class TestMetainfoParser {
assert found;
Assert.assertEquals(0, application.getConfigFiles().size());
}
+
+ @Test
+ public void testJsonParse() throws IOException {
+ String metaInfo1_json = "{\n"
+ + "\"schemaVersion\":\"2.2\",\n"
+ + "\"application\":{\n"
+ + "\"name\": \"MEMCACHED\","
+ + "\"exportGroups\": ["
+ + "{"
+ + "\"name\": \"Servers\","
+ + "\"exports\": ["
+ + "{"
+ + "\"name\": \"host_port\","
+ + "\"value\": \"${MEMCACHED_HOST}:${site.global.port}\""
+ + "}"
+ + "]"
+ + "}"
+ + "],"
+ + "\"components\": ["
+ + "{"
+ + "\"name\": \"MEMCACHED\","
+ + "\"compExports\": \"Servers-host_port\","
+ + "\"commands\": ["
+ + "{"
+ + "\"exec\": \"java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main\""
+ + "}"
+ + "]"
+ + "},"
+ + "{"
+ + "\"name\": \"MEMCACHED2\","
+ + "\"commands\": ["
+ + "{"
+ + "\"exec\": \"scripts/config.py\","
+ + "\"type\": \"PYTHON\","
+ + "\"name\": \"CONFIGURE\""
+ + "}"
+ + "],"
+ + "\"dockerContainers\": ["
+ + "{"
+ + "\"name\": \"redis\","
+ + "\"image\": \"dockerhub/redis\","
+ + "\"options\": \"-net=bridge\","
+ + "\"mounts\": ["
+ + "{"
+ + "\"containerMount\": \"/tmp/conf\","
+ + "\"hostMount\": \"{$conf:@//site/global/app_root}/conf\""
+ + "}"
+ + "]"
+ + "}"
+ + "]"
+ + "}"
+ + "]"
+ + "}"
+ + "}";
+
+ MetainfoParser parser = new MetainfoParser();
+ Metainfo mInfo = parser.fromJsonString(metaInfo1_json);
+ Assert.assertEquals("2.2", mInfo.getSchemaVersion());
+
+ Application app = mInfo.getApplication();
+ Assert.assertNotNull(app);
+
+ Assert.assertEquals("MEMCACHED", app.getName());
+ List<ExportGroup> egs = app.getExportGroups();
+ Assert.assertEquals(1, egs.size());
+ ExportGroup eg = egs.get(0);
+ Assert.assertEquals("Servers", eg.getName());
+ List<Export> exports = eg.getExports();
+ Assert.assertEquals(1, exports.size());
+ Export export = exports.get(0);
+ Assert.assertEquals("host_port", export.getName());
+ Assert.assertEquals("${MEMCACHED_HOST}:${site.global.port}", export.getValue());
+
+ List<Component> components = app.getComponents();
+ Assert.assertEquals(2, components.size());
+
+ Component c1 = mInfo.getApplicationComponent("MEMCACHED");
+ Assert.assertNotNull(c1);
+ Assert.assertEquals("MEMCACHED", c1.getName());
+ Assert.assertEquals("Servers-host_port", c1.getCompExports());
+ Assert.assertEquals(1, c1.getCommands().size());
+ ComponentCommand cmd = c1.getCommands().get(0);
+ Assert.assertEquals("START", cmd.getName());
+ Assert.assertEquals("SHELL", cmd.getType());
+ Assert.assertEquals("java -classpath /usr/myapps/memcached/*:/usr/lib/hadoop/lib/* com.thimbleware.jmemcached.Main",
+ cmd.getExec());
+
+ Component c2 = mInfo.getApplicationComponent("MEMCACHED2");
+ Assert.assertNotNull(c2);
+ Assert.assertEquals("MEMCACHED2", c2.getName());
+ Assert.assertEquals(1, c2.getCommands().size());
+ cmd = c2.getCommands().get(0);
+ Assert.assertEquals("CONFIGURE", cmd.getName());
+ Assert.assertEquals("PYTHON", cmd.getType());
+ Assert.assertEquals("scripts/config.py", cmd.getExec());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
index 95021b8..1c94eae 100644
--- a/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
+++ b/slider-funtest/src/main/groovy/org/apache/slider/funtest/framework/CommandTestBase.groovy
@@ -19,6 +19,7 @@
package org.apache.slider.funtest.framework
import groovy.transform.CompileStatic
+import org.apache.commons.lang.StringUtils
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem as HadoopFS
import org.apache.hadoop.fs.Path
@@ -622,6 +623,85 @@ abstract class CommandTestBase extends SliderTestUtils {
}
/**
+ * Create a slider app using the alternate packaging capability
+ * <p>
+ * If the extraArgs list does not contain a --wait parm then a wait
+ * duration of THAW_WAIT_TIME will be added to the launch args.
+ * @param name name
+ * @param metaInfo application metaInfo
+ * @param appTemplate application template
+ * @param resourceTemplate resource template
+ * @param extraArgs list of extra arguments to the command
+ * @param launchReportFile optional file to save the AM launch report to
+ * @return the shell
+ */
+ public SliderShell createSliderApplicationMinPkg(
+ String name,
+ String metaInfo,
+ String resourceTemplate,
+ String appTemplate,
+ List<String> extraArgs = [],
+ File launchReportFile = null) {
+
+ if (!launchReportFile) {
+ launchReportFile = createTempJsonFile()
+ }
+ // delete any previous copy of the file
+ launchReportFile.delete();
+
+ List<String> commands = [
+ ACTION_CREATE, name,
+ ARG_METAINFO, metaInfo,
+ ARG_OUTPUT, launchReportFile.absolutePath
+ ]
+
+ if (StringUtils.isNotBlank(appTemplate)) {
+ commands << ARG_TEMPLATE << appTemplate
+ }
+ if (StringUtils.isNotBlank(resourceTemplate)) {
+ commands << ARG_RESOURCES << resourceTemplate
+ }
+ if (!extraArgs.contains(ARG_WAIT)) {
+ commands << ARG_WAIT << Integer.toString(THAW_WAIT_TIME)
+ }
+
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_LOGIN_KEYTAB_NAME));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_HDFS_KEYTAB_DIR));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_AM_KEYTAB_LOCAL_PATH));
+ maybeAddCommandOption(commands,
+ [ARG_COMP_OPT, SliderKeys.COMPONENT_AM, SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL],
+ SLIDER_CONFIG.getTrimmed(SliderXmlConfKeys.KEY_KEYTAB_PRINCIPAL));
+ commands.addAll(extraArgs)
+ SliderShell shell = new SliderShell(commands)
+ if (0 != shell.execute()) {
+ // app has failed.
+
+ // grab the app report of the last known instance of this app
+ // which may not be there if it was a config failure; may be out of date
+ // from a previous run
+ log.error("Launch failed with exit code ${shell.ret}")
+ shell.dumpOutput()
+
+ // now grab that app report if it is there
+ def appReport = maybeLookupFromLaunchReport(launchReportFile)
+ String extraText = ""
+ if (appReport) {
+ log.error("Application report:\n$appReport")
+ extraText = appReport.diagnostics
+ }
+
+ fail("Application Launch Failure, exit code ${shell.ret}\n${extraText}")
+ }
+ return shell
+ }
+
+ /**
* Create a templated slider app.
* <p>
* If the extraArgs list does not contain a --wait parm then a wait
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
new file mode 100644
index 0000000..e7b0454
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentMinSleepIT.groovy
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.Endpoint
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.test.Outcome
+
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentMinSleepIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ static String CLUSTER = "test-agent-sleep-100"
+
+ static String APP_RESOURCE11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/resources.json"
+ static String APP_META11 = "../slider-core/src/test/app_packages/test_min_pkg/sleep_cmd/metainfo.json"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ @Test
+ public void testAgentRegistry() throws Throwable {
+ describe("Create a cluster using metainfo and resources only that executes sleep 100")
+ def clusterpath = buildClusterPath(CLUSTER)
+ File launchReportFile = createTempJsonFile();
+
+ SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
+ APP_META11,
+ APP_RESOURCE11,
+ null,
+ [],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ //at this point the cluster should exist.
+ assertPathExists(
+ clusterFS,
+ "Cluster parent directory does not exist",
+ clusterpath.parent)
+
+ assertPathExists(clusterFS, "Cluster directory does not exist", clusterpath)
+
+ status(0, CLUSTER)
+ expectLiveContainerCountReached(CLUSTER, "SLEEP_100", 1,
+ CONTAINER_LAUNCH_TIMEOUT)
+
+ // sleep for some manual test
+ describe("You may quickly perform manual tests against the application instance " + CLUSTER)
+ sleep(1000 * 30)
+
+ //stop
+ freeze(0, CLUSTER,
+ [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/704e8136/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
new file mode 100644
index 0000000..0d8724f
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentPingSocketIT.groovy
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import groovy.json.*
+import org.apache.hadoop.net.NetUtils
+import org.apache.hadoop.registry.client.binding.RegistryUtils
+import org.apache.hadoop.registry.client.types.Endpoint
+import org.apache.hadoop.registry.client.types.ServiceRecord
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderKeys
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.test.Outcome
+
+import static org.apache.slider.core.registry.info.CustomRegistryConstants.*
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentPingSocketIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ static String CLUSTER = "test-agent-ping-port"
+
+ static String APP_RESOURCE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/resources.json"
+ static String APP_META12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/metainfo.json"
+ static String APP_TEMPLATE12 = "../slider-core/src/test/app_packages/test_min_pkg/nc_ping_cmd/appConfig.json"
+
+
+ @Before
+ public void prepareCluster() {
+ setupCluster(CLUSTER)
+ }
+
+ @After
+ public void destroyCluster() {
+ cleanup(CLUSTER)
+ }
+
+ @Test
+ public void testAgentRegistry() throws Throwable {
+ describe("Create a cluster using metainfo, resources, and appConfig that calls nc to listen on a port")
+ def clusterpath = buildClusterPath(CLUSTER)
+ File launchReportFile = createTempJsonFile();
+
+ SliderShell shell = createSliderApplicationMinPkg(CLUSTER,
+ APP_META12,
+ APP_RESOURCE12,
+ APP_TEMPLATE12,
+ [],
+ launchReportFile)
+
+ logShell(shell)
+
+ def appId = ensureYarnApplicationIsUp(launchReportFile)
+
+ describe("Checking the exported port value and pinging it for " + CLUSTER)
+ def outfile = tmpFile(".txt")
+
+ def commands = [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ CLUSTER,
+ ARG_LISTEXP,
+ ARG_OUTPUT,
+ outfile.absolutePath
+ ]
+
+ awaitRegistryOutfileContains(outfile, commands, "servers")
+
+ // get Servers host_port folders
+ slider(EXIT_SUCCESS,
+ [
+ ACTION_REGISTRY,
+ ARG_NAME,
+ CLUSTER,
+ ARG_GETEXP,
+ "servers",
+ ARG_OUTPUT,
+ outfile.absolutePath])
+
+ describe(outfile.absolutePath)
+
+ def result = new JsonSlurper().parseText(outfile.text)
+ Map jsonResult = (Map) result
+ List host_ports = (List)jsonResult.get("host_port")
+ Map host_port = (Map)host_ports[0]
+ String host_port_val = host_port.get("value")
+ def tokens = host_port_val.tokenize(':')
+ def host = tokens[0]
+ def port = tokens[1].toInteger()
+
+ try {
+ def socket = new Socket();
+ def addr = new InetSocketAddress(host, port)
+ socket.connect(addr, 2000)
+ socket.close()
+ } catch (IOException e) {
+ throw NetUtils.wrapException(host, port, "localhost", 0, e)
+ }
+
+ //stop
+ freeze(0, CLUSTER,
+ [
+ ARG_WAIT, Integer.toString(FREEZE_WAIT_TIME),
+ ARG_MESSAGE, "final-shutdown"
+ ])
+
+ assertInYarnState(appId, YarnApplicationState.FINISHED)
+ destroy(0, CLUSTER)
+
+ //cluster now missing
+ exists(EXIT_UNKNOWN_INSTANCE, CLUSTER)
+ }
+}