You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by st...@apache.org on 2015/02/26 02:11:13 UTC
[36/48] incubator-slider git commit: SLIDER-763. Add install client
command to slider
SLIDER-763. Add install client command to slider
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/522983bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/522983bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/522983bd
Branch: refs/heads/feature/SLIDER-779_Move_to_Groovy_2.4.0
Commit: 522983bd9a760110697ccbd93dfd5c29872a1656
Parents: e23893c
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Thu Feb 19 13:38:12 2015 -0800
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Thu Feb 19 13:40:51 2015 -0800
----------------------------------------------------------------------
.../slider-pkg/clientInstallConfig-default.json | 8 +
.../command-logger/slider-pkg/metainfo.xml | 10 +
.../slider-pkg/package/scripts/cl_client.py | 48 +++
.../slider-pkg/package/scripts/client_params.py | 29 ++
.../storm/clientInstallConfig-default.json | 6 +
app-packages/storm/metainfo.xml | 10 +
app-packages/storm/package/files/storm-slider | 63 ++++
.../storm/package/files/storm-slider.py | 370 +++++++++++++++++++
.../storm/package/scripts/client_params.py | 28 ++
.../storm/package/scripts/storm_client.py | 56 +++
.../package/templates/storm-slider-env.sh.j2 | 38 ++
app-packages/storm/src/assembly/storm.xml | 7 +
.../org/apache/slider/client/SliderClient.java | 66 +++-
.../apache/slider/client/SliderClientAPI.java | 14 +-
.../slider/common/params/ActionClientArgs.java | 65 ++++
.../slider/common/params/ActionPackageArgs.java | 4 +-
.../apache/slider/common/params/Arguments.java | 3 +-
.../apache/slider/common/params/ClientArgs.java | 19 +-
.../slider/common/params/SliderActions.java | 3 +
.../providers/AbstractClientProvider.java | 20 +
.../providers/agent/AgentClientProvider.java | 304 ++++++++++++++-
.../providers/agent/AgentProviderService.java | 7 +-
.../test_command_log/client-config.json | 7 +
.../agent/actions/TestActionPackage.groovy | 20 +-
.../agent/TestAgentClientProvider2.java | 173 ++++++++-
.../lifecycle/AgentClientInstallIT.groovy | 79 ++++
.../clusters/remote/slider/slider-client.xml | 10 +-
27 files changed, 1434 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json b/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json
new file mode 100644
index 0000000..f0b84a5
--- /dev/null
+++ b/app-packages/command-logger/slider-pkg/clientInstallConfig-default.json
@@ -0,0 +1,8 @@
+{
+ "schema":"http://example.org/specification/v2.0.0",
+ "global":{
+ "client_root":"{app_install_dir}",
+ "application_id": "application_id_1"
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/metainfo.xml b/app-packages/command-logger/slider-pkg/metainfo.xml
index 5de2c37..eb02f6a 100644
--- a/app-packages/command-logger/slider-pkg/metainfo.xml
+++ b/app-packages/command-logger/slider-pkg/metainfo.xml
@@ -36,6 +36,16 @@
<timeout>600</timeout>
</commandScript>
</component>
+
+ <component>
+ <name>COMMAND_LOGGER_CLIENT</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/cl_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
</components>
<osSpecifics>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py b/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py
new file mode 100644
index 0000000..73a2675
--- /dev/null
+++ b/app-packages/command-logger/slider-pkg/package/scripts/cl_client.py
@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+import os
+from resource_management import *
+
+
+class Client(Script):
+ def install(self, env):
+ import client_params
+ self.install_packages(env)
+ File(format(client_params.client_root + '/operations.log'),
+ mode=0755,
+ content=Template('operations.log.j2', container_id = client_params.container_id, application_id = client_params.application_id)
+ )
+
+ def configure(self, env):
+ pass
+
+ def start(self, env):
+ pass
+
+ def stop(self, env):
+ pass
+
+ def status(self, env):
+ pass
+
+if __name__ == "__main__":
+ Client().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/command-logger/slider-pkg/package/scripts/client_params.py
----------------------------------------------------------------------
diff --git a/app-packages/command-logger/slider-pkg/package/scripts/client_params.py b/app-packages/command-logger/slider-pkg/package/scripts/client_params.py
new file mode 100644
index 0000000..4d5a4ff
--- /dev/null
+++ b/app-packages/command-logger/slider-pkg/package/scripts/client_params.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import *
+
+config = Script.get_config()
+
+app_install_dir = config['configurations']['global']['app_install_dir']
+client_root = config['configurations']['global']['client_root']
+
+container_id = config['configurations']['global']['container_id']
+application_id = config['configurations']['global']['application_id']
+
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/clientInstallConfig-default.json
----------------------------------------------------------------------
diff --git a/app-packages/storm/clientInstallConfig-default.json b/app-packages/storm/clientInstallConfig-default.json
new file mode 100644
index 0000000..898439d
--- /dev/null
+++ b/app-packages/storm/clientInstallConfig-default.json
@@ -0,0 +1,6 @@
+{
+ "schema":"http://example.org/specification/v2.0.0",
+ "global":{
+ "client_root":"{app_install_dir}/apache-storm-${pkg.version}"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/metainfo.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/metainfo.xml b/app-packages/storm/metainfo.xml
index d2625f4..cc96628 100644
--- a/app-packages/storm/metainfo.xml
+++ b/app-packages/storm/metainfo.xml
@@ -125,6 +125,16 @@
<timeout>600</timeout>
</commandScript>
</component>
+
+ <component>
+ <name>STORM_CLIENT</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/storm_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
</components>
<osSpecifics>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/files/storm-slider
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/storm-slider b/app-packages/storm/package/files/storm-slider
new file mode 100644
index 0000000..af492bd
--- /dev/null
+++ b/app-packages/storm/package/files/storm-slider
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Copyright 2014 The Apache Software Foundation
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Resolve links - $0 may be a softlink
+PRG="${0}"
+
+while [ -h "${PRG}" ]; do
+ ls=`ls -ld "${PRG}"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "${PRG}"`/"$link"
+ fi
+done
+
+# find python >= 2.6
+if [ -a /usr/bin/python2.6 ]; then
+ PYTHON=/usr/bin/python2.6
+fi
+
+if [ -z "$PYTHON" ]; then
+ PYTHON=/usr/bin/python
+fi
+
+# check for version
+majversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f1`
+minversion=`$PYTHON -V 2>&1 | awk '{print $2}' | cut -d'.' -f2`
+numversion=$(( 10 * $majversion + $minversion))
+if (( $numversion < 26 )); then
+ echo "Need python version > 2.6"
+ exit 1
+fi
+
+STORM_BIN_DIR=`dirname ${PRG}`
+STORM_BASE_DIR=`cd ${STORM_BIN_DIR}/..;pwd`
+export STORM_BASE_DIR
+
+export STORM_CONF_DIR="${STORM_CONF_DIR:-$STORM_BASE_DIR/conf}"
+
+if [ -f "${STORM_CONF_DIR}/storm-slider-env.sh" ]; then
+ . "${STORM_CONF_DIR}/storm-slider-env.sh"
+fi
+
+$PYTHON ${STORM_BIN_DIR}/storm-slider.py $@
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/files/storm-slider.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/files/storm-slider.py b/app-packages/storm/package/files/storm-slider.py
new file mode 100644
index 0000000..cbe441d
--- /dev/null
+++ b/app-packages/storm/package/files/storm-slider.py
@@ -0,0 +1,370 @@
+#!/usr/bin/python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import os
+from os.path import expanduser
+import sys, tempfile
+import json
+import subprocess as sub
+import re
+import shlex
+
+def is_windows():
+ return sys.platform.startswith('win')
+
+def identity(x):
+ return x
+
+def cygpath(x):
+ command = ["cygpath", "-wp", x]
+ p = sub.Popen(command,stdout=sub.PIPE)
+ output, errors = p.communicate()
+ lines = output.split("\n")
+ return lines[0]
+
+if sys.platform == "cygwin":
+ normclasspath = cygpath
+else:
+ normclasspath = identity
+
+SLIDER_DIR = os.getenv('SLIDER_HOME', None)
+
+if SLIDER_DIR == None or (not os.path.exists(SLIDER_DIR)):
+ print "Unable to find SLIDER_HOME. Please configure SLIDER_HOME before running storm-slider"
+ sys.exit(1)
+
+USER_CONF_DIR = os.path.expanduser("~/.storm")
+
+if os.getenv('STORM_BASE_DIR', None) != None:
+ STORM_DIR = os.getenv('STORM_BASE_DIR', None)
+elif os.getenv('STORM_HOME', None) != None:
+ STORM_DIR = os.getenv('STORM_HOME', None)
+else:
+ print "Either STORM_BASE_DIR or STORM_HOME must be set."
+ sys.exit(1)
+
+CMD_OPTS = {}
+CONFIG_OPTS = []
+JAR_JVM_OPTS = shlex.split(os.getenv('STORM_JAR_JVM_OPTS', ''))
+pid = os.getpid()
+CONFFILE = os.path.join(tempfile.gettempdir(),"storm."+str(pid)+".json")
+SLIDER_CLIENT_CONF = os.path.join(SLIDER_DIR,'conf','slider-client.xml')
+SLIDER_CMD = os.path.join(SLIDER_DIR,'bin','slider.py')
+JAVA_HOME = os.getenv('JAVA_HOME', None)
+JAVA_CMD= 'java' if not JAVA_HOME else os.path.join(JAVA_HOME, 'bin', 'java')
+if is_windows():
+ JAVA_CMD = JAVA_CMD + '.exe'
+STORM_THRIFT_TRANSPORT_KEY = "storm.thrift.transport"
+NIMBUS_HOST_KEY = "nimbus.host"
+NIMBUS_THRIFT_PORT_KEY = "nimbus.thrift.port"
+
+if not os.path.isfile(JAVA_CMD):
+ print "Unable to find "+JAVA_CMD+" please check JAVA_HOME"
+ sys.exit(1)
+
+def get_config_opts():
+ global CONFIG_OPTS
+ return "-Dstorm.options=" + (','.join(CONFIG_OPTS)).replace(' ', "%%%%")
+
+def get_jars_full(adir):
+ files = os.listdir(adir)
+ ret = []
+ for f in files:
+ if f.endswith(".jar"):
+ ret.append(os.path.join(adir, f))
+ return ret
+
+def get_classpath(extrajars):
+ ret = (get_jars_full(os.path.join(STORM_DIR ,"lib")))
+ ret.extend(extrajars)
+
+ sep = ";" if is_windows() else ":"
+ return normclasspath(sep.join(ret))
+
+def print_remoteconfvalue(name):
+ """Syntax: [storm-slider --app remoteconfvalue conf-name]
+
+ Prints out the value for conf-name in the cluster's Storm configs.
+ This command must be run on a cluster machine.
+ """
+ storm_conf = storm_conf_values([name])
+ for conf in storm_conf:
+ print conf
+
+def parse_args(string):
+ r"""Takes a string of whitespace-separated tokens and parses it into a list.
+ Whitespace inside tokens may be quoted with single quotes, double quotes or
+ backslash (similar to command-line arguments in bash).
+
+ >>> parse_args(r'''"a a" 'b b' c\ c "d'd" 'e"e' 'f\'f' "g\"g" "i""i" 'j''j' k" "k l' l' mm n\\n''')
+ ['a a', 'b b', 'c c', "d'd", 'e"e', "f'f", 'g"g', 'ii', 'jj', 'k k', 'l l', 'mm', r'n\n']
+ """
+ re_split = re.compile(r'''((?:
+ [^\s"'\\] |
+ "(?: [^"\\] | \\.)*" |
+ '(?: [^'\\] | \\.)*' |
+ \\.
+ )+)''', re.VERBOSE)
+ args = re_split.split(string)[1::2]
+ args = [re.compile(r'"((?:[^"\\]|\\.)*)"').sub('\\1', x) for x in args]
+ args = [re.compile(r"'((?:[^'\\]|\\.)*)'").sub('\\1', x) for x in args]
+ return [re.compile(r'\\(.)').sub('\\1', x) for x in args]
+
+def exec_storm_class(klass, jvmtype="-server", jvmopts=[], extrajars=[], args=[], fork=False):
+ global CONFFILE
+ storm_log_dir = os.path.join(STORM_DIR, "logs")
+ all_args = [
+ "java", jvmtype, get_config_opts(),
+ "-Dstorm.home=" + STORM_DIR,
+ "-cp", get_classpath(extrajars),
+ ] + jvmopts + [klass] + list(args)
+ print "Running: " + " ".join(all_args)
+ if is_windows():
+ sub.call([JAVA_CMD] + all_args[1:])
+ else:
+ os.execvp(JAVA_CMD, all_args) # replaces the current process and never returns
+
+def jar(jarfile, klass, *args):
+ """Syntax: [storm-slider --app jar topology-jar-path class ...]
+
+ Runs the main method of class with the specified arguments.
+ The storm jars and configs in ~/.storm are put on the classpath.
+ The process is configured so that StormSubmitter
+ (http://nathanmarz.github.com/storm/doc/backtype/storm/StormSubmitter.html)
+ will upload the jar at topology-jar-path when the topology is submitted.
+ """
+ exec_storm_class(
+ klass,
+ jvmtype="-client",
+ extrajars=[jarfile, USER_CONF_DIR, os.path.join(STORM_DIR ,"bin")],
+ args=args,
+ jvmopts=JAR_JVM_OPTS + ["-Dstorm.jar=" + jarfile])
+
+def kill(*args):
+ """Syntax: [storm-slider --app kill topology-name [-w wait-time-secs]]
+
+ Kills the topology with the name topology-name. Storm will
+ first deactivate the topology's spouts for the duration of
+ the topology's message timeout to allow all messages currently
+ being processed to finish processing. Storm will then shutdown
+ the workers and clean up their state. You can override the length
+ of time Storm waits between deactivation and shutdown with the -w flag.
+ """
+ exec_storm_class(
+ "backtype.storm.command.kill_topology",
+ args=args,
+ jvmtype="-client",
+ extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")])
+
+def activate(*args):
+ """Syntax: [storm-slider --app activate topology-name]
+
+ Activates the specified topology's spouts.
+ """
+ exec_storm_class(
+ "backtype.storm.command.activate",
+ args=args,
+ jvmtype="-client",
+ extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")])
+
+def listtopos(*args):
+ """Syntax: [storm-slider --app list]
+
+ List the running topologies and their statuses.
+ """
+ exec_storm_class(
+ "backtype.storm.command.list",
+ args=args,
+ jvmtype="-client",
+ extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")])
+
+def deactivate(*args):
+ """Syntax: [storm-slider --app deactivate topology-name]
+
+ Deactivates the specified topology's spouts.
+ """
+ exec_storm_class(
+ "backtype.storm.command.deactivate",
+ args=args,
+ jvmtype="-client",
+ extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR , "bin")])
+
+def rebalance(*args):
+ """Syntax: [storm-slider --app rebalance topology-name [-w wait-time-secs] [-n new-num-workers] [-e component=parallelism]*]
+
+ Sometimes you may wish to spread out where the workers for a topology
+ are running. For example, let's say you have a 10 node cluster running
+ 4 workers per node, and then let's say you add another 10 nodes to
+ the cluster. You may wish to have Storm spread out the workers for the
+ running topology so that each node runs 2 workers. One way to do this
+ is to kill the topology and resubmit it, but Storm provides a "rebalance"
+ command that provides an easier way to do this.
+
+ Rebalance will first deactivate the topology for the duration of the
+ message timeout (overridable with the -w flag) and then redistribute
+ the workers evenly around the cluster. The topology will then return to
+ its previous state of activation (so a deactivated topology will still
+ be deactivated and an activated topology will go back to being activated).
+
+ The rebalance command can also be used to change the parallelism of a running topology.
+ Use the -n and -e switches to change the number of workers or number of executors of a component
+ respectively.
+ """
+ exec_storm_class(
+ "backtype.storm.command.rebalance",
+ args=args,
+ jvmtype="-client",
+ extrajars=[USER_CONF_DIR, os.path.join(STORM_DIR,"bin")])
+
+def version():
+ """Syntax: [storm-slider --app version]
+ Prints the version number of this Storm release.
+ """
+ releasefile = os.path.join(STORM_DIR, "RELEASE")
+ if os.path.exists(releasefile):
+ print open(releasefile).readline().strip()
+ else:
+ print "Unknown"
+
+def quicklinks():
+ """Syntax: [storm-slider --app version]
+ Prints the quicklinks information of storm-slider registry"
+ """
+ global CMD_OPTS
+ all_args = ["slider", "registry", "--getconf", "quicklinks","--format", "json", "--name"]
+ if 'app_name' in CMD_OPTS.keys():
+ all_args.append(CMD_OPTS['app_name'])
+ else:
+ print_usage()
+ sys.exit(1)
+
+ if 'user' in CMD_OPTS.keys():
+ all_args.append( "--user "+CMD_OPTS['user'])
+
+ #os.spawnvp(os.P_WAIT,SLIDER_CMD, all_args)
+ cmd = [SLIDER_CMD] + all_args[1:]
+ if is_windows():
+ cmd = ['python'] + cmd
+
+ sub.call(cmd)
+
+def get_storm_config_json():
+ global CMD_OPTS
+ all_args = ["slider.py", "registry", "--getconf", "storm-site","--format", "json", "--dest", CONFFILE, "--name"]
+ if 'app_name' in CMD_OPTS.keys():
+ all_args.append(CMD_OPTS['app_name'])
+ else:
+ print_usage()
+ sys.exit(1)
+
+ if 'user' in CMD_OPTS.keys():
+ all_args.append( "--user "+CMD_OPTS['user'])
+
+ #os.spawnvp(os.P_WAIT,SLIDER_CMD, all_args)
+ cmd = [SLIDER_CMD] + all_args[1:]
+
+ if is_windows():
+ cmd = ['python'] + cmd
+
+ sub.call(cmd)
+ if not os.path.exists(CONFFILE):
+ print "Failed to read slider deployed storm config"
+ sys.exit(1)
+
+def storm_conf_values(keys):
+ file = open(CONFFILE,"r")
+ data = json.load(file)
+ storm_args = []
+ for key in keys:
+ if data.has_key(key):
+ storm_args.append(key+"="+data[key])
+ return storm_args
+
+def print_commands():
+ """Print all client commands and link to documentation"""
+ print "Commands:\n\t", "\n\t".join(sorted(COMMANDS.keys()))
+ print "\nHelp:", "\n\thelp", "\n\thelp <command>"
+
+def print_usage(command=None):
+ """Print one help message or list of available commands"""
+ if command != None:
+ if COMMANDS.has_key(command):
+ print (COMMANDS[command].__doc__ or
+ "No documentation provided for <%s>" % command)
+ else:
+ print "<%s> is not a valid command" % command
+ else:
+ print "Please provide yarn app name followed by command"
+ print "storm-slider --app --user"
+ print_commands()
+
+
+def unknown_command(*args):
+ print "Unknown command: [storm-slider %s]" % ' '.join(sys.argv[1:])
+ print_usage()
+
+COMMANDS = {"jar": jar, "kill": kill, "remoteconfvalue": print_remoteconfvalue,
+ "activate": activate, "deactivate": deactivate, "rebalance": rebalance, "help": print_usage,
+ "list": listtopos, "version": version, "quicklinks" : quicklinks}
+
+def parse_config(config_list):
+ global CONFIG_OPTS
+ if len(config_list) > 0:
+ for config in config_list:
+ CONFIG_OPTS.append(config)
+
+def parse_config_opts(args):
+ curr = args[:]
+ curr.reverse()
+ global CMD_OPTS
+ global CONFIG_OPTS
+ args_list = []
+ while len(curr) > 0:
+ token = curr.pop()
+ if token == "--app":
+ CMD_OPTS['app_name'] = curr.pop() if (len(curr) != 0) else None
+ elif token == "--user":
+ CMD_OPTS['user'] = curr.pop() if (len(curr) != 0) else None
+ elif token == "-c" and len(curr) != 0:
+ CONFIG_OPTS.append(curr.pop())
+ else:
+ args_list.append(token)
+ return args_list
+
+def main():
+ args = parse_config_opts(sys.argv[1:])
+ if len(args) < 1:
+ print_usage()
+ sys.exit(-1)
+ COMMAND = args[0]
+ ARGS = args[1:]
+ if (COMMAND != 'help'):
+ get_storm_config_json()
+ storm_conf = storm_conf_values([NIMBUS_HOST_KEY,NIMBUS_THRIFT_PORT_KEY,STORM_THRIFT_TRANSPORT_KEY])
+ parse_config(storm_conf)
+ try:
+ (COMMANDS.get(COMMAND, unknown_command))(*ARGS)
+ except:
+ print "Unexpected error:", sys.exc_info()[0]
+ raise
+ finally:
+ if (os.path.isfile(CONFFILE)):
+ os.remove(CONFFILE)
+
+if __name__ == "__main__":
+ main()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/scripts/client_params.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/client_params.py b/app-packages/storm/package/scripts/client_params.py
new file mode 100644
index 0000000..924a202
--- /dev/null
+++ b/app-packages/storm/package/scripts/client_params.py
@@ -0,0 +1,28 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import *
+
+config = Script.get_config()
+
+app_install_dir = config['configurations']['global']['app_install_dir']
+client_root = config['configurations']['global']['client_root']
+
+slider_home_dir = config['configurations']['global']['slider_home_dir']
+java64_home = config['configurations']['global']['java64_home']
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/scripts/storm_client.py
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/scripts/storm_client.py b/app-packages/storm/package/scripts/storm_client.py
new file mode 100644
index 0000000..6d36edd
--- /dev/null
+++ b/app-packages/storm/package/scripts/storm_client.py
@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from storm import storm
+from service import service
+
+class Client(Script):
+ def install(self, env):
+ import client_params
+ self.install_packages(env)
+ File(format(client_params.client_root + '/bin/storm-slider'),
+ content=StaticFile("storm-slider"),
+ mode=0755
+ )
+ File(format(client_params.client_root + '/bin/storm-slider.py'),
+ content=StaticFile("storm-slider.py"),
+ mode=0755
+ )
+ File(format(client_params.client_root + '/conf/storm-slider-env.sh'),
+ mode=0755,
+ content=Template('storm-slider-env.sh.j2', java64_home = client_params.java64_home, slider_home_dir = client_params.slider_home_dir)
+ )
+
+ def configure(self, env):
+ pass
+
+ def start(self, env):
+ pass
+
+ def stop(self, env):
+ pass
+
+ def status(self, env):
+ pass
+
+if __name__ == "__main__":
+ Client().execute()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/package/templates/storm-slider-env.sh.j2
----------------------------------------------------------------------
diff --git a/app-packages/storm/package/templates/storm-slider-env.sh.j2 b/app-packages/storm/package/templates/storm-slider-env.sh.j2
new file mode 100644
index 0000000..8022a4b
--- /dev/null
+++ b/app-packages/storm/package/templates/storm-slider-env.sh.j2
@@ -0,0 +1,38 @@
+{#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#}
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#/*
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+export JAVA_HOME={{java64_home}}
+export SLIDER_HOME={{slider_home_dir}}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/app-packages/storm/src/assembly/storm.xml
----------------------------------------------------------------------
diff --git a/app-packages/storm/src/assembly/storm.xml b/app-packages/storm/src/assembly/storm.xml
index f7dcf13..6a6e750 100644
--- a/app-packages/storm/src/assembly/storm.xml
+++ b/app-packages/storm/src/assembly/storm.xml
@@ -42,6 +42,12 @@
<fileMode>0755</fileMode>
</file>
<file>
+ <source>clientInstall-default.json</source>
+ <outputDirectory>/</outputDirectory>
+ <filtered>true</filtered>
+ <fileMode>0755</fileMode>
+ </file>
+ <file>
<source>metainfo.xml</source>
<outputDirectory>/</outputDirectory>
<filtered>true</filtered>
@@ -65,6 +71,7 @@
<exclude>target/**</exclude>
<exclude>appConfig-default.json</exclude>
<exclude>appConfig-secured-default.json</exclude>
+ <exclude>clientInstall-default.json</exclude>
<exclude>metainfo.xml</exclude>
</excludes>
<fileMode>0755</fileMode>
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
index 5cbe945..77b5e96 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClient.java
@@ -78,6 +78,7 @@ import org.apache.slider.common.SliderKeys;
import org.apache.slider.common.SliderXmlConfKeys;
import org.apache.slider.common.params.AbstractActionArgs;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
+import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionExistsArgs;
import org.apache.slider.common.params.ActionInstallKeytabArgs;
@@ -172,6 +173,8 @@ import java.io.StringWriter;
import java.io.Writer;
import java.net.InetSocketAddress;
import java.net.URISyntaxException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -204,7 +207,7 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
private String deployedClusterName;
/**
- * Cluster opaerations against the deployed cluster -will be null
+ * Cluster operations against the deployed cluster -will be null
* if no bonding has yet taken place
*/
private SliderClusterOperations sliderClusterOperations;
@@ -385,6 +388,8 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
// actions
if (ACTION_PACKAGE.equals(action)) {
exitCode = actionPackage(serviceArgs.getActionPackageArgs());
+ } else if (ACTION_CLIENT.equals(action)) {
+ exitCode = actionClient(serviceArgs.getActionClientArgs());
} else if (ACTION_INSTALL_PACKAGE.equals(action)) {
exitCode = actionInstallPkg(serviceArgs.getActionInstallPackageArgs());
} else if (ACTION_INSTALL_KEYTAB.equals(action)) {
@@ -869,6 +874,65 @@ public class SliderClient extends AbstractSliderLaunchedService implements RunSe
}
@Override
+ public int actionClient(ActionClientArgs clientInfo) throws
+ SliderException,
+ IOException {
+
+ if(!clientInfo.install) {
+ throw new BadCommandArgumentsException(
+ "Only install command is supported for the client.\n"
+ + CommonArgs.usage(serviceArgs, ACTION_CLIENT));
+ }
+
+ if (clientInfo.installLocation == null) {
+ throw new BadCommandArgumentsException(
+ "A valid install location must be provided for the client.\n"
+ + CommonArgs.usage(serviceArgs, ACTION_CLIENT));
+ } else {
+ if (!clientInfo.installLocation.exists()) {
+ throw new BadCommandArgumentsException("Install path does not exist at " +
+ clientInfo.installLocation.getAbsolutePath());
+ }
+ if (!clientInfo.installLocation.isDirectory()) {
+ throw new BadCommandArgumentsException("Install path is not a valid directory " +
+ clientInfo.installLocation.getAbsolutePath());
+ }
+ }
+
+ File pkgFile;
+ if (StringUtils.isEmpty(clientInfo.packageURI)) {
+ throw new BadCommandArgumentsException("A valid application package location required.");
+ } else {
+ pkgFile = new File(clientInfo.packageURI);
+ if (!pkgFile.exists() || pkgFile.isDirectory()) {
+ throw new BadCommandArgumentsException("Unable to access supplied pkg file at " +
+ pkgFile.getAbsolutePath());
+ }
+ }
+
+ JSONObject config = null;
+ if(clientInfo.clientConfig != null) {
+ try {
+ byte[] encoded = Files.readAllBytes(clientInfo.clientConfig.toPath());
+ config = new JSONObject(new String(encoded, Charset.defaultCharset()));
+ }catch(JSONException jsonEx) {
+ log.error("Unable to read supplied config", jsonEx);
+ throw new SliderException("Invalid configuration. Must be a valid json file.", jsonEx);
+ }
+ }
+
+ // Only INSTALL is supported
+ AbstractClientProvider provider = createClientProvider(SliderProviderFactory.DEFAULT_CLUSTER_TYPE);
+ provider.processClientOperation(sliderFileSystem,
+ "INSTALL",
+ clientInfo.installLocation,
+ pkgFile,
+ config);
+ return EXIT_SUCCESS;
+ }
+
+
+ @Override
public int actionPackage(ActionPackageArgs actionPackageInfo)
throws YarnException, IOException {
if (actionPackageInfo.install) {
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
index 4fd4136..efb1f7f 100644
--- a/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
+++ b/slider-core/src/main/java/org/apache/slider/client/SliderClientAPI.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.slider.api.types.SliderInstanceDescription;
import org.apache.slider.common.params.AbstractClusterBuildingActionArgs;
import org.apache.slider.common.params.ActionAMSuicideArgs;
+import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.params.ActionDiagnosticArgs;
import org.apache.slider.common.params.ActionEchoArgs;
import org.apache.slider.common.params.ActionFlexArgs;
@@ -125,6 +126,17 @@ public interface SliderClientAPI extends Service {
throws YarnException, IOException;
/**
+ * Perform client operations such as install or configure
+ *
+ * @param clientInfo the arguments needed for client operations
+ *
+ * @throws SliderException bad arguments.
+ * @throws IOException problems related to package and destination folders
+ */
+ int actionClient(ActionClientArgs clientInfo)
+ throws IOException, SliderException;
+
+ /**
* Managing slider application package
*
* @param pkgInfo the arguments needed to upload, delete or list the package
@@ -283,7 +295,7 @@ public interface SliderClientAPI extends Service {
/**
* diagnostic operation
*
- * @param diagosticArgs diagnostic Arguments
+ * @param diagnosticArgs diagnostic Arguments
* @return 0 for success, -1 for some issues that aren't errors, just
* failures to retrieve information (e.g. no application name
* specified)
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
new file mode 100644
index 0000000..4154c9f
--- /dev/null
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionClientArgs.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.common.params;
+
+import com.beust.jcommander.Parameter;
+import com.beust.jcommander.Parameters;
+
+import java.io.File;
+
+@Parameters(commandNames = {SliderActions.ACTION_CLIENT},
+ commandDescription = SliderActions.DESCRIBE_ACTION_CLIENT)
+
+public class ActionClientArgs extends AbstractActionArgs {
+
+ @Override
+ public String getActionName() {
+ return SliderActions.ACTION_CLIENT;
+ }
+
+ @Parameter(names = {ARG_INSTALL},
+ description = "Install client")
+ public boolean install;
+
+ @Parameter(names = {ARG_PACKAGE},
+ description = "Path to app package")
+ public String packageURI;
+
+ @Parameter(names = {ARG_DEST},
+ description = "The location where to install the client")
+ public File installLocation;
+
+ @Parameter(names = {ARG_CONFIG},
+ description = "Client configuration")
+ public File clientConfig;
+
+ /**
+ * Get the min #of params expected
+ *
+ * @return the min number of params in the {@link #parameters} field
+ */
+ public int getMinParams() {
+ return 0;
+ }
+
+ @Override
+ public int getMaxParams() {
+ return 1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
index d5bd293..e98aba1 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ActionPackageArgs.java
@@ -22,7 +22,7 @@ import com.beust.jcommander.Parameter;
import com.beust.jcommander.Parameters;
@Parameters(commandNames = {SliderActions.ACTION_PACKAGE},
- commandDescription = SliderActions.DESCRIBE_ACTION_INSTALL_PACKAGE)
+ commandDescription = SliderActions.DESCRIBE_ACTION_PACKAGE)
public class ActionPackageArgs extends AbstractActionArgs {
@@ -31,7 +31,7 @@ public class ActionPackageArgs extends AbstractActionArgs {
return SliderActions.ACTION_PACKAGE;
}
- @Parameter(names = {ARG_PKGINSTALL},
+ @Parameter(names = {ARG_INSTALL},
description = "Install package operation")
public boolean install;
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
index 511d01e..789b285 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/Arguments.java
@@ -81,7 +81,7 @@ public interface Arguments {
String ARG_PACKAGE = "--package";
String ARG_PATH = "--path";
String ARG_PKGDELETE = "--delete";
- String ARG_PKGINSTALL = "--install";
+ String ARG_INSTALL = "--install";
String ARG_PKGINSTANCES = "--instances";
String ARG_PKGLIST = "--list";
String ARG_PROVIDER = "--provider";
@@ -105,6 +105,7 @@ public interface Arguments {
String ARG_ZKHOSTS = "--zkhosts";
String ARG_ZKPATH = "--zkpath";
String ARG_ZKPORT = "--zkport";
+ String ARG_CONFIG = "--config";
/**
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
index fc466a1..fcc1b81 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/ClientArgs.java
@@ -58,6 +58,7 @@ public class ClientArgs extends CommonArgs {
private final ActionHelpArgs actionHelpArgs = new ActionHelpArgs();
private final ActionInstallPackageArgs actionInstallPackageArgs = new ActionInstallPackageArgs();
private final ActionPackageArgs actionPackageArgs = new ActionPackageArgs();
+ private final ActionClientArgs actionClientArgs = new ActionClientArgs();
private final ActionInstallKeytabArgs actionInstallKeytabArgs = new ActionInstallKeytabArgs();
private final ActionKeytabArgs actionKeytabArgs = new ActionKeytabArgs();
private final ActionKillContainerArgs actionKillContainerArgs =
@@ -96,6 +97,7 @@ public class ClientArgs extends CommonArgs {
actionHelpArgs,
actionInstallPackageArgs,
actionPackageArgs,
+ actionClientArgs,
actionInstallKeytabArgs,
actionKeytabArgs,
actionKillContainerArgs,
@@ -141,17 +143,15 @@ public class ClientArgs extends CommonArgs {
return actionBuildArgs;
}
- public ActionInstallPackageArgs getActionInstallPackageArgs() {
- return actionInstallPackageArgs; }
+ public ActionInstallPackageArgs getActionInstallPackageArgs() { return actionInstallPackageArgs; }
- public ActionPackageArgs getActionPackageArgs() {
- return actionPackageArgs; }
+ public ActionClientArgs getActionClientArgs() { return actionClientArgs; }
- public ActionInstallKeytabArgs getActionInstallKeytabArgs() {
- return actionInstallKeytabArgs; }
+ public ActionPackageArgs getActionPackageArgs() { return actionPackageArgs; }
- public ActionKeytabArgs getActionKeytabArgs() {
- return actionKeytabArgs; }
+ public ActionInstallKeytabArgs getActionInstallKeytabArgs() { return actionInstallKeytabArgs; }
+
+ public ActionKeytabArgs getActionKeytabArgs() { return actionKeytabArgs; }
public ActionUpdateArgs getActionUpdateArgs() {
return actionUpdateArgs;
@@ -257,6 +257,9 @@ public class ClientArgs extends CommonArgs {
} else if (SliderActions.ACTION_PACKAGE.equals(action)) {
bindCoreAction(actionPackageArgs);
+ } else if (SliderActions.ACTION_CLIENT.equals(action)) {
+ bindCoreAction(actionClientArgs);
+
} else if (SliderActions.ACTION_INSTALL_KEYTAB.equals(action)) {
bindCoreAction(actionInstallKeytabArgs);
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
index e58ad2d..2599927 100644
--- a/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
+++ b/slider-core/src/main/java/org/apache/slider/common/params/SliderActions.java
@@ -48,6 +48,7 @@ public interface SliderActions {
String ACTION_INSTALL_PACKAGE = "install-package";
String ACTION_PACKAGE = "package";
String ACTION_INSTALL_KEYTAB = "install-keytab";
+ String ACTION_CLIENT = "client";
String ACTION_KEYTAB = "keytab";
String DESCRIBE_ACTION_AM_SUICIDE =
"Tell the Slider Application Master to simulate a process failure by terminating itself";
@@ -86,6 +87,8 @@ public interface SliderActions {
String DESCRIBE_ACTION_VERSION =
"Print the Slider version information";
String DESCRIBE_ACTION_INSTALL_PACKAGE = "Install the application package in the home directory under sub-folder packages";
+ String DESCRIBE_ACTION_PACKAGE = "Install/list/delete application packages and list app instances that use this package";
+ String DESCRIBE_ACTION_CLIENT = "Install the application client in the specified directory";
String DESCRIBE_ACTION_INSTALL_KEYTAB = "Install the Kerberos keytab file in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_KEYTAB = "Manage a Kerberos keytab file (install, delete, list) in the sub-folder 'keytabs' of the user's Slider base directory";
String DESCRIBE_ACTION_DIAGNOSTIC = "Diagnose the configuration of the running slider application and slider client";
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
index f89f842..b5aafd4 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/AbstractClientProvider.java
@@ -28,9 +28,11 @@ import org.apache.slider.core.conf.MapOperations;
import org.apache.slider.core.exceptions.BadClusterStateException;
import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.core.launch.AbstractLauncher;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
@@ -218,4 +220,22 @@ public abstract class AbstractClientProvider extends Configured {
return Collections.emptySet();
}
+ /**
+ * Process client operations for applications such as install, configure
+ * @param fileSystem
+ * @param operation
+ * @param config
+ * @param clientPackage
+ * @param clientInstallPath
+ * @throws SliderException
+ */
+ public void processClientOperation(SliderFileSystem fileSystem,
+ String operation,
+ File clientInstallPath,
+ File clientPackage,
+ JSONObject config)
+ throws SliderException {
+ throw new SliderException("Provider does not support client operations.");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
index 0ef8a33..94d4c97 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentClientProvider.java
@@ -18,7 +18,12 @@
package org.apache.slider.providers.agent;
+import com.google.common.io.Files;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
+import org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.slider.api.InternalKeys;
@@ -38,16 +43,34 @@ import org.apache.slider.providers.ProviderUtils;
import org.apache.slider.providers.agent.application.metadata.Application;
import org.apache.slider.providers.agent.application.metadata.Component;
import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.MetainfoParser;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
/** This class implements the client-side aspects of the agent deployer */
public class AgentClientProvider extends AbstractClientProvider
@@ -229,7 +252,7 @@ public class AgentClientProvider extends AbstractClientProvider
Path agentPath = new Path(tempPath.getParent(), AgentKeys.PROVIDER_AGENT);
log.info("Automatically uploading the agent tarball at {}", agentPath);
fileSystem.getFileSystem().mkdirs(agentPath);
- if(ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
+ if (ProviderUtils.addAgentTar(this, AGENT_TAR, fileSystem, agentPath)) {
instanceDefinition.getInternalOperations().set(
InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH,
new Path(agentPath, AGENT_TAR).toUri());
@@ -249,7 +272,7 @@ public class AgentClientProvider extends AbstractClientProvider
throw new SliderException("Error retrieving metainfo", e);
}
- if(metainfo == null) {
+ if (metainfo == null) {
log.error("Error retrieving metainfo from {}", appDef);
throw new SliderException("Error parsing metainfo file, possibly bad structure.");
}
@@ -262,4 +285,281 @@ public class AgentClientProvider extends AbstractClientProvider
return tags;
}
+
+ @Override
+ public void processClientOperation(SliderFileSystem fileSystem,
+ String operation,
+ File clientInstallPath,
+ File appPackage,
+ JSONObject config) throws SliderException {
+ // create temp folder
+ // create sub-folders app_pkg, agent_pkg, command
+ File tmpDir = Files.createTempDir();
+ log.info("Command is being executed at {}", tmpDir.getAbsolutePath());
+ File appPkgDir = new File(tmpDir, "app_pkg");
+ appPkgDir.mkdir();
+
+ File agentPkgDir = new File(tmpDir, "agent_pkg");
+ agentPkgDir.mkdir();
+
+ File cmdDir = new File(tmpDir, "command");
+ cmdDir.mkdir();
+
+ Metainfo metaInfo = null;
+ JSONObject defaultConfig = null;
+ try {
+ // expand app package into /app_pkg
+ ZipInputStream zipInputStream = null;
+ try {
+ zipInputStream = new ZipInputStream(new FileInputStream(appPackage));
+ {
+ ZipEntry zipEntry = zipInputStream.getNextEntry();
+ while (zipEntry != null) {
+ if ("metainfo.xml".equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zipInputStream.read(content, offset, size - offset);
+ }
+ metaInfo = new MetainfoParser().parse(new ByteArrayInputStream(content));
+ }
+ } else if ("clientInstallConfig-default.json".equals(zipEntry.getName())) {
+ int size = (int) zipEntry.getSize();
+ if (size != -1) {
+ log.info("Reading {} of size {}", zipEntry.getName(),
+ zipEntry.getSize());
+ byte[] content = new byte[size];
+ int offset = 0;
+ while (offset < size) {
+ offset += zipInputStream.read(content, offset, size - offset);
+ }
+ try {
+ defaultConfig = new JSONObject(new String(content, Charset.defaultCharset()));
+ } catch (JSONException jex) {
+ throw new SliderException("Unable to read default client config.", jex);
+ }
+ }
+ }
+ String filePath = appPkgDir + File.separator + zipEntry.getName();
+ if (!zipEntry.isDirectory()) {
+ extractFile(zipInputStream, filePath);
+ } else {
+ File dir = new File(filePath);
+ dir.mkdir();
+ }
+ zipInputStream.closeEntry();
+ zipEntry = zipInputStream.getNextEntry();
+ }
+ }
+ } finally {
+ zipInputStream.close();
+ }
+
+ if (metaInfo == null) {
+ throw new SliderException("Not a valid app package. Could not read metainfo.xml.");
+ }
+
+ expandAgentTar(agentPkgDir);
+
+ JSONObject commandJson = getCommandJson(defaultConfig, config, metaInfo, clientInstallPath);
+ FileWriter file = new FileWriter(new File(cmdDir, "command.json"));
+ try {
+ file.write(commandJson.toString());
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ file.flush();
+ file.close();
+ }
+
+ String client_script = null;
+ for (Component component : metaInfo.getApplication().getComponents()) {
+ if (component.getCategory().equals("CLIENT")) {
+ client_script = component.getCommandScript().getScript();
+ log.info("Installing CLIENT {} using script {}", component.getName(), client_script);
+ break;
+ }
+ }
+
+ if (SliderUtils.isUnset(client_script)) {
+ throw new SliderException("No valid CLIENT component found. Aborting install.");
+ }
+
+ runCommand(appPkgDir, agentPkgDir, cmdDir, client_script);
+
+ } catch (IOException ioex) {
+ log.warn("Error while executing INSTALL command {}", ioex.getMessage());
+ throw new SliderException("INSTALL client failed.");
+ }
+ }
+
+ protected void runCommand(
+ File appPkgDir,
+ File agentPkgDir,
+ File cmdDir,
+ String clientScript) throws SliderException {
+ int exitCode = 0;
+ Exception exp = null;
+ try {
+ String clientScriptPath = appPkgDir.getAbsolutePath() + File.separator + "package" +
+ File.separator + clientScript;
+ List<String> command = Arrays.asList(AgentKeys.PYTHON_EXE,
+ "-S",
+ clientScriptPath,
+ "INSTALL",
+ cmdDir.getAbsolutePath() + File.separator + "command.json",
+ appPkgDir.getAbsolutePath() + File.separator + "package",
+ cmdDir.getAbsolutePath() + File.separator + "command-out.json",
+ "DEBUG");
+ ProcessBuilder pb = new ProcessBuilder(command);
+ log.info("Command: " + StringUtils.join(pb.command(), " "));
+ pb.environment().put(SliderKeys.PYTHONPATH,
+ agentPkgDir.getAbsolutePath()
+ + File.separator + "slider-agent:"
+ + agentPkgDir.getAbsolutePath()
+ + File.separator + "slider-agent/jinja2");
+ log.info("{}={}", SliderKeys.PYTHONPATH, pb.environment().get(SliderKeys.PYTHONPATH));
+
+ Process proc = pb.start();
+ InputStream stderr = proc.getErrorStream();
+ InputStream stdout = proc.getInputStream();
+ BufferedReader stdOutReader = new BufferedReader(new InputStreamReader(stdout));
+ BufferedReader stdErrReader = new BufferedReader(new InputStreamReader(stderr));
+
+ proc.waitFor();
+
+ String line;
+ while ((line = stdOutReader.readLine()) != null) {
+ log.info("Stdout: " + line);
+ }
+ while ((line = stdErrReader.readLine()) != null) {
+ log.info("Stderr: " + line);
+ }
+
+ exitCode = proc.exitValue();
+ log.info("Exit value is {}", exitCode);
+ } catch (IOException e) {
+ exp = e;
+ } catch (InterruptedException e) {
+ exp = e;
+ }
+
+ if (exitCode != 0) {
+ throw new SliderException("INSTALL client failed with exit code " + exitCode);
+ }
+
+ if (exp != null) {
+ log.error("Error while executing INSTALL command {}. Stack trace {}",
+ exp.getMessage(),
+ ExceptionUtils.getStackTrace(exp));
+ throw new SliderException("INSTALL client failed.", exp);
+ }
+ }
+
+ private void expandAgentTar(File agentPkgDir) throws IOException {
+ String libDirProp =
+ System.getProperty(SliderKeys.PROPERTY_LIB_DIR);
+ File tarFile = new File(libDirProp, SliderKeys.AGENT_TAR);
+ TarArchiveInputStream tarIn = new TarArchiveInputStream(
+ new GzipCompressorInputStream(
+ new BufferedInputStream(
+ new FileInputStream(tarFile)
+ )
+ )
+ );
+ try {
+ TarArchiveEntry tarEntry = tarIn.getNextTarEntry();
+ while (tarEntry != null) {
+ File destPath = new File(agentPkgDir, tarEntry.getName());
+ if (tarEntry.isDirectory()) {
+ destPath.mkdirs();
+ } else {
+ destPath.createNewFile();
+ byte[] byteToRead = new byte[1024];
+ BufferedOutputStream buffOut =
+ new BufferedOutputStream(new FileOutputStream(destPath));
+ try {
+ int len;
+ while ((len = tarIn.read(byteToRead)) != -1) {
+ buffOut.write(byteToRead, 0, len);
+ }
+ } finally {
+ buffOut.close();
+ }
+ }
+ tarEntry = tarIn.getNextTarEntry();
+ }
+ } finally {
+ tarIn.close();
+ }
+ }
+
+ protected JSONObject getCommandJson(JSONObject defaultConfig,
+ JSONObject inputConfig,
+ Metainfo metainfo,
+ File clientInstallPath) throws SliderException {
+ try {
+ JSONObject pkgList = new JSONObject();
+ pkgList.put(AgentKeys.PACKAGE_LIST,
+ AgentProviderService.getPackageListFromApplication(metainfo.getApplication()));
+ JSONObject obj = new JSONObject();
+ obj.put("hostLevelParams", pkgList);
+
+ JSONObject configuration = new JSONObject();
+ JSONObject global = new JSONObject();
+ global.put("app_install_dir", clientInstallPath.getAbsolutePath());
+
+ if (defaultConfig != null) {
+ readConfigEntries(defaultConfig, clientInstallPath, global);
+ }
+ if (inputConfig != null) {
+ readConfigEntries(inputConfig, clientInstallPath, global);
+ }
+
+ configuration.put("global", global);
+ obj.put("configurations", configuration);
+ return obj;
+ } catch (JSONException jex) {
+ log.warn("Error while executing INSTALL command {}", jex.getMessage());
+ throw new SliderException("INSTALL client failed.");
+ }
+ }
+
+ private void readConfigEntries(JSONObject inpConfig,
+ File clientInstallPath,
+ JSONObject globalConfig)
+ throws JSONException {
+ JSONObject globalSection = inpConfig.getJSONObject("global");
+ Iterator it = globalSection.keys();
+ while (it.hasNext()) {
+ String key = (String) it.next();
+ String value = globalSection.getString(key);
+ if (SliderUtils.isSet(value)) {
+ value = value.replace("{app_install_dir}", clientInstallPath.getAbsolutePath());
+ }
+ if (globalConfig.has(key)) {
+ // last one wins
+ globalConfig.remove(key);
+ }
+ globalConfig.put(key, value);
+ }
+ }
+
+ private void extractFile(ZipInputStream zipInputStream, String filePath) throws IOException {
+ BufferedOutputStream output = new BufferedOutputStream(new FileOutputStream(filePath));
+ try {
+ byte[] bytesRead = new byte[4096];
+ int read = 0;
+ while ((read = zipInputStream.read(bytesRead)) != -1) {
+ output.write(bytesRead, 0, read);
+ }
+ } finally {
+ output.close();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index 0730beb..60236a5 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -1663,11 +1663,10 @@ public class AgentProviderService extends AbstractProviderService implements
response.addExecutionCommand(cmd);
}
- private String getPackageList() {
+ protected static String getPackageListFromApplication(Application application) {
String pkgFormatString = "{\"type\":\"%s\",\"name\":\"%s\"}";
String pkgListFormatString = "[%s]";
List<String> packages = new ArrayList();
- Application application = getMetainfo().getApplication();
if (application != null) {
List<OSSpecific> osSpecifics = application.getOSSpecifics();
if (osSpecifics != null && osSpecifics.size() > 0) {
@@ -1688,6 +1687,10 @@ public class AgentProviderService extends AbstractProviderService implements
}
}
+ private String getPackageList() {
+ return getPackageListFromApplication(getMetainfo().getApplication());
+ }
+
private void prepareExecutionCommand(ExecutionCommand cmd) {
cmd.setTaskId(taskId.incrementAndGet());
cmd.setCommandId(cmd.getTaskId() + "-1");
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/app_packages/test_command_log/client-config.json
----------------------------------------------------------------------
diff --git a/slider-core/src/test/app_packages/test_command_log/client-config.json b/slider-core/src/test/app_packages/test_command_log/client-config.json
new file mode 100644
index 0000000..c4cdac6
--- /dev/null
+++ b/slider-core/src/test/app_packages/test_command_log/client-config.json
@@ -0,0 +1,7 @@
+{
+ "schema": "http://example.org/specification/v2.0.0",
+ "global": {
+ "java_home": "/usr/jdk64/jdk1.7.0_67",
+ "container_id": "cid_1"
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
----------------------------------------------------------------------
diff --git a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
index b0b832a..fa6145e 100644
--- a/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
+++ b/slider-core/src/test/groovy/org/apache/slider/agent/actions/TestActionPackage.groovy
@@ -52,7 +52,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL
+ Arguments.ARG_INSTALL
],
)
fail("expected an exception, got a status code " + launcher.serviceExitCode)
@@ -70,7 +70,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase"
],
)
@@ -89,7 +89,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
],
@@ -100,7 +100,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
],
@@ -120,7 +120,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "unlikely_to_be_a_file_path",
],
@@ -140,7 +140,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
],
@@ -151,7 +151,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "hbase",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
Arguments.ARG_REPLACE_PKG
@@ -171,7 +171,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_PKGLIST
],
)
@@ -189,7 +189,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_PKGINSTANCES
],
)
@@ -207,7 +207,7 @@ class TestActionPackage extends AgentMiniClusterTestBase {
//varargs list of command line params
[
SliderActions.ACTION_PACKAGE,
- Arguments.ARG_PKGINSTALL,
+ Arguments.ARG_INSTALL,
Arguments.ARG_NAME, "storm",
Arguments.ARG_PACKAGE, "src/test/resources/log4j.properties",
],
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
index 0ff613a..1e4d834 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentClientProvider2.java
@@ -16,15 +16,27 @@
*/
package org.apache.slider.providers.agent;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.log4j.BasicConfigurator;
import org.apache.slider.api.InternalKeys;
+import org.apache.slider.client.SliderClient;
+import org.apache.slider.common.params.ActionClientArgs;
import org.apache.slider.common.tools.SliderFileSystem;
import org.apache.slider.core.conf.AggregateConf;
import org.apache.slider.core.conf.ConfTree;
+import org.apache.slider.core.exceptions.BadCommandArgumentsException;
+import org.apache.slider.core.exceptions.SliderException;
import org.apache.slider.providers.ProviderUtils;
+import org.apache.slider.providers.agent.application.metadata.Application;
+import org.apache.slider.providers.agent.application.metadata.Metainfo;
+import org.apache.slider.providers.agent.application.metadata.OSPackage;
+import org.apache.slider.providers.agent.application.metadata.OSSpecific;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
@@ -35,6 +47,12 @@ import org.powermock.modules.junit4.PowerMockRunner;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
import static org.easymock.EasyMock.anyObject;
import static org.easymock.EasyMock.expect;
@@ -42,13 +60,19 @@ import static org.easymock.EasyMock.expect;
*
*/
@RunWith(PowerMockRunner.class)
-@PrepareForTest(ProviderUtils.class)
+@PrepareForTest({ProviderUtils.class, ProcessBuilder.class, AgentClientProvider.class})
public class TestAgentClientProvider2 {
protected static final Logger log =
LoggerFactory.getLogger(TestAgentClientProvider2.class);
@Rule
public TemporaryFolder folder = new TemporaryFolder();
+ @BeforeClass
+ public static void initialize() {
+ BasicConfigurator.resetConfiguration();
+ BasicConfigurator.configure();
+ }
+
@Test
public void testPrepareAMAndConfigForLaunch() throws Exception {
AgentClientProvider provider = new AgentClientProvider(null);
@@ -85,4 +109,151 @@ public class TestAgentClientProvider2 {
PowerMock.verify(sfs, fs, ProviderUtils.class);
Assert.assertTrue(tree.global.containsKey(InternalKeys.INTERNAL_APPLICATION_IMAGE_PATH));
}
+
+ @Test
+ public void testGetCommandJson() throws Exception {
+ AgentClientProvider provider = new AgentClientProvider(null);
+ JSONObject defaultConfig = null;
+
+ JSONObject inputConfig = new JSONObject();
+ JSONObject global = new JSONObject();
+ global.put("a", "b");
+ global.put("d", "{app_install_dir}/d");
+ inputConfig.put("global", global);
+
+ Metainfo metainfo = new Metainfo();
+ Application app = new Application();
+ metainfo.setApplication(app);
+ OSSpecific osSpecific = new OSSpecific();
+ osSpecific.setOsType("any");
+ app.addOSSpecific(osSpecific);
+ OSPackage pkg = new OSPackage();
+ osSpecific.addOSPackage(pkg);
+ pkg.setName("app.tar");
+ pkg.setType("tarball");
+
+ File clientInstallPath = new File("/tmp/file1");
+
+ JSONObject output = provider.getCommandJson(defaultConfig,
+ inputConfig,
+ metainfo,
+ clientInstallPath);
+ JSONObject outConfigs = output.getJSONObject("configurations");
+ Assert.assertNotNull(outConfigs);
+ JSONObject outGlobal = outConfigs.getJSONObject("global");
+ Assert.assertNotNull(outGlobal);
+ Assert.assertEquals("b", outGlobal.getString("a"));
+ Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d"));
+
+ defaultConfig = new JSONObject();
+ global = new JSONObject();
+ global.put("a1", "b2");
+ global.put("a", "b-not");
+ global.put("d1", "{app_install_dir}/d");
+ defaultConfig.put("global", global);
+
+ output = provider.getCommandJson(defaultConfig,
+ inputConfig,
+ metainfo,
+ clientInstallPath);
+ outConfigs = output.getJSONObject("configurations");
+ Assert.assertNotNull(outConfigs);
+ outGlobal = outConfigs.getJSONObject("global");
+ Assert.assertNotNull(outGlobal);
+ Assert.assertEquals("b", outGlobal.getString("a"));
+ Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d"));
+ Assert.assertEquals("b2", outGlobal.getString("a1"));
+ Assert.assertEquals("/tmp/file1/d", outGlobal.getString("d1"));
+ }
+
+
+ @Test
+ public void testRunCommand() throws Exception {
+ AgentClientProvider provider = new AgentClientProvider(null);
+ File appPkgDir = new File("/tmp/pkg");
+ File agentPkgDir = new File("/tmp/agt");
+ File cmdDir = new File("/tmp/cmd");
+ String client_script = "scripts/abc.py";
+
+ List<String> commands =
+ Arrays.asList("python", "-S", "/tmp/pkg/package/scripts/abc.py", "INSTALL", "/tmp/cmd/command.json",
+ "/tmp/pkg/package", "/tmp/cmd/command-out.json", "DEBUG");
+ ProcessBuilder pbMock = PowerMock.createMock(ProcessBuilder.class);
+ Process procMock = PowerMock.createMock(Process.class);
+ PowerMock.expectNew(ProcessBuilder.class, commands).andReturn(pbMock);
+
+ expect(pbMock.environment()).andReturn(new HashMap<String, String>()).anyTimes();
+ expect(pbMock.start()).andReturn(procMock);
+ expect(pbMock.command()).andReturn(new ArrayList<String>());
+ expect(procMock.waitFor()).andReturn(0);
+ expect(procMock.exitValue()).andReturn(0);
+ expect(procMock.getErrorStream()).andReturn(IOUtils.toInputStream("stderr", "UTF-8"));
+ expect(procMock.getInputStream()).andReturn(IOUtils.toInputStream("stdout", "UTF-8"));
+
+ PowerMock.replayAll();
+
+ provider.runCommand(appPkgDir,
+ agentPkgDir,
+ cmdDir,
+ client_script);
+ PowerMock.verifyAll();
+ }
+
+ @Test
+ public void testSliderClientForInstallFailures() throws Exception {
+ SliderClient client = new SliderClient();
+ client.bindArgs(new Configuration(), "client", "--dest", "a_random_path/none", "--package", "a_random_pkg.zip");
+ ActionClientArgs args = new ActionClientArgs();
+ args.install = false;
+ try {
+ client.actionClient(args);
+ }catch(BadCommandArgumentsException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("Only install command is supported for the client"));
+ }
+
+ args.install = true;
+ try {
+ client.actionClient(args);
+ }catch(BadCommandArgumentsException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("A valid install location must be provided for the client"));
+ }
+
+ File tmpFile = File.createTempFile("del", "");
+ File dest = new File(tmpFile.getParentFile(), tmpFile.getName() + "dir");
+ args.installLocation = dest;
+ try {
+ client.actionClient(args);
+ }catch(BadCommandArgumentsException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("Install path does not exist at"));
+ }
+
+ dest.mkdir();
+ try {
+ client.actionClient(args);
+ }catch(BadCommandArgumentsException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("A valid application package location required"));
+ }
+
+ tmpFile = File.createTempFile("del", ".zip");
+ args.packageURI = tmpFile.toString();
+ args.clientConfig = tmpFile;
+ try {
+ client.actionClient(args);
+ }catch(SliderException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("Invalid configuration. Must be a valid json file"));
+ }
+
+ args.clientConfig = null;
+ try {
+ client.actionClient(args);
+ }catch(SliderException e) {
+ log.info(e.getMessage());
+ Assert.assertTrue(e.getMessage().contains("Not a valid app package. Could not read metainfo.xml"));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
----------------------------------------------------------------------
diff --git a/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
new file mode 100644
index 0000000..ef75821
--- /dev/null
+++ b/slider-funtest/src/test/groovy/org/apache/slider/funtest/lifecycle/AgentClientInstallIT.groovy
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.slider.funtest.lifecycle
+
+import groovy.io.FileType
+import groovy.transform.CompileStatic
+import groovy.util.logging.Slf4j
+import org.apache.hadoop.yarn.api.records.YarnApplicationState
+import org.apache.slider.api.ClusterDescription
+import org.apache.slider.api.StatusKeys
+import org.apache.slider.client.SliderClient
+import org.apache.slider.common.SliderExitCodes
+import org.apache.slider.common.SliderXmlConfKeys
+import org.apache.slider.common.params.Arguments
+import org.apache.slider.common.params.SliderActions
+import org.apache.slider.funtest.framework.AgentCommandTestBase
+import org.apache.slider.funtest.framework.FuntestProperties
+import org.apache.slider.funtest.framework.SliderShell
+import org.junit.After
+import org.junit.Before
+import org.junit.Test
+
+@CompileStatic
+@Slf4j
+public class AgentClientInstallIT extends AgentCommandTestBase
+ implements FuntestProperties, Arguments, SliderExitCodes, SliderActions {
+
+
+ @Test
+ public void testAgentClientInstall() throws Throwable {
+
+ describe "Install command logger client"
+ File zipFileName = new File(TEST_APP_PKG_DIR, TEST_APP_PKG_FILE).canonicalFile
+ File tmpFile = File.createTempFile("del", "");
+ File dest = new File(tmpFile.getParentFile(), tmpFile.getName() + "dir");
+ String CLIENT_CONFIG =
+ "../slider-core/src/test/app_packages/test_command_log/client-config.json"
+
+ dest.mkdir()
+
+ SliderShell shell = slider(EXIT_SUCCESS,
+ [
+ ACTION_CLIENT,
+ "--install",
+ ARG_DEST, dest.canonicalPath,
+ ARG_PACKAGE, zipFileName.absolutePath,
+ ARG_CONFIG, CLIENT_CONFIG
+ ])
+ logShell(shell)
+
+ def list = []
+
+ dest.eachFileRecurse (FileType.FILES) { file ->
+ list << file.toString()
+ }
+
+ String expectedFile1 = new File(dest, "operations.log").toString();
+ String expectedFile2 = new File(new File(dest, "command-logger-app"), "operations.log").toString();
+
+ assert list.contains(expectedFile1)
+ assert list.contains(expectedFile2)
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/522983bd/src/test/clusters/remote/slider/slider-client.xml
----------------------------------------------------------------------
diff --git a/src/test/clusters/remote/slider/slider-client.xml b/src/test/clusters/remote/slider/slider-client.xml
index f8d88eb..41dc32e 100644
--- a/src/test/clusters/remote/slider/slider-client.xml
+++ b/src/test/clusters/remote/slider/slider-client.xml
@@ -31,22 +31,22 @@
<property>
<name>slider.zookeeper.quorum</name>
- <value>c6403.ambari.apache.org:2181</value>
+ <value>c6401.ambari.apache.org:2181</value>
</property>
<property>
<name>yarn.resourcemanager.address</name>
- <value>c6403.ambari.apache.org:8050</value>
+ <value>c6401.ambari.apache.org:8050</value>
</property>
<property>
<name>yarn.resourcemanager.scheduler.address</name>
- <value>c6403.ambari.apache.org:8030</value>
+ <value>c6401.ambari.apache.org:8030</value>
</property>
<property>
<name>fs.defaultFS</name>
- <value>hdfs://c6403.ambari.apache.org:8020</value>
+ <value>hdfs://c6401.ambari.apache.org:8020</value>
</property>
<property>
@@ -74,7 +74,7 @@
<property>
<name>slider.test.agent.tar</name>
- <value>hdfs://c6403.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
+ <value>hdfs://c6401.ambari.apache.org:8020/slider/agent/slider-agent.tar.gz</value>
</property>
</configuration>