You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/06/11 20:28:51 UTC
[1/3] git commit: SLIDER-119. Minor cleanup of slider-agent code and
new test cases
Repository: incubator-slider
Updated Branches:
refs/heads/develop 94e177f87 -> f8b143a0f
SLIDER-119. Minor cleanup of slider-agent code and new test cases
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/2079fb86
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/2079fb86
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/2079fb86
Branch: refs/heads/develop
Commit: 2079fb86c5ac6ef9eedbe7880a2352800d34dd62
Parents: 0961276
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Mon Jun 9 14:52:59 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Mon Jun 9 14:52:59 2014 -0700
----------------------------------------------------------------------
.../src/main/python/agent/PythonExecutor.py | 15 ++++++--
.../python/resource_management/core/logger.py | 4 +-
.../core/providers/package/apt.py | 4 +-
.../python/resource_management/core/shell.py | 19 +++++++---
.../libraries/functions/__init__.py | 1 +
.../libraries/functions/format.py | 4 +-
.../libraries/functions/get_port_from_url.py | 40 ++++++++++++++++++++
.../functions/get_unique_id_and_date.py | 2 +-
.../libraries/providers/copy_from_local.py | 2 +-
.../libraries/providers/execute_hadoop.py | 2 +-
.../libraries/script/hook.py | 10 ++++-
.../libraries/script/script.py | 28 +++++++++++---
.../src/test/python/agent/TestPythonExecutor.py | 10 ++---
.../resource_management/TestLibraryFunctions.py | 33 ++++++++++++++++
.../resource_management/TestPackageResource.py | 2 +-
15 files changed, 146 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/agent/PythonExecutor.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/PythonExecutor.py b/slider-agent/src/main/python/agent/PythonExecutor.py
index 142fcdd..5f29e5e 100644
--- a/slider-agent/src/main/python/agent/PythonExecutor.py
+++ b/slider-agent/src/main/python/agent/PythonExecutor.py
@@ -52,7 +52,7 @@ class PythonExecutor:
pass
def run_file(self, script, script_params, tmpoutfile, tmperrfile, timeout,
- tmpstructedoutfile, override_output_files=True,
+ tmpstructedoutfile, logger_level, override_output_files=True,
environment_vars=None):
"""
Executes the specified python file in a separate subprocess.
@@ -69,7 +69,16 @@ class PythonExecutor:
else: # Append to files
tmpout = open(tmpoutfile, 'a')
tmperr = open(tmperrfile, 'a')
- script_params += [tmpstructedoutfile]
+
+ # need to remove this file for the following case:
+ # status call 1 does not write to file; call 2 writes to file;
+ # call 3 does not write to file, so contents are still call 2's result
+ try:
+ os.unlink(tmpstructedoutfile)
+ except OSError:
+ pass # no error
+
+ script_params += [tmpstructedoutfile, logger_level]
pythonCommand = self.python_command(script, script_params)
logger.info("Running command " + pprint.pformat(pythonCommand))
process = self.launch_python_subprocess(pythonCommand, tmpout, tmperr,
@@ -100,7 +109,7 @@ class PythonExecutor:
}
logger.warn(structured_out)
else:
- structured_out = '{}'
+ structured_out = {}
if self.python_process_has_been_killed:
error = str(error) + "\n Python script has been killed due to timeout"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/core/logger.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/logger.py b/slider-agent/src/main/python/resource_management/core/logger.py
index 7124ef0..7370c97 100644
--- a/slider-agent/src/main/python/resource_management/core/logger.py
+++ b/slider-agent/src/main/python/resource_management/core/logger.py
@@ -59,6 +59,8 @@ class Logger:
@staticmethod
def _get_resource_repr(resource):
MESSAGE_MAX_LEN = 256
+ logger_level = logging._levelNames[Logger.logger.level]
+
arguments_str = ""
for x,y in resource.arguments.iteritems():
@@ -70,7 +72,7 @@ class Logger:
val = repr(y).lstrip('u')
# don't show dicts of configurations
# usually too long
- elif isinstance(y, dict):
+ elif logger_level != 'DEBUG' and isinstance(y, dict):
val = "..."
# for configs which didn't come
elif isinstance(y, UnknownConfiguration):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/package/apt.py b/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
index 8d53f96..acfc212 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/package/apt.py
@@ -15,7 +15,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-Ambari Agent
+Slider Agent
"""
@@ -23,7 +23,7 @@ from resource_management.core.providers.package import PackageProvider
from resource_management.core import shell
from resource_management.core.logger import Logger
-INSTALL_CMD = "/usr/bin/apt-get --assume-yes install %s"
+INSTALL_CMD = "/usr/bin/apt-get --force-yes --assume-yes install %s"
REMOVE_CMD = "/usr/bin/apt-get -y -q remove %s"
CHECK_CMD = "dpkg --get-selections %s | grep -v deinstall"
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/shell.py b/slider-agent/src/main/python/resource_management/core/shell.py
index 32e917c..92312d5 100644
--- a/slider-agent/src/main/python/resource_management/core/shell.py
+++ b/slider-agent/src/main/python/resource_management/core/shell.py
@@ -20,9 +20,9 @@ Slider Agent
"""
-__all__ = ["checked_call", "call"]
+__all__ = ["checked_call", "call", "quote_bash_args"]
-import pipes
+import string
import subprocess
import threading
from multiprocessing import Queue
@@ -52,7 +52,7 @@ def _call(command, logoutput=False, throw_on_failure=True,
"""
# convert to string and escape
if isinstance(command, (list, tuple)):
- command = ' '.join(pipes.quote(x) for x in command)
+ command = ' '.join(quote_bash_args(x) for x in command)
"""
Do not su to the supplied user (need to differentiate between when to call su and when not to)
@@ -89,7 +89,7 @@ def _call(command, logoutput=False, throw_on_failure=True,
Logger.info(out)
if throw_on_failure and code:
- err_msg = ("Execution of '%s' returned %d. %s") % (command[-1], code, out)
+ err_msg = Logger.get_protected_text(("Execution of '%s' returned %d. %s") % (command[-1], code, out))
raise Fail(err_msg)
return code, out
@@ -100,4 +100,13 @@ def on_timeout(proc, q):
try:
proc.terminate()
except:
- pass
\ No newline at end of file
+ pass
+
+def quote_bash_args(command):
+ if not command:
+ return "''"
+ valid = set(string.ascii_letters + string.digits + '@%_-+=:,./')
+ for char in command:
+ if char not in valid:
+ return "'" + command.replace("'", "'\"'\"'") + "'"
+ return command
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
index 41cf666..ad30707 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/__init__.py
@@ -28,3 +28,4 @@ from resource_management.libraries.functions.check_process_status import *
from resource_management.libraries.functions.is_empty import *
from resource_management.libraries.functions.substitute_vars import *
from resource_management.libraries.functions.os_check import *
+from resource_management.libraries.functions.get_port_from_url import *
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/functions/format.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/format.py b/slider-agent/src/main/python/resource_management/libraries/functions/format.py
index 9594791..a4ea111 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/format.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/format.py
@@ -22,12 +22,12 @@ Slider Agent
__all__ = ["format"]
import sys
-import pipes
from string import Formatter
from resource_management.core.exceptions import Fail
from resource_management.core.utils import checked_unite
from resource_management.core.environment import Environment
from resource_management.core.logger import Logger
+from resource_management.core.shell import quote_bash_args
class ConfigurationFormatter(Formatter):
@@ -66,7 +66,7 @@ class ConfigurationFormatter(Formatter):
def _convert_field(self, value, conversion, is_protected):
if conversion == 'e':
- return pipes.quote(str(value))
+ return quote_bash_args(str(value))
elif conversion == 'h':
return "[PROTECTED]" if is_protected else value
elif conversion == 'p':
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py b/slider-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
new file mode 100644
index 0000000..15131f5
--- /dev/null
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/get_port_from_url.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Slider Agent
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.is_empty import *
+from resource_management.core.exceptions import Fail
+import re
+
+def get_port_from_url(address):
+ """
+ Return port from URL. If address is UnknownConfiguration,
+ UnknownConfiguration will be returned. If no port was found, Fail will be
+ raised.
+ """
+ if not is_empty(address):
+ port = re.findall(":([\d]{1,5})(?=/|$)", address)
+ if port:
+ return port[0]
+ raise Fail("No port in URL:{0}".format(address))
+ else:
+ return address
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py b/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
index d23d097..431b56c 100644
--- a/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
+++ b/slider-agent/src/main/python/resource_management/libraries/functions/get_unique_id_and_date.py
@@ -25,7 +25,7 @@ import datetime
from resource_management.core import shell
def get_unique_id_and_date():
- out = shell.checked_call("hostid")[1]
+ out = shell.checked_call("hostid")[1].split('\n')[-1] # bugfix: take the lastline (stdin is not tty part cut)
id = out.strip()
now = datetime.datetime.now()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py b/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
index 0dfecd7..f6cd496 100644
--- a/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/copy_from_local.py
@@ -63,7 +63,7 @@ class CopyFromLocalProvider(Provider):
pass
if mode:
- dir_mode = oct(mode)
+ dir_mode = oct(mode)[1:]
chmod_cmd = format('fs -chmod {dir_mode} {dest_path}')
ExecuteHadoop(chmod_cmd,
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
index 61b2ddb..cbc6668 100644
--- a/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
+++ b/slider-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py
@@ -32,7 +32,7 @@ class ExecuteHadoopProvider(Provider):
principal = self.resource.principal
if isinstance(command, (list, tuple)):
- command = ' '.join(pipes.quote(x) for x in command)
+ command = ' '.join(quote_bash_args(x) for x in command)
with Environment.get_instance_copy() as env:
if self.resource.security_enabled and not self.resource.kinit_override:
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/script/hook.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/hook.py b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
index 19b5204..399baac 100644
--- a/slider-agent/src/main/python/resource_management/libraries/script/hook.py
+++ b/slider-agent/src/main/python/resource_management/libraries/script/hook.py
@@ -45,10 +45,16 @@ class Hook(Script):
Runs custom hook
"""
args = sys.argv
+
#Hook script to run
- args[0] = args[0].replace(args[1], command)
+ args[0] = args[0].replace('before-'+args[1], command)
+ args[0] = args[0].replace('after-'+args[1], command)
+
#Hook script base directory
- args[3] = args[3].replace(args[1], command)
+ args[3] = args[3].replace('before-'+args[1], command)
+ args[3] = args[3].replace('after-'+args[1], command)
+
+ args[1] = command.split("-")[1]
cmd = [sys.executable]
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/libraries/script/script.py b/slider-agent/src/main/python/resource_management/libraries/script/script.py
index 12fda80..624d65e 100644
--- a/slider-agent/src/main/python/resource_management/libraries/script/script.py
+++ b/slider-agent/src/main/python/resource_management/libraries/script/script.py
@@ -33,6 +33,15 @@ from resource_management.core.resources import Directory
from resource_management.libraries.script.config_dictionary import ConfigDictionary
from resource_management.libraries.script.repo_installer import RepoInstaller
+USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL>
+
+<COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...)
+<JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json
+<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/stacks/HDP/2.0.6/services/HDFS
+<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
+<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
+"""
+
class Script(object):
"""
Executes a command for custom service. stdout and stderr are written to
@@ -52,9 +61,8 @@ class Script(object):
def put_structured_out(self, sout):
Script.structuredOut.update(sout)
try:
- structuredOut = json.dumps(Script.structuredOut)
with open(self.stroutfile, 'w') as fp:
- json.dump(structuredOut, fp)
+ json.dump(Script.structuredOut, fp)
except IOError:
Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
@@ -75,15 +83,23 @@ class Script(object):
cherr.setFormatter(formatter)
logger.addHandler(cherr)
logger.addHandler(chout)
+
# parse arguments
- if len(sys.argv) < 5:
- logger.error("Script expects at least 4 arguments")
- sys.exit(1)
+ if len(sys.argv) < 6:
+ logger.error("Script expects at least 5 arguments")
+ print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
+ sys.exit(1)
+
command_name = str.lower(sys.argv[1])
- # parse command parameters
command_data_file = sys.argv[2]
basedir = sys.argv[3]
self.stroutfile = sys.argv[4]
+ logging_level = sys.argv[5]
+
+ logging_level_str = logging._levelNames[logging_level]
+ chout.setLevel(logging_level_str)
+ logger.setLevel(logging_level_str)
+
try:
with open(command_data_file, "r") as f:
pass
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/test/python/agent/TestPythonExecutor.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestPythonExecutor.py b/slider-agent/src/test/python/agent/TestPythonExecutor.py
index 7cec409..7d0eaaf 100644
--- a/slider-agent/src/test/python/agent/TestPythonExecutor.py
+++ b/slider-agent/src/test/python/agent/TestPythonExecutor.py
@@ -57,7 +57,7 @@ class TestPythonExecutor(TestCase):
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = None
thread = Thread(target = executor.run_file, args = ("fake_puppetFile",
- ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout))
+ ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout,"INFO"))
thread.start()
time.sleep(0.1)
subproc_mock.finished_event.wait()
@@ -87,7 +87,7 @@ class TestPythonExecutor(TestCase):
subproc_mock.returncode = 0
thread = Thread(target = executor.run_file, args = ("fake_puppetFile", ["arg1", "arg2"],
tmpoutfile, tmperrfile,
- PYTHON_TIMEOUT_SECONDS, tmpstrucout))
+ PYTHON_TIMEOUT_SECONDS, tmpstrucout, "INFO"))
thread.start()
time.sleep(0.1)
subproc_mock.should_finish_event.set()
@@ -103,7 +103,7 @@ class TestPythonExecutor(TestCase):
executor = PythonExecutor("/tmp", AgentConfig("", ""))
environment_vars = [("PYTHONPATH", "a:b")]
os_env_copy_mock.return_value = actual_vars
- executor.run_file("script.pynot", ["a","b"], "", "", 10, "", True, environment_vars)
+ executor.run_file("script.pynot", ["a","b"], "", "", 10, "", "INFO", True, environment_vars)
self.assertEquals(2, len(os_env_copy_mock.return_value))
def test_execution_results(self):
@@ -124,9 +124,9 @@ class TestPythonExecutor(TestCase):
executor.runShellKillPgrp = runShellKillPgrp_method
subproc_mock.returncode = 0
subproc_mock.should_finish_event.set()
- result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile)
+ result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile, "INFO")
self.assertEquals(result, {'exitcode': 0, 'stderr': 'Dummy err', 'stdout': 'Dummy output',
- 'structuredOut': {'msg': 'Unable to read structured output from ' + tmpstroutfile}})
+ 'structuredOut': {}})
def test_is_successfull(self):
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/test/python/resource_management/TestLibraryFunctions.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/resource_management/TestLibraryFunctions.py b/slider-agent/src/test/python/resource_management/TestLibraryFunctions.py
new file mode 100644
index 0000000..4e6b6c3
--- /dev/null
+++ b/slider-agent/src/test/python/resource_management/TestLibraryFunctions.py
@@ -0,0 +1,33 @@
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+from unittest import TestCase
+from resource_management.libraries.functions import get_port_from_url
+from resource_management.core.exceptions import Fail
+class TestLibraryFunctions(TestCase):
+
+ def test_get_port_from_url(self):
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080"))
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080/"))
+ self.assertEqual("8080",get_port_from_url("host:8080"))
+ self.assertEqual("8080",get_port_from_url("host:8080/"))
+ self.assertEqual("8080",get_port_from_url("host:8080/dots_in_url8888:"))
+ self.assertEqual("8080",get_port_from_url("protocol://host:8080/dots_in_url8888:"))
+ self.assertEqual("8080",get_port_from_url("127.0.0.1:8080"))
+ self.assertRaises(Fail, get_port_from_url, "http://host/no_port")
+ self.assertRaises(Fail, get_port_from_url, "127.0.0.1:808080")
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/2079fb86/slider-agent/src/test/python/resource_management/TestPackageResource.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/resource_management/TestPackageResource.py b/slider-agent/src/test/python/resource_management/TestPackageResource.py
index 02aa3fb..af1f9cf 100644
--- a/slider-agent/src/test/python/resource_management/TestPackageResource.py
+++ b/slider-agent/src/test/python/resource_management/TestPackageResource.py
@@ -36,7 +36,7 @@ class TestPackageResource(TestCase):
Package("some_package",
)
call_mock.assert_called_with('dpkg --get-selections some_package | grep -v deinstall')
- shell_mock.assert_called_with("/usr/bin/apt-get --assume-yes install some_package")
+ shell_mock.assert_called_with("/usr/bin/apt-get --force-yes --assume-yes install some_package")
@patch.object(shell, "call")
[3/3] git commit: Merge branch 'feature/slider_119_agent_cleanup'
into develop
Posted by sm...@apache.org.
Merge branch 'feature/slider_119_agent_cleanup' into develop
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/f8b143a0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/f8b143a0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/f8b143a0
Branch: refs/heads/develop
Commit: f8b143a0fe2d7555ef6995ba6fd345186f35576e
Parents: 94e177f 3dfb319
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jun 11 10:55:07 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jun 11 10:55:07 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo-v1_5/appConfig.json | 2 +-
app-packages/hbase-v0_96/appConfig.json | 4 +-
.../python/agent/CustomServiceOrchestrator.py | 2 +
.../src/main/python/agent/PythonExecutor.py | 15 ++++++--
.../python/resource_management/core/logger.py | 4 +-
.../core/providers/package/apt.py | 4 +-
.../python/resource_management/core/shell.py | 19 +++++++---
.../libraries/functions/__init__.py | 1 +
.../libraries/functions/format.py | 4 +-
.../libraries/functions/get_port_from_url.py | 40 ++++++++++++++++++++
.../functions/get_unique_id_and_date.py | 2 +-
.../libraries/providers/copy_from_local.py | 2 +-
.../libraries/providers/execute_hadoop.py | 2 +-
.../libraries/script/hook.py | 10 ++++-
.../libraries/script/script.py | 28 +++++++++++---
.../src/test/python/agent/TestActionQueue.py | 1 +
.../agent/TestCustomServiceOrchestrator.py | 2 +-
.../src/test/python/agent/TestPythonExecutor.py | 10 ++---
.../resource_management/TestLibraryFunctions.py | 33 ++++++++++++++++
.../resource_management/TestPackageResource.py | 2 +-
.../providers/agent/AgentProviderService.java | 17 ++++++---
.../agent/TestAgentProviderService.java | 1 +
22 files changed, 165 insertions(+), 40 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/f8b143a0/app-packages/accumulo-v1_5/appConfig.json
----------------------------------------------------------------------
diff --cc app-packages/accumulo-v1_5/appConfig.json
index 49ac995,f453b91..aee8213
--- a/app-packages/accumulo-v1_5/appConfig.json
+++ b/app-packages/accumulo-v1_5/appConfig.json
@@@ -25,8 -25,7 +25,8 @@@
"site.global.accumulo_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
+ "site.global.monitor_protocol": "http",
- "site.accumulo-site.instance.dfs.dir": "/apps/accumulo/data",
+ "site.accumulo-site.instance.dfs.dir": "${DEFAULT_DATA_DIR}/data",
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.secret": "DEFAULT",
"site.accumulo-site.tserver.memory.maps.max": "80M",
[2/3] git commit: SLIDER-115. Move default data folder location for
HBase and Accumulo as sub-folder of the cluster directory
Posted by sm...@apache.org.
SLIDER-115. Move default data folder location for HBase and Accumulo as sub-folder of the cluster directory
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3dfb3197
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3dfb3197
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3dfb3197
Branch: refs/heads/develop
Commit: 3dfb31978a2a7f85f6b8e98317ab641a5eda060b
Parents: 2079fb8
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Wed Jun 11 10:02:06 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Wed Jun 11 10:02:06 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo-v1_5/appConfig.json | 2 +-
app-packages/hbase-v0_96/appConfig.json | 4 ++--
.../main/python/agent/CustomServiceOrchestrator.py | 2 ++
.../src/test/python/agent/TestActionQueue.py | 1 +
.../python/agent/TestCustomServiceOrchestrator.py | 2 +-
.../providers/agent/AgentProviderService.java | 17 +++++++++++------
.../providers/agent/TestAgentProviderService.java | 1 +
7 files changed, 19 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/app-packages/accumulo-v1_5/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo-v1_5/appConfig.json b/app-packages/accumulo-v1_5/appConfig.json
index 52f8624..f453b91 100644
--- a/app-packages/accumulo-v1_5/appConfig.json
+++ b/app-packages/accumulo-v1_5/appConfig.json
@@ -25,7 +25,7 @@
"site.global.accumulo_root_password": "secret",
"site.global.user_group": "hadoop",
"site.global.security_enabled": "false",
- "site.accumulo-site.instance.dfs.dir": "/apps/accumulo/data",
+ "site.accumulo-site.instance.dfs.dir": "${DEFAULT_DATA_DIR}/data",
"site.accumulo-site.instance.zookeeper.host": "${ZK_HOST}",
"site.accumulo-site.instance.secret": "DEFAULT",
"site.accumulo-site.tserver.memory.maps.max": "80M",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/app-packages/hbase-v0_96/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase-v0_96/appConfig.json b/app-packages/hbase-v0_96/appConfig.json
index fd884cb..1d828e2 100644
--- a/app-packages/hbase-v0_96/appConfig.json
+++ b/app-packages/hbase-v0_96/appConfig.json
@@ -23,8 +23,8 @@
"site.hbase-site.hbase.hstore.flush.retries.number": "120",
"site.hbase-site.hbase.client.keyvalue.maxsize": "10485760",
"site.hbase-site.hbase.hstore.compactionThreshold": "3",
- "site.hbase-site.hbase.rootdir": "${NN_URI}/apps/hbase/data",
- "site.hbase-site.hbase.stagingdir": "${NN_URI}/apps/hbase/staging",
+ "site.hbase-site.hbase.rootdir": "${DEFAULT_DATA_DIR}/data",
+ "site.hbase-site.hbase.stagingdir": "${DEFAULT_DATA_DIR}/staging",
"site.hbase-site.hbase.regionserver.handler.count": "60",
"site.hbase-site.hbase.regionserver.global.memstore.lowerLimit": "0.38",
"site.hbase-site.hbase.hregion.memstore.block.multiplier": "2",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
index 328084d..6296033 100644
--- a/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
+++ b/slider-agent/src/main/python/agent/CustomServiceOrchestrator.py
@@ -88,6 +88,7 @@ class CustomServiceOrchestrator():
py_file_list = [script_tuple]
# filter None values
filtered_py_file_list = [i for i in py_file_list if i]
+ logger_level = logging.getLevelName(logger.level)
# Executing hooks and script
ret = None
@@ -101,6 +102,7 @@ class CustomServiceOrchestrator():
ret = self.python_executor.run_file(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
+ logger_level,
override_output_files,
environment_vars)
# Next run_file() invocations should always append to current output
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/test/python/agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestActionQueue.py b/slider-agent/src/test/python/agent/TestActionQueue.py
index 2e1e4cf..b3a840c 100644
--- a/slider-agent/src/test/python/agent/TestActionQueue.py
+++ b/slider-agent/src/test/python/agent/TestActionQueue.py
@@ -304,6 +304,7 @@ class TestActionQueue(TestCase):
def side_effect(py_file, script_params,
tmpoutfile, tmperrfile, timeout,
tmpstrucoutfile,
+ loglevel,
override_output_files,
environment_vars):
unfreeze_flag.wait()
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
index 6f20db9..d2439b1 100644
--- a/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
+++ b/slider-agent/src/test/python/agent/TestCustomServiceOrchestrator.py
@@ -152,7 +152,7 @@ class TestCustomServiceOrchestrator(TestCase):
}
ret = orchestrator.runCommand(command, "out.txt", "err.txt")
## Check that override_output_files was true only during first call
- self.assertEquals(run_file_mock.call_args_list[0][0][6], True)
+ self.assertEquals(run_file_mock.call_args_list[0][0][7], True)
run_file_mock.reset_mock()
# Case when we force another command
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
index f62198c..5ac142b 100644
--- a/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
+++ b/slider-core/src/main/java/org/apache/slider/providers/agent/AgentProviderService.java
@@ -462,7 +462,7 @@ public class AgentProviderService extends AbstractProviderService implements
}
private void processFolderPaths(Map<String, String> folders, String containerId, String hostFqdn) {
- for(String key : folders.keySet()) {
+ for (String key : folders.keySet()) {
workFolders.put(String.format("%s-%s-%s", hostFqdn, containerId, key), folders.get(key));
}
@@ -657,7 +657,7 @@ public class AgentProviderService extends AbstractProviderService implements
return cmdParams;
}
- private void setInstallCommandConfigurations(ExecutionCommand cmd) {
+ private void setInstallCommandConfigurations(ExecutionCommand cmd) throws SliderException {
ConfTreeOperations appConf = getAmState().getAppConfSnapshot();
Map<String, Map<String, String>> configurations = buildCommandConfigurations(appConf);
cmd.setConfigurations(configurations);
@@ -751,7 +751,8 @@ public class AgentProviderService extends AbstractProviderService implements
return this.allocatedPorts;
}
- private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf) {
+ private Map<String, Map<String, String>> buildCommandConfigurations(ConfTreeOperations appConf)
+ throws SliderException {
Map<String, Map<String, String>> configurations = new TreeMap<>();
Map<String, String> tokens = getStandardTokenMap(appConf);
@@ -767,12 +768,16 @@ public class AgentProviderService extends AbstractProviderService implements
return configurations;
}
- private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) {
+ private Map<String, String> getStandardTokenMap(ConfTreeOperations appConf) throws SliderException {
Map<String, String> tokens = new HashMap<>();
String nnuri = appConf.get("site.fs.defaultFS");
tokens.put("${NN_URI}", nnuri);
tokens.put("${NN_HOST}", URI.create(nnuri).getHost());
tokens.put("${ZK_HOST}", appConf.get(OptionKeys.ZOOKEEPER_HOSTS));
+ tokens.put("${DEFAULT_DATA_DIR}", getAmState()
+ .getInternalsSnapshot()
+ .getGlobalOptions()
+ .getMandatoryOption(OptionKeys.INTERNAL_DATA_DIR_PATH));
return tokens;
}
@@ -855,7 +860,7 @@ public class AgentProviderService extends AbstractProviderService implements
@Override
public void applyInitialRegistryDefinitions(URL amWebAPI,
- ServiceInstanceData instanceData) throws IOException {
+ ServiceInstanceData instanceData) throws IOException {
super.applyInitialRegistryDefinitions(amWebAPI, instanceData);
try {
@@ -863,7 +868,7 @@ public class AgentProviderService extends AbstractProviderService implements
CustomRegistryConstants.AGENT_REST_API,
new RegisteredEndpoint(
new URL(amWebAPI, SLIDER_PATH_AGENTS),
- "Agent REST API") );
+ "Agent REST API"));
} catch (URISyntaxException e) {
throw new IOException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3dfb3197/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
----------------------------------------------------------------------
diff --git a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
index b0ac967..918fda3 100644
--- a/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
+++ b/slider-core/src/test/java/org/apache/slider/providers/agent/TestAgentProviderService.java
@@ -761,6 +761,7 @@ public class TestAgentProviderService {
treeOps.getGlobalOptions().put(AgentKeys.JAVA_HOME, "java_home");
treeOps.set(OptionKeys.APPLICATION_NAME, "HBASE");
treeOps.set("site.fs.defaultFS", "hdfs://HOST1:8020/");
+ treeOps.set("internal.data.dir.path", "hdfs://HOST1:8020/database");
treeOps.set(OptionKeys.ZOOKEEPER_HOSTS, "HOST1");
treeOps.set("config_types", "hbase-site");
treeOps.getGlobalOptions().put("site.hbase-site.a.port", "${HBASE_MASTER.ALLOCATED_PORT}");