You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@slider.apache.org by sm...@apache.org on 2014/07/28 16:20:39 UTC
git commit: SLIDER-116. Log handling for long-lived applications
Repository: incubator-slider
Updated Branches:
refs/heads/develop a2521c011 -> 3b565ba4d
SLIDER-116. Log handling for long-lived applications
Project: http://git-wip-us.apache.org/repos/asf/incubator-slider/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-slider/commit/3b565ba4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-slider/tree/3b565ba4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-slider/diff/3b565ba4
Branch: refs/heads/develop
Commit: 3b565ba4d9c40176ed67acc6c02e7bf9ac1fafe3
Parents: a2521c0
Author: Sumit Mohanty <sm...@hortonworks.com>
Authored: Sun Jul 27 21:47:24 2014 -0700
Committer: Sumit Mohanty <sm...@hortonworks.com>
Committed: Sun Jul 27 21:47:24 2014 -0700
----------------------------------------------------------------------
app-packages/accumulo/appConfig.json | 2 +-
app-packages/hbase/appConfig.json | 2 +-
slider-agent/conf/agent.ini | 6 ++--
.../src/main/python/agent/AgentConfig.py | 6 ++--
slider-agent/src/main/python/agent/main.py | 13 +++++++--
.../core/providers/system.py | 30 +++++++++++++-------
slider-agent/src/test/python/agent/TestMain.py | 18 ++++++------
.../src/test/python/agent/TestRegistration.py | 6 ++--
8 files changed, 52 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/app-packages/accumulo/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/accumulo/appConfig.json b/app-packages/accumulo/appConfig.json
index 8fe9a18..6b7033e 100644
--- a/app-packages/accumulo/appConfig.json
+++ b/app-packages/accumulo/appConfig.json
@@ -8,7 +8,7 @@
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/accumulo-${accumulo.version}-bin.tar.gz",
"site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/accumulo-${accumulo.version}",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/app-packages/hbase/appConfig.json
----------------------------------------------------------------------
diff --git a/app-packages/hbase/appConfig.json b/app-packages/hbase/appConfig.json
index 20cd436..03349eb 100644
--- a/app-packages/hbase/appConfig.json
+++ b/app-packages/hbase/appConfig.json
@@ -9,7 +9,7 @@
"java_home": "/usr/jdk64/jdk1.7.0_45",
"package_list": "files/hbase-${hbase.version}-bin.tar.gz",
"site.global.app_user": "yarn",
- "site.global.app_log_dir": "${AGENT_LOG_ROOT}/app/log",
+ "site.global.app_log_dir": "${AGENT_LOG_ROOT}",
"site.global.app_pid_dir": "${AGENT_WORK_ROOT}/app/run",
"site.global.app_root": "${AGENT_WORK_ROOT}/app/install/hbase-${hbase.version}",
"site.global.app_install_dir": "${AGENT_WORK_ROOT}/app/install",
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/conf/agent.ini
----------------------------------------------------------------------
diff --git a/slider-agent/conf/agent.ini b/slider-agent/conf/agent.ini
index b52bec9..7b9d57d 100644
--- a/slider-agent/conf/agent.ini
+++ b/slider-agent/conf/agent.ini
@@ -28,11 +28,11 @@ app_run_dir=app/run
app_dbg_cmd=
debug_mode_enabled=true
-app_task_dir=app/command-log
-app_log_dir=app/log
+app_task_dir=.
+app_log_dir=.
app_tmp_dir=app/tmp
-log_dir=infra/log
+log_dir=.
run_dir=infra/run
version_file=infra/version
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/src/main/python/agent/AgentConfig.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/AgentConfig.py b/slider-agent/src/main/python/agent/AgentConfig.py
index 7f8a350..57b1542 100644
--- a/slider-agent/src/main/python/agent/AgentConfig.py
+++ b/slider-agent/src/main/python/agent/AgentConfig.py
@@ -44,11 +44,11 @@ app_run_dir=app/run
app_dbg_cmd=
debug_mode_enabled=true
-app_task_dir=app/command-log
-app_log_dir=app/log
+app_task_dir=.
+app_log_dir=.
app_tmp_dir=app/tmp
-log_dir=infra/log
+log_dir=.
run_dir=infra/run
version_file=infra/version
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/src/main/python/agent/main.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/agent/main.py b/slider-agent/src/main/python/agent/main.py
index a6a661c..105df5a 100644
--- a/slider-agent/src/main/python/agent/main.py
+++ b/slider-agent/src/main/python/agent/main.py
@@ -185,11 +185,17 @@ def main():
(options, args) = parser.parse_args()
if not 'AGENT_WORK_ROOT' in os.environ:
- parser.error("AGENT_WORK_ROOT environment variable must be set.");
+ parser.error("AGENT_WORK_ROOT environment variable must be set.")
options.root_folder = os.environ['AGENT_WORK_ROOT']
if not 'AGENT_LOG_ROOT' in os.environ:
- parser.error("AGENT_LOG_ROOT environment variable must be set.");
+ parser.error("AGENT_LOG_ROOT environment variable must be set.")
options.log_folder = os.environ['AGENT_LOG_ROOT']
+ all_log_folders = [x.strip() for x in options.log_folder.split(',')]
+ if len(all_log_folders) > 1:
+ options.log_folder = all_log_folders[0]
+
+ # If there are multiple log folder, separate by comma, pick one
+
if not options.label:
parser.error("label is required.");
@@ -232,6 +238,9 @@ def main():
logger.info("Using AGENT_WORK_ROOT = " + options.root_folder)
logger.info("Using AGENT_LOG_ROOT = " + options.log_folder)
+ if len(all_log_folders) > 1:
+ logger.info("Selected log folder from available: " + ",".join(all_log_folders))
+
server_url = SERVER_STATUS_URL.format(
agentConfig.get(AgentConfig.SERVER_SECTION, 'hostname'),
agentConfig.get(AgentConfig.SERVER_SECTION, 'port'),
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/src/main/python/resource_management/core/providers/system.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/main/python/resource_management/core/providers/system.py b/slider-agent/src/main/python/resource_management/core/providers/system.py
index 23000f5..e273498 100644
--- a/slider-agent/src/main/python/resource_management/core/providers/system.py
+++ b/slider-agent/src/main/python/resource_management/core/providers/system.py
@@ -22,9 +22,8 @@ Slider Agent
from __future__ import with_statement
-#import grp
+import platform
import os
-#import pwd
import time
import shutil
from resource_management.core import shell
@@ -33,15 +32,23 @@ from resource_management.core import ExecuteTimeoutException
from resource_management.core.providers import Provider
from resource_management.core.logger import Logger
+IS_WINDOWS = platform.system() == "Windows"
+
+if not IS_WINDOWS:
+ import grp
+ import pwd
def _coerce_uid(user):
try:
uid = int(user)
except ValueError:
- #try:
- # uid = pwd.getpwnam(user).pw_uid
- #except KeyError:
- raise Fail("User %s doesn't exist." % user)
+ if not IS_WINDOWS:
+ try:
+ uid = pwd.getpwnam(user).pw_uid
+ except KeyError:
+ raise Fail("User %s doesn't exist." % user)
+ else:
+ raise Fail("User %s doesn't exist." % user)
return uid
@@ -49,10 +56,13 @@ def _coerce_gid(group):
try:
gid = int(group)
except ValueError:
- #try:
- # gid = grp.getgrnam(group).gr_gid
- #except KeyError:
- raise Fail("Group %s doesn't exist." % group)
+ if not IS_WINDOWS:
+ try:
+ gid = grp.getgrnam(group).gr_gid
+ except KeyError:
+ raise Fail("Group %s doesn't exist." % group)
+ else:
+ raise Fail("User %s doesn't exist." % user)
return gid
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/src/test/python/agent/TestMain.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestMain.py b/slider-agent/src/test/python/agent/TestMain.py
index 9ef1cad..dc3a199 100644
--- a/slider-agent/src/test/python/agent/TestMain.py
+++ b/slider-agent/src/test/python/agent/TestMain.py
@@ -33,6 +33,7 @@ import tempfile
from agent.Controller import Controller
from optparse import OptionParser
+logger = logging.getLogger()
class TestMain(unittest.TestCase):
def setUp(self):
@@ -148,8 +149,7 @@ class TestMain(unittest.TestCase):
@patch("sys.exit")
@patch("os.path.isfile")
@patch("os.path.isdir")
- @patch("hostname.hostname")
- def test_perform_prestart_checks(self, hostname_mock, isdir_mock, isfile_mock,
+ def test_perform_prestart_checks(self, isdir_mock, isfile_mock,
exit_mock, remove_mock):
main.config = AgentConfig("", "")
@@ -221,7 +221,6 @@ class TestMain(unittest.TestCase):
@patch.object(main, "setup_logging")
@patch.object(main, "bind_signal_handlers")
- @patch.object(main, "stop_agent")
@patch.object(main, "update_config_from_file")
@patch.object(main, "perform_prestart_checks")
@patch.object(main, "write_pid")
@@ -231,13 +230,15 @@ class TestMain(unittest.TestCase):
@patch.object(Controller, "start")
@patch.object(Controller, "join")
@patch("optparse.OptionParser.parse_args")
- def test_main(self, parse_args_mock, join_mock, start_mock,
+ @patch.object(Controller, "is_alive")
+ def test_main(self, isAlive_mock, parse_args_mock, join_mock, start_mock,
Controller_init_mock, try_to_connect_mock,
update_log_level_mock, write_pid_mock,
perform_prestart_checks_mock,
- update_config_from_file_mock, stop_mock,
+ update_config_from_file_mock,
bind_signal_handlers_mock, setup_logging_mock):
Controller_init_mock.return_value = None
+ isAlive_mock.return_value = False
options = MagicMock()
parse_args_mock.return_value = (options, MagicMock)
@@ -245,16 +246,16 @@ class TestMain(unittest.TestCase):
#testing call without command-line arguments
os.environ["AGENT_WORK_ROOT"] = os.path.join(tmpdir, "work")
- os.environ["AGENT_LOG_ROOT"] = os.path.join(tmpdir, "log")
+ os.environ["AGENT_LOG_ROOT"] = ",".join([os.path.join(tmpdir, "log"),os.path.join(tmpdir, "log2")])
main.main()
self.assertTrue(setup_logging_mock.called)
self.assertTrue(bind_signal_handlers_mock.called)
- self.assertTrue(stop_mock.called)
self.assertTrue(update_config_from_file_mock.called)
self.assertTrue(perform_prestart_checks_mock.called)
self.assertTrue(write_pid_mock.called)
self.assertTrue(update_log_level_mock.called)
+ self.assertTrue(options.log_folder == os.path.join(tmpdir, "log"))
try_to_connect_mock.assert_called_once_with(ANY, -1, ANY)
self.assertTrue(start_mock.called)
@@ -281,7 +282,7 @@ class TestMain(unittest.TestCase):
@patch.object(Controller, "join")
@patch.object(Controller, "is_alive")
@patch("optparse.OptionParser.parse_args")
- def test_main(self, parse_args_mock, isAlive_mock, join_mock, start_mock,
+ def test_main2(self, parse_args_mock, isAlive_mock, join_mock, start_mock,
Controller_init_mock, AgentConfig_set_mock,
try_to_connect_mock,
update_log_level_mock, write_pid_mock,
@@ -305,4 +306,5 @@ class TestMain(unittest.TestCase):
if __name__ == "__main__":
+ logging.basicConfig(format='%(asctime)s %(message)s', level=logging.DEBUG)
unittest.main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-slider/blob/3b565ba4/slider-agent/src/test/python/agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/slider-agent/src/test/python/agent/TestRegistration.py b/slider-agent/src/test/python/agent/TestRegistration.py
index 356e480..8647d0d 100644
--- a/slider-agent/src/test/python/agent/TestRegistration.py
+++ b/slider-agent/src/test/python/agent/TestRegistration.py
@@ -58,9 +58,9 @@ class TestRegistration(TestCase):
self.assertEquals(os.path.join(tmpdir, "app/definition"), config.getResolvedPath("app_pkg_dir"))
self.assertEquals(os.path.join(tmpdir, "app/install"), config.getResolvedPath("app_install_dir"))
- self.assertEquals(os.path.join(ver_dir, "app/log"), config.getResolvedPath("app_log_dir"))
- self.assertEquals(os.path.join(ver_dir, "infra/log"), config.getResolvedPath("log_dir"))
- self.assertEquals(os.path.join(ver_dir, "app/command-log"), config.getResolvedPath("app_task_dir"))
+ self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_log_dir"))
+ self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("log_dir"))
+ self.assertEquals(os.path.join(ver_dir, "."), config.getResolvedPath("app_task_dir"))
os.remove(ver_file)
os.removedirs(ver_dir)