You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2014/12/29 06:20:56 UTC
[2/2] ambari git commit: AMBARI-8907 - Upgrade Pack for Falcon
(jonathanhurley)
AMBARI-8907 - Upgrade Pack for Falcon (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/4af27666
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/4af27666
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/4af27666
Branch: refs/heads/trunk
Commit: 4af27666afb0892a8403302ac95333393c026aeb
Parents: 9c73f2c
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Dec 24 11:03:02 2014 -0500
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Mon Dec 29 00:19:19 2014 -0500
----------------------------------------------------------------------
.../0.5.0.2.1/package/scripts/falcon_client.py | 16 +
.../0.5.0.2.1/package/scripts/falcon_server.py | 28 +-
.../package/scripts/falcon_server_upgrade.py | 102 ++
.../FALCON/0.5.0.2.1/package/scripts/params.py | 20 +-
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 22 +-
.../stacks/2.1/FALCON/test_falcon_server.py | 33 +-
.../stacks/2.2/configs/falcon-upgrade.json | 1093 ++++++++++++++++++
7 files changed, 1307 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
index fd8c005..6f1cd56 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_client.py
@@ -25,14 +25,30 @@ class FalconClient(Script):
self.install_packages(env)
self.configure(env)
+
def configure(self, env):
import params
env.set_params(params)
falcon('client', action='config')
+
def status(self, env):
raise ClientComponentHasNoStatus()
+
+ def pre_rolling_restart(self, env):
+ import params
+ env.set_params(params)
+
+ # this function should not execute if the version can't be determined or
+ # is not at least HDP 2.2.0.0
+ if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
+ return
+
+ Logger.info("Executing Falcon Client Rolling Upgrade pre-restart")
+ Execute(format("hdp-select set hadoop-client {version}"))
+
+
if __name__ == "__main__":
FalconClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
index 0460460..536bd49 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server.py
@@ -17,7 +17,10 @@ limitations under the License.
"""
+import falcon_server_upgrade
+
from resource_management import *
+from resource_management.libraries.functions.version import *
from falcon import falcon
class FalconServer(Script):
@@ -27,7 +30,8 @@ class FalconServer(Script):
self.install_packages(env)
env.set_params(params)
- def start(self, env):
+
+ def start(self, env, rolling_restart=False):
import params
env.set_params(params)
@@ -35,13 +39,18 @@ class FalconServer(Script):
falcon('server', action='start')
- def stop(self, env):
+
+ def stop(self, env, rolling_restart=False):
import params
env.set_params(params)
falcon('server', action='stop')
+ # if performing an upgrade, backup some directories after stopping falcon
+ if rolling_restart:
+ falcon_server_upgrade.post_stop_backup()
+
def configure(self, env):
import params
@@ -50,6 +59,7 @@ class FalconServer(Script):
falcon('server', action='config')
+
def status(self, env):
import status_params
@@ -57,5 +67,19 @@ class FalconServer(Script):
check_process_status(status_params.server_pid_file)
+ def pre_rolling_restart(self, env):
+ import params
+ env.set_params(params)
+
+ # this function should not execute if the version can't be determined or
+ # is not at least HDP 2.2.0.0
+ if not params.version or compare_versions(format_hdp_stack_version(params.version), '2.2.0.0') < 0:
+ return
+
+ Logger.info("Executing Falcon Server Rolling Upgrade pre-restart")
+ Execute(format("hdp-select set falcon-server {version}"))
+ falcon_server_upgrade.pre_start_restore()
+
+
if __name__ == "__main__":
FalconServer().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
new file mode 100644
index 0000000..e6819d2
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon_server_upgrade.py
@@ -0,0 +1,102 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+import shutil
+import tarfile
+import tempfile
+
+from resource_management.core.logger import Logger
+from resource_management.core.exceptions import Fail
+
+BACKUP_TEMP_DIR = "falcon-upgrade-backup"
+BACKUP_DATA_ARCHIVE = "falcon-local-backup.tar"
+BACKUP_CONF_ARCHIVE = "falcon-conf-backup.tar"
+
+def post_stop_backup():
+ """
+ Backs up the falcon configuration and data directories as part of the
+ upgrade process.
+ :return:
+ """
+ Logger.info('Backing up Falcon data and configuration directories before upgrade...')
+ directoryMappings = _get_directory_mappings()
+
+ absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
+ if not os.path.isdir(absolute_backup_dir):
+ os.makedirs(absolute_backup_dir)
+
+ for directory in directoryMappings:
+ if not os.path.isdir(directory):
+ raise Fail("Unable to backup missing directory {0}".format(directory))
+
+ archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
+ Logger.info('Compressing {0} to {1}'.format(directory, archive))
+
+ if os.path.exists(archive):
+ os.remove(archive)
+
+ tarball = None
+ try:
+ tarball = tarfile.open(archive, "w")
+ tarball.add(directory, arcname=os.path.basename(directory))
+ finally:
+ if tarball:
+ tarball.close()
+
+
+def pre_start_restore():
+ """
+ Restores the data and configuration backups to their proper locations
+ after an upgrade has completed.
+ :return:
+ """
+ Logger.info('Restoring Falcon data and configuration directories after upgrade...')
+ directoryMappings = _get_directory_mappings()
+
+ for directory in directoryMappings:
+ archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
+ directoryMappings[directory])
+
+ if not os.path.isfile(archive):
+ raise Fail("Unable to restore missing backup archive {0}".format(archive))
+
+ Logger.info('Extracting {0} to {1}'.format(archive, directory))
+
+ tarball = None
+ try:
+ tarball = tarfile.open(archive, "r")
+ tarball.extractall(directory)
+ finally:
+ if tarball:
+ tarball.close()
+
+ # cleanup
+ shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
+
+
+def _get_directory_mappings():
+ """
+ Gets a dictionary of directory to archive name that represents the
+ directories that need to be backed up and their output tarball archive targets
+ :return: the dictionary of directory to tarball mappings
+ """
+ import params
+
+ return { params.falcon_local_dir : BACKUP_DATA_ARCHIVE,
+ params.falcon_conf_dir : BACKUP_CONF_ARCHIVE }
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
index 73258fe..82cd470 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params.py
@@ -24,14 +24,28 @@ from status_params import *
config = Script.get_config()
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
-#hadoop params
+# hadoop params
if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
- falcon_webapp_dir = "/usr/hdp/current/falcon-client/webapp"
- falcon_home = "/usr/hdp/current/falcon-client"
+
+ # if this is a server action, then use the server binaries; smoke tests
+ # use the client binaries
+ server_role_dir_mapping = { 'FALCON_SERVER' : 'falcon-server',
+ 'FALCON_SERVICE_CHECK' : 'falcon-client' }
+
+ command_role = default("/role", "")
+ if command_role not in server_role_dir_mapping:
+ command_role = 'FALCON_SERVICE_CHECK'
+
+ falcon_root = server_role_dir_mapping[command_role]
+ falcon_webapp_dir = format('/usr/hdp/current/{falcon_root}/webapp')
+ falcon_home = format('/usr/hdp/current/{falcon_root}')
else:
hadoop_bin_dir = "/usr/bin"
falcon_webapp_dir = '/var/lib/falcon/webapp'
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 8d8fe00..dd80134 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -75,6 +75,13 @@
</service>
</group>
+ <group name="FALCON" title="Falcon">
+ <service name="Falcon">
+ <component>FALCON_SERVER</component>
+ <component>FALCON_CLIENT</component>
+ </service>
+ </group>
+
<group name="CLIENTS" title="Client Components">
<service name="HDFS">
<component>HDFS_CLIENT</component>
@@ -125,7 +132,6 @@
</task>
</execute-stage>
</group>
-
</order>
@@ -365,5 +371,19 @@
</upgrade>
</component>
</service>
+
+ <service name="FALCON">
+ <component name="FALCON_SERVER">
+ <upgrade>
+ <task xsi:type="restart" />
+ </upgrade>
+ </component>
+ <component name="FALCON_CLIENT">
+ <upgrade>
+ <task xsi:type="restart" />
+ </upgrade>
+ </component>
+ </service>
+
</processing>
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/4af27666/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
index 5451cf8..9b8579a 100644
--- a/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
+++ b/ambari-server/src/test/python/stacks/2.1/FALCON/test_falcon_server.py
@@ -18,12 +18,13 @@ See the License for the specific language governing permissions and
limitations under the License.
'''
+from mock.mock import MagicMock, patch
from stacks.utils.RMFTestCase import *
-
class TestFalconServer(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "FALCON/0.5.0.2.1/package"
STACK_VERSION = "2.1"
+ UPGRADE_STACK_VERSION = "2.2"
def test_start_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
@@ -142,3 +143,33 @@ class TestFalconServer(RMFTestCase):
)
+ @patch("shutil.rmtree", new = MagicMock())
+ @patch("tarfile.open")
+ @patch("os.path.isdir")
+ @patch("os.path.exists")
+ @patch("os.path.isfile")
+ def test_upgrade(self, isfile_mock, exists_mock, isdir_mock,
+ tarfile_open_mock):
+
+ isdir_mock.return_value = True
+ exists_mock.side_effect = [False,False,True]
+ isfile_mock.return_value = True
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/falcon_server.py",
+ classname = "FalconServer", command = "restart", config_file = "falcon-upgrade.json",
+ hdp_stack_version = self.UPGRADE_STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+ self.assertResourceCalled('Execute',
+ '/usr/hdp/current/falcon-server/bin/falcon-stop',
+ path = ['/usr/hdp/current/hadoop-client/bin'], user='falcon')
+
+ self.assertResourceCalled('File', '/var/run/falcon/falcon.pid',
+ action = ['delete'])
+
+ self.assertResourceCalled('Execute', 'hdp-select set falcon-server 2.2.1.0-2135')
+
+ # 4 calls to tarfile.open (2 directories * read + write)
+ self.assertTrue(tarfile_open_mock.called)
+ self.assertEqual(tarfile_open_mock.call_count,4)
+