You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/07/01 21:11:29 UTC
ambari git commit: AMBARI-12228 - Oozie Rolling Upgrade Orchestration
Is Incorrect (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk ce27023f3 -> 1148d12da
AMBARI-12228 - Oozie Rolling Upgrade Orchestration Is Incorrect (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1148d12d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1148d12d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1148d12d
Branch: refs/heads/trunk
Commit: 1148d12da1bc4716a7f50ab37ca0d15130e5fab7
Parents: ce27023
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Wed Jul 1 13:07:18 2015 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Wed Jul 1 15:08:37 2015 -0400
----------------------------------------------------------------------
.../state/stack/upgrade/ExecuteHostType.java | 49 +++
.../server/state/stack/upgrade/ExecuteTask.java | 7 +-
.../state/stack/upgrade/TaskWrapperBuilder.java | 4 +-
.../4.0.0.2.0/package/scripts/oozie_server.py | 15 +-
.../package/scripts/oozie_server_upgrade.py | 440 +++++++++++--------
.../0.9.1.2.1/package/scripts/storm_upgrade.py | 1 -
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 25 ++
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 25 +-
.../stacks/HDP/2.3/upgrades/upgrade-2.3.xml | 25 ++
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 176 ++++----
.../stacks/2.2/configs/oozie-upgrade.json | 3 +-
11 files changed, 471 insertions(+), 299 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteHostType.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteHostType.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteHostType.java
new file mode 100644
index 0000000..b36dca4
--- /dev/null
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteHostType.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ambari.server.state.stack.upgrade;
+
+import javax.xml.bind.annotation.XmlEnum;
+import javax.xml.bind.annotation.XmlEnumValue;
+
+/**
+ * The {@link ExecuteHostType} enum is used to represent where an
+ * {@link ExecuteTask} will run.
+ */
+@XmlEnum
+public enum ExecuteHostType {
+ /**
+ * The term "master" can mean something unique to each service. In terms of
+ * where to run, this means calculate which is the "master" component and run
+ * it on that.
+ */
+ @XmlEnumValue("master")
+ MASTER,
+
+ /**
+ * Run on a single host that satifies the condition of the {@link ExecuteTask}
+ * .
+ */
+ @XmlEnumValue("any")
+ ANY,
+
+ /**
+ * Run on all of the hosts.
+ */
+ @XmlEnumValue("all")
+ ALL;
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteTask.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteTask.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteTask.java
index 59af413..a0afdfb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteTask.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/ExecuteTask.java
@@ -19,11 +19,11 @@ package org.apache.ambari.server.state.stack.upgrade;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import javax.xml.bind.annotation.XmlType;
-import javax.xml.bind.annotation.XmlAttribute;
/**
* Used to represent an execution that should occur on an agent.
@@ -38,10 +38,11 @@ public class ExecuteTask extends Task {
private Task.Type type = Task.Type.EXECUTE;
/**
- * Host to run on, if not specified, run on all. Other values include "master", "any"
+ * The hosts to run the task on. Default to running on
+ * {@link ExecuteHostType#ALL}.
*/
@XmlAttribute
- public String hosts;
+ public ExecuteHostType hosts = ExecuteHostType.ALL;
/**
* Similar to a command, but instead it is a call to invoke the script (using its relative path).
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
index 719078f..9a5ba8c 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/state/stack/upgrade/TaskWrapperBuilder.java
@@ -48,7 +48,7 @@ public class TaskWrapperBuilder {
List<TaskWrapper> collection = new ArrayList<TaskWrapper>();
for (Task t : tasks) {
if (t.getType().equals(Task.Type.EXECUTE)) {
- if (((ExecuteTask) t).hosts != null && ((ExecuteTask) t).hosts.equalsIgnoreCase("master")) {
+ if (((ExecuteTask) t).hosts != null && ((ExecuteTask) t).hosts == ExecuteHostType.MASTER) {
if (hostsType.master != null) {
collection.add(new TaskWrapper(service, component, Collections.singleton(hostsType.master), t));
continue;
@@ -58,7 +58,7 @@ public class TaskWrapperBuilder {
}
}
// Pick a random host.
- if (((ExecuteTask) t).hosts != null && ((ExecuteTask) t).hosts.equalsIgnoreCase("any")) {
+ if (((ExecuteTask) t).hosts != null && ((ExecuteTask) t).hosts == ExecuteHostType.ANY) {
if (hostsType.hosts != null && !hostsType.hosts.isEmpty()) {
collection.add(new TaskWrapper(service, component, Collections.singleton(hostsType.hosts.iterator().next()), t));
continue;
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
index 611046e..c9bdd78 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server.py
@@ -18,11 +18,7 @@ limitations under the License.
"""
-import oozie_server_upgrade
-
from resource_management.core import Logger
-from resource_management.core.resources.system import Execute
-from resource_management.libraries.functions import format
from resource_management.libraries.script import Script
from resource_management.libraries.functions import compare_versions
from resource_management.libraries.functions import conf_select
@@ -33,11 +29,13 @@ from resource_management.libraries.functions.security_commons import cached_kini
from resource_management.libraries.functions.security_commons import get_params_from_filesystem
from resource_management.libraries.functions.security_commons import validate_security_config_properties
from resource_management.libraries.functions.security_commons import FILE_TYPE_XML
+
from ambari_commons import OSConst
from ambari_commons.os_family_impl import OsFamilyImpl
from oozie import oozie
from oozie_service import oozie_service
+from oozie_server_upgrade import OozieUpgrade
from check_oozie_server_status import check_oozie_server_status
@@ -159,14 +157,15 @@ class OozieServerDefault(OozieServer):
Logger.info("Executing Oozie Server Rolling Upgrade pre-restart")
- oozie_server_upgrade.backup_configuration()
+ OozieUpgrade.backup_configuration()
conf_select.select(params.stack_name, "oozie", params.version)
hdp_select.select("oozie-server", params.version)
- oozie_server_upgrade.restore_configuration()
- oozie_server_upgrade.prepare_libext_directory()
- oozie_server_upgrade.upgrade_oozie()
+ OozieUpgrade.restore_configuration()
+ OozieUpgrade.prepare_libext_directory()
+ OozieUpgrade.prepare_warfile()
+
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class OozieServerWindows(OozieServer):
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
index 9d348e1..bd423a2 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_server_upgrade.py
@@ -29,193 +29,269 @@ from resource_management.core.resources.system import Execute
from resource_management.libraries.functions import Direction
from resource_management.libraries.functions import format
from resource_management.libraries.functions import compare_versions
+from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import format_hdp_stack_version
from resource_management.libraries.functions import tar_archive
+from resource_management.libraries.script.script import Script
from resource_management.core.resources import File
from resource_management.core.source import DownloadSource
BACKUP_TEMP_DIR = "oozie-upgrade-backup"
BACKUP_CONF_ARCHIVE = "oozie-conf-backup.tar"
-
-def backup_configuration():
- """
- Backs up the oozie configuration as part of the upgrade process.
- :return:
- """
- Logger.info('Backing up Oozie configuration directory before upgrade...')
- directoryMappings = _get_directory_mappings()
-
- absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
- if not os.path.isdir(absolute_backup_dir):
- os.makedirs(absolute_backup_dir)
-
- for directory in directoryMappings:
- if not os.path.isdir(directory):
- raise Fail("Unable to backup missing directory {0}".format(directory))
-
- archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
- Logger.info('Compressing {0} to {1}'.format(directory, archive))
-
- if os.path.exists(archive):
- os.remove(archive)
-
- # backup the directory, following symlinks instead of including them
- tar_archive.archive_directory_dereference(archive, directory)
-
-
-def restore_configuration():
- """
- Restores the configuration backups to their proper locations after an
- upgrade has completed.
- :return:
- """
- Logger.info('Restoring Oozie configuration directory after upgrade...')
- directoryMappings = _get_directory_mappings()
-
- for directory in directoryMappings:
- archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
- directoryMappings[directory])
-
- if not os.path.isfile(archive):
- raise Fail("Unable to restore missing backup archive {0}".format(archive))
-
- Logger.info('Extracting {0} to {1}'.format(archive, directory))
-
- tarball = None
- try:
- tarball = tarfile.open(archive, "r")
- tarball.extractall(directory)
- finally:
- if tarball:
- tarball.close()
-
- # cleanup
- shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
-
-
-def prepare_libext_directory():
- """
- Creates /usr/hdp/current/oozie/libext-customer and recursively sets
- 777 permissions on it and its parents.
- Also, downloads jdbc driver and provides other staff
- """
- import params
-
- # some versions of HDP don't need the lzo compression libraries
- target_version_needs_compression_libraries = compare_versions(
- format_hdp_stack_version(params.version), '2.2.1.0') >= 0
-
- if not os.path.isdir(params.oozie_libext_customer_dir):
- os.makedirs(params.oozie_libext_customer_dir, 0o777)
-
- # ensure that it's rwx for all
- os.chmod(params.oozie_libext_customer_dir, 0o777)
-
- # get all hadooplzo* JAR files
- # hdp-select set hadoop-client has not run yet, therefore we cannot use
- # /usr/hdp/current/hadoop-client ; we must use params.version directly
- # however, this only works when upgrading beyond 2.2.0.0; don't do this
- # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
- # This can also be called during a Downgrade.
- # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
- # if lzo is enabled.
- if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
- hadoop_lzo_pattern = 'hadoop-lzo*.jar'
- hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
-
- files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
- if not files:
- raise Fail("There are no files at {0} matching {1}".format(
- hadoop_client_new_lib_dir, hadoop_lzo_pattern))
-
- # copy files into libext
- files_copied = False
- for file in files:
- if os.path.isfile(file):
- Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
- shutil.copy2(file, params.oozie_libext_customer_dir)
- files_copied = True
-
- if not files_copied:
- raise Fail("There are no files at {0} matching {1}".format(
- hadoop_client_new_lib_dir, hadoop_lzo_pattern))
-
- # copy ext ZIP to customer dir
- oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
- if not os.path.isfile(oozie_ext_zip_file):
- raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
-
- Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir))
- shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
-
- # Redownload jdbc driver to a new current location
- if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
- params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
- params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
- File(params.downloaded_custom_connector,
- content = DownloadSource(params.driver_curl_source),
- )
-
- Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
- #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
- path=["/bin", "/usr/bin/"],
- sudo = True)
-
- File ( params.target,
- owner = params.oozie_user,
- group = params.user_group
- )
-
-
-def upgrade_oozie():
- """
- Performs the upgrade of the oozie WAR file and database.
- :return:
- """
- import params
-
- # get the kerberos token if necessary to execute commands as oozie
- if params.security_enabled:
- oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname)
- command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}")
- Execute(command, user=params.oozie_user)
-
-
- params.HdfsResource(format("{oozie_hdfs_user_dir}/share"),
- action = "create_on_execute",
- type = "directory",
- owner = "oozie",
- group = "hadoop",
- mode = 0755,
- recursive_chmod = True
- )
- params.HdfsResource(None, action = "execute")
-
- # upgrade oozie DB
- command = format("{oozie_home}/bin/ooziedb.sh upgrade -run")
- Execute(command, user=params.oozie_user)
-
- # prepare the oozie WAR
- command = format("{oozie_setup_sh} prepare-war {oozie_secure} -d {oozie_libext_customer_dir}")
- return_code, oozie_output = shell.call(command)
-
- if return_code != 0 or "New Oozie WAR file with added" not in oozie_output:
- message = "Unexpected Oozie WAR preparation output {0}".format(oozie_output)
- Logger.error(message)
- raise Fail(message)
-
- # install new sharelib to HDFS
- command = format("{oozie_setup_sh} sharelib create -fs {fs_root}")
- Execute(command, user=params.oozie_user)
-
-
-def _get_directory_mappings():
- """
- Gets a dictionary of directory to archive name that represents the
- directories that need to be backed up and their output tarball archive targets
- :return: the dictionary of directory to tarball mappings
- """
- import params
-
- return { params.conf_dir + "/" : BACKUP_CONF_ARCHIVE } # the trailing "/" is important here so as to not include the "conf" folder itself
+class OozieUpgrade(Script):
+
+ @staticmethod
+ def backup_configuration():
+ """
+ Backs up the oozie configuration as part of the upgrade process.
+ :return:
+ """
+ Logger.info('Backing up Oozie configuration directory before upgrade...')
+ directoryMappings = OozieUpgrade._get_directory_mappings()
+
+ absolute_backup_dir = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR)
+ if not os.path.isdir(absolute_backup_dir):
+ os.makedirs(absolute_backup_dir)
+
+ for directory in directoryMappings:
+ if not os.path.isdir(directory):
+ raise Fail("Unable to backup missing directory {0}".format(directory))
+
+ archive = os.path.join(absolute_backup_dir, directoryMappings[directory])
+ Logger.info('Compressing {0} to {1}'.format(directory, archive))
+
+ if os.path.exists(archive):
+ os.remove(archive)
+
+ # backup the directory, following symlinks instead of including them
+ tar_archive.archive_directory_dereference(archive, directory)
+
+
+ @staticmethod
+ def restore_configuration():
+ """
+ Restores the configuration backups to their proper locations after an
+ upgrade has completed.
+ :return:
+ """
+ Logger.info('Restoring Oozie configuration directory after upgrade...')
+ directoryMappings = OozieUpgrade._get_directory_mappings()
+
+ for directory in directoryMappings:
+ archive = os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR,
+ directoryMappings[directory])
+
+ if not os.path.isfile(archive):
+ raise Fail("Unable to restore missing backup archive {0}".format(archive))
+
+ Logger.info('Extracting {0} to {1}'.format(archive, directory))
+
+ tarball = None
+ try:
+ tarball = tarfile.open(archive, "r")
+ tarball.extractall(directory)
+ finally:
+ if tarball:
+ tarball.close()
+
+ # cleanup
+ shutil.rmtree(os.path.join(tempfile.gettempdir(), BACKUP_TEMP_DIR))
+
+
+ @staticmethod
+ def prepare_libext_directory():
+ """
+ Creates /usr/hdp/current/oozie/libext-customer and recursively sets
+ 777 permissions on it and its parents.
+ Also, downloads jdbc driver and provides other staff
+ """
+ import params
+
+ # some versions of HDP don't need the lzo compression libraries
+ target_version_needs_compression_libraries = compare_versions(
+ format_hdp_stack_version(params.version), '2.2.1.0') >= 0
+
+ if not os.path.isdir(params.oozie_libext_customer_dir):
+ os.makedirs(params.oozie_libext_customer_dir, 0o777)
+
+ # ensure that it's rwx for all
+ os.chmod(params.oozie_libext_customer_dir, 0o777)
+
+ # get all hadooplzo* JAR files
+ # hdp-select set hadoop-client has not run yet, therefore we cannot use
+ # /usr/hdp/current/hadoop-client ; we must use params.version directly
+ # however, this only works when upgrading beyond 2.2.0.0; don't do this
+ # for downgrade to 2.2.0.0 since hadoop-lzo will not be present
+ # This can also be called during a Downgrade.
+ # When a version is Intalled, it is responsible for downloading the hadoop-lzo packages
+ # if lzo is enabled.
+ if params.lzo_enabled and (params.upgrade_direction == Direction.UPGRADE or target_version_needs_compression_libraries):
+ hadoop_lzo_pattern = 'hadoop-lzo*.jar'
+ hadoop_client_new_lib_dir = format("/usr/hdp/{version}/hadoop/lib")
+
+ files = glob.iglob(os.path.join(hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+ if not files:
+ raise Fail("There are no files at {0} matching {1}".format(
+ hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+
+ # copy files into libext
+ files_copied = False
+ for file in files:
+ if os.path.isfile(file):
+ Logger.info("Copying {0} to {1}".format(str(file), params.oozie_libext_customer_dir))
+ shutil.copy2(file, params.oozie_libext_customer_dir)
+ files_copied = True
+
+ if not files_copied:
+ raise Fail("There are no files at {0} matching {1}".format(
+ hadoop_client_new_lib_dir, hadoop_lzo_pattern))
+
+ # copy ext ZIP to customer dir
+ oozie_ext_zip_file = '/usr/share/HDP-oozie/ext-2.2.zip'
+ if not os.path.isfile(oozie_ext_zip_file):
+ raise Fail("Unable to copy {0} because it does not exist".format(oozie_ext_zip_file))
+
+ Logger.info("Copying {0} to {1}".format(oozie_ext_zip_file, params.oozie_libext_customer_dir))
+ shutil.copy2(oozie_ext_zip_file, params.oozie_libext_customer_dir)
+
+ # Redownload jdbc driver to a new current location
+ if params.jdbc_driver_name=="com.mysql.jdbc.Driver" or \
+ params.jdbc_driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver" or \
+ params.jdbc_driver_name=="oracle.jdbc.driver.OracleDriver":
+
+ File(params.downloaded_custom_connector,
+ content = DownloadSource(params.driver_curl_source))
+
+ Execute(('cp', '--remove-destination', params.downloaded_custom_connector, params.target),
+ #creates=params.target, TODO: uncomment after ranger_hive_plugin will not provide jdbc
+ path=["/bin", "/usr/bin/"], sudo = True)
+
+ File(params.target, owner = params.oozie_user, group = params.user_group)
+
+
+ @staticmethod
+ def prepare_warfile():
+ """
+ Invokes the 'prepare-war' command in Oozie in order to create the WAR.
+ This method will also perform a kinit if necessary.
+ :return:
+ """
+ import params
+
+ # get the kerberos token if necessary to execute commands as oozie
+ if params.security_enabled:
+ oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname)
+ command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}")
+ Execute(command, user=params.oozie_user, logoutput=True)
+
+ # prepare the oozie WAR
+ command = format("{oozie_setup_sh} prepare-war {oozie_secure} -d {oozie_libext_customer_dir}")
+ return_code, oozie_output = shell.call(command, user=params.oozie_user,
+ logoutput=False, quiet=False)
+
+ # set it to "" in to prevent a possible iteration issue
+ if oozie_output is None:
+ oozie_output = ""
+
+ if return_code != 0 or "New Oozie WAR file with added".lower() not in oozie_output.lower():
+ message = "Unexpected Oozie WAR preparation output {0}".format(oozie_output)
+ Logger.error(message)
+ raise Fail(message)
+
+
+ def upgrade_oozie_database_and_sharelib(self, env):
+ """
+ Performs the creation and upload of the sharelib and the upgrade of the
+ database. This method will also perform a kinit if necessary.
+ It is run before the upgrade of oozie begins exactly once as part of the
+ upgrade orchestration.
+
+ Since this runs before the upgrade has occurred, it should not use any
+ "current" directories since they will still be pointing to the older
+ version of Oozie. Instead, it should use versioned directories to ensure
+ that the commands running are from the oozie version about to be upgraded to.
+ :return:
+ """
+ import params
+ env.set_params(params)
+
+ # get the kerberos token if necessary to execute commands as oozie
+ if params.security_enabled:
+ oozie_principal_with_host = params.oozie_principal.replace("_HOST", params.hostname)
+ command = format("{kinit_path_local} -kt {oozie_keytab} {oozie_principal_with_host}")
+ Execute(command, user=params.oozie_user, logoutput=True)
+
+ upgrade_stack = hdp_select._get_upgrade_stack()
+ if upgrade_stack is None or len(upgrade_stack) < 2 or upgrade_stack[1] is None:
+ raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
+
+ stack_version = upgrade_stack[1]
+
+ # upgrade oozie DB
+ Logger.info('Upgrading the Oozie database...')
+ database_upgrade_command = "/usr/hdp/{0}/oozie/bin/ooziedb.sh upgrade -run".format(stack_version)
+ Execute(database_upgrade_command, user=params.oozie_user, logoutput=True)
+
+ # install new sharelib to HDFS
+ self.create_sharelib(env)
+
+
+ def create_sharelib(self, env):
+ """
+ Performs the creation and upload of the sharelib.
+ This method will also perform a kinit if necessary.
+ It is run before the upgrade of oozie begins exactly once as part of the
+ upgrade orchestration.
+
+ Since this runs before the upgrade has occurred, it should not use any
+ "current" directories since they will still be pointing to the older
+ version of Oozie. Instead, it should use versioned directories to ensure
+ that the commands running are from the oozie version about to be upgraded to.
+ :param env:
+ :return:
+ """
+ import params
+ env.set_params(params)
+
+ Logger.info('Creating a new sharelib and uploading it to HDFS...')
+
+ # ensure the oozie directory exists for the sharelib
+ params.HdfsResource(format("{oozie_hdfs_user_dir}/share"),
+ action = "create_on_execute",
+ type = "directory",
+ owner = "oozie",
+ group = "hadoop",
+ mode = 0755,
+ recursive_chmod = True)
+
+ params.HdfsResource(None, action = "execute")
+
+ upgrade_stack = hdp_select._get_upgrade_stack()
+ if upgrade_stack is None or upgrade_stack[1] is None:
+ raise Fail("Unable to determine the stack that is being upgraded to or downgraded to.")
+
+ stack_version = upgrade_stack[1]
+
+ # install new sharelib to HDFS
+ sharelib_command = "/usr/hdp/{0}/oozie/bin/oozie-setup.sh sharelib create -fs {1}".format(
+ stack_version, params.fs_root)
+
+ Execute(sharelib_command, user=params.oozie_user, logoutput=True)
+
+
+ @staticmethod
+ def _get_directory_mappings():
+ """
+ Gets a dictionary of directory to archive name that represents the
+ directories that need to be backed up and their output tarball archive targets
+ :return: the dictionary of directory to tarball mappings
+ """
+ import params
+
+ # the trailing "/" is important here so as to not include the "conf" folder itself
+ return { params.conf_dir + "/" : BACKUP_CONF_ARCHIVE }
+
+
+if __name__ == "__main__":
+ OozieUpgrade().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py
index 4b455c0..5d01dc6 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm_upgrade.py
@@ -26,7 +26,6 @@ from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import File
from resource_management.core.resources.system import Execute
from resource_management.libraries.script.script import Script
-from resource_management.libraries.functions import format
from resource_management.libraries.functions.default import default
class StormUpgrade(Script):
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 1760936..95834fd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -537,10 +537,35 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>upgrade_oozie_database_and_sharelib</function>
+ </task>
+ </pre-upgrade>
+
+ <pre-downgrade>
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>create_sharelib</function>
+ </task>
+ </pre-downgrade>
+
<upgrade>
<task xsi:type="restart" />
</upgrade>
</component>
+
<component name="OOZIE_CLIENT">
<upgrade>
<task xsi:type="restart" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 56220d7..c5fb156 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -999,7 +999,6 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
- <pre-downgrade /> <!-- no-op to prevent config changes on downgrade -->
<pre-upgrade>
<task xsi:type="configure">
<summary>Updating oozie-site to remove redundant configurations</summary>
@@ -1031,11 +1030,35 @@
</transfer>
<set key="oozie.credentials.credentialclasses" value="hcat=org.apache.oozie.action.hadoop.HCatCredentials,hive2=org.apache.oozie.action.hadoop.Hive2Credentials" />
</task>
+
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>upgrade_oozie_database_and_sharelib</function>
+ </task>
</pre-upgrade>
+
+ <pre-downgrade>
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>create_sharelib</function>
+ </task>
+ </pre-downgrade>
+
<upgrade>
<task xsi:type="restart" />
</upgrade>
</component>
+
<component name="OOZIE_CLIENT">
<upgrade>
<task xsi:type="restart" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
index f489bb0..938d16d 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/upgrades/upgrade-2.3.xml
@@ -599,10 +599,35 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Upgrading the database and creating a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>upgrade_oozie_database_and_sharelib</function>
+ </task>
+ </pre-upgrade>
+
+ <pre-downgrade>
+ <task xsi:type="execute" hosts="all" summary="Shut down all Oozie servers">
+ <script>scripts/oozie_server.py</script>
+ <function>stop</function>
+ </task>
+
+ <task xsi:type="execute" hosts="any" summary="Create a new sharelib">
+ <script>scripts/oozie_server_upgrade.py</script>
+ <function>create_sharelib</function>
+ </task>
+ </pre-downgrade>
+
<upgrade>
<task xsi:type="restart" />
</upgrade>
</component>
+
<component name="OOZIE_CLIENT">
<upgrade>
<task xsi:type="restart" />
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index b92f6fa..5b7d199 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -910,8 +910,14 @@ class TestOozieServer(RMFTestCase):
def test_upgrade(self, glob_mock, chmod_mock, remove_mock,
isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+ def exists_mock_side_effect(path):
+ if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar':
+ return True
+
+ return False
+
+ exists_mock.side_effect = exists_mock_side_effect
isdir_mock.return_value = True
- exists_mock.side_effect = [False,False,True]
isfile_mock.return_value = True
glob_mock.return_value = ["/usr/hdp/2.2.1.0-2187/hadoop/lib/hadoop-lzo-0.6.0.2.2.1.0-2187.jar"]
@@ -950,38 +956,6 @@ class TestOozieServer(RMFTestCase):
glob_mock.assert_called_with('/usr/hdp/2.2.1.0-2135/hadoop/lib/hadoop-lzo*.jar')
self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.1.0-2135'), sudo=True,)
- self.assertResourceCalled('HdfsResource', '/user/oozie/share',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- user = 'hdfs',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- recursive_chmod = True,
- owner = 'oozie',
- group = 'hadoop',
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- type = 'directory',
- action = ['create_on_execute'],
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', None,
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- action = ['execute'],
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- )
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/ooziedb.sh upgrade -run', user='oozie')
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie')
-
self.assertNoMoreResources()
@patch("tarfile.open")
@@ -996,18 +970,25 @@ class TestOozieServer(RMFTestCase):
def test_upgrade_23(self, glob_mock, chmod_mock, remove_mock,
isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
+ def exists_mock_side_effect(path):
+ if path == '/tmp/oozie-upgrade-backup/oozie-conf-backup.tar':
+ return True
+
+ return False
+
isdir_mock.return_value = True
- exists_mock.side_effect = [False,False,True]
+ exists_mock.side_effect = exists_mock_side_effect
isfile_mock.return_value = True
glob_mock.return_value = ["/usr/hdp/2.2.1.0-2187/hadoop/lib/hadoop-lzo-0.6.0.2.2.1.0-2187.jar"]
prepare_war_stdout = """INFO: Adding extension: libext/mysql-connector-java.jar
New Oozie WAR file with added 'JARs' at /var/lib/oozie/oozie-server/webapps/oozie.war"""
-
config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/oozie-upgrade.json"
+
with open(config_file, "r") as f:
json_content = json.load(f)
+
version = '2.3.0.0-1234'
json_content['commandParams']['version'] = version
@@ -1037,40 +1018,6 @@ class TestOozieServer(RMFTestCase):
glob_mock.assert_called_with('/usr/hdp/2.3.0.0-1234/hadoop/lib/hadoop-lzo*.jar')
self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.3.0.0-1234'), sudo=True)
- self.assertResourceCalled('HdfsResource', '/user/oozie/share',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- user = 'hdfs',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- recursive_chmod = True,
- owner = 'oozie',
- group = 'hadoop',
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- type = 'directory',
- action = ['create_on_execute'],
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', None,
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/current/hadoop-client/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- action = ['execute'],
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- )
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/ooziedb.sh upgrade -run',
- user = 'oozie',
- )
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie')
-
self.assertNoMoreResources()
self.assertEquals(3, mocks_dict['call'].call_count)
@@ -1119,37 +1066,7 @@ class TestOozieServer(RMFTestCase):
isfile_mock.assert_called_with('/usr/share/HDP-oozie/ext-2.2.zip')
self.assertResourceCalled('Execute', ('hdp-select', 'set', 'oozie-server', '2.2.0.0-0000'), sudo=True)
- self.assertResourceCalled('HdfsResource', '/user/oozie/share',
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/2.2.0.0-0000/hadoop/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- user = 'hdfs',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- recursive_chmod = True,
- owner = 'oozie',
- group = 'hadoop',
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- type = 'directory',
- action = ['create_on_execute'],
- mode = 0755,
- )
- self.assertResourceCalled('HdfsResource', None,
- security_enabled = False,
- hadoop_bin_dir = '/usr/hdp/2.2.0.0-0000/hadoop/bin',
- keytab = UnknownConfigurationMock(),
- default_fs = 'hdfs://c6401.ambari.apache.org:8020',
- hdfs_site = UnknownConfigurationMock(),
- kinit_path_local = '/usr/bin/kinit',
- principal_name = UnknownConfigurationMock(),
- user = 'hdfs',
- action = ['execute'],
- hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
- )
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/ooziedb.sh upgrade -run', user='oozie')
- self.assertResourceCalled('Execute', '/usr/hdp/current/oozie-server/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020', user='oozie')
+ self.assertNoMoreResources()
@patch("tarfile.open")
@@ -1165,7 +1082,7 @@ class TestOozieServer(RMFTestCase):
isfile_mock, exists_mock, isdir_mock, tarfile_open_mock):
isdir_mock.return_value = True
- exists_mock.side_effect = [False,False,True]
+ exists_mock.return_value = False
isfile_mock.return_value = True
try:
@@ -1178,3 +1095,60 @@ class TestOozieServer(RMFTestCase):
except Fail,f:
pass
+
+ def test_upgrade_database_sharelib(self):
+ """
+ Tests that the upgrade script runs the proper commands before the
+ actual upgrade begins.
+ :return:
+ """
+ config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/oozie-upgrade.json"
+
+ with open(config_file, "r") as f:
+ json_content = json.load(f)
+
+ version = '2.3.0.0-1234'
+ json_content['commandParams']['version'] = version
+ json_content['hostLevelParams']['stack_name'] = "HDP"
+
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/oozie_server_upgrade.py",
+ classname = "OozieUpgrade", command = "upgrade_oozie_database_and_sharelib",
+ config_dict = json_content,
+ hdp_stack_version = self.UPGRADE_STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES )
+
+ self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/ooziedb.sh upgrade -run',
+ user = 'oozie', logoutput = True )
+
+ self.assertResourceCalled('HdfsResource', '/user/oozie/share',
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ user = 'hdfs',
+ hdfs_site = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ recursive_chmod = True,
+ owner = 'oozie',
+ group = 'hadoop',
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf',
+ type = 'directory',
+ action = ['create_on_execute'],
+ mode = 0755 )
+
+ self.assertResourceCalled('HdfsResource', None,
+ security_enabled = False,
+ hadoop_bin_dir = '/usr/hdp/2.3.0.0-1234/hadoop/bin',
+ keytab = UnknownConfigurationMock(),
+ default_fs = 'hdfs://c6401.ambari.apache.org:8020',
+ hdfs_site = UnknownConfigurationMock(),
+ kinit_path_local = '/usr/bin/kinit',
+ principal_name = UnknownConfigurationMock(),
+ user = 'hdfs',
+ action = ['execute'],
+ hadoop_conf_dir = '/usr/hdp/current/hadoop-client/conf' )
+
+ self.assertResourceCalled('Execute', '/usr/hdp/2.3.0.0-1234/oozie/bin/oozie-setup.sh sharelib create -fs hdfs://c6401.ambari.apache.org:8020',
+ user='oozie', logoutput = True)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/1148d12d/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
index f0798c8..1f98793 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/oozie-upgrade.json
@@ -21,7 +21,8 @@
"script": "scripts/oozie_server.py",
"hooks_folder": "HDP/2.0.6/hooks",
"restart_type": "rolling_upgrade",
- "version": "2.2.1.0-2135",
+ "version": "2.2.1.0-2135",
+ "upgrade_direction": "upgrade",
"command_timeout": "1200",
"script_type": "PYTHON"
},