You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jo...@apache.org on 2015/05/14 19:40:47 UTC
ambari git commit: AMBARI-11048 - Oozie Upgrade Pack For HDP-2.2 to
HDP-2.3 (jonathanhurley)
Repository: ambari
Updated Branches:
refs/heads/trunk 8263f2f1b -> f4a4d0d9a
AMBARI-11048 - Oozie Upgrade Pack For HDP-2.2 to HDP-2.3 (jonathanhurley)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/f4a4d0d9
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/f4a4d0d9
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/f4a4d0d9
Branch: refs/heads/trunk
Commit: f4a4d0d9a224d1d679dbc21286eb57a3ecb129a9
Parents: 8263f2f
Author: Jonathan Hurley <jh...@hortonworks.com>
Authored: Thu May 14 12:18:08 2015 -0400
Committer: Jonathan Hurley <jh...@hortonworks.com>
Committed: Thu May 14 13:40:40 2015 -0400
----------------------------------------------------------------------
.../internal/UpgradeResourceProvider.java | 4 -
.../serveraction/upgrades/ConfigureAction.java | 4 +-
.../package/alerts/alert_check_oozie_server.py | 12 +-
.../4.0.0.2.0/package/scripts/oozie_service.py | 17 ++-
.../OOZIE/5.0.0.2.3/configuration/oozie-env.xml | 142 +++++++++++++++++++
.../stacks/HDP/2.2/upgrades/upgrade-2.3.xml | 52 +++++--
.../stacks/2.0.6/OOZIE/test_oozie_server.py | 4 +
.../python/stacks/2.2/KNOX/test_knox_gateway.py | 2 +
8 files changed, 216 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
index bdb2784..ca72934 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/UpgradeResourceProvider.java
@@ -1041,10 +1041,6 @@ public class UpgradeResourceProvider extends AbstractControllerResourceProvider
detail.append(String.format("/%s to %s", key, value));
}
- if (!transfers.isEmpty()) {
- detail.append(String.format("; transferring %d properties", transfers.size()));
- }
-
itemDetail = detail.toString();
if (null != ct.summary) {
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
index ce03861..645c4bb 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/serveraction/upgrades/ConfigureAction.java
@@ -209,7 +209,9 @@ public class ConfigureAction extends AbstractServerAction {
changedValues = true;
// append standard output
- outputBuffer.append(MessageFormat.format("Copied {0}/{1}\n", configType, key));
+ outputBuffer.append(MessageFormat.format("Copied {0}/{1}\n", configType,
+ transfer.toKey));
+
} else if (StringUtils.isNotBlank(transfer.defaultValue)) {
newValues.put(transfer.toKey, transfer.defaultValue);
changedValues = true;
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py
index c0dc18a..a4a24b7 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/alerts/alert_check_oozie_server.py
@@ -39,6 +39,8 @@ OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}'
SECURITY_ENABLED = '{{cluster-env/security_enabled}}'
OOZIE_PRINCIPAL = '{{oozie-site/oozie.authentication.kerberos.principal}}'
OOZIE_KEYTAB = '{{oozie-site/oozie.authentication.kerberos.keytab}}'
+OOZIE_CONF_DIR = '/usr/hdp/current/oozie-server/conf'
+OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf'
class KerberosPropertiesNotFound(Exception): pass
@@ -107,7 +109,15 @@ def get_check_command(oozie_url, host_name, configurations):
# kinit
Execute(kinit_command, environment=kerberos_env)
- command = format("source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status")
+
+ # oozie configuration directory uses a symlink when > HDP 2.2
+ oozie_config_directory = OOZIE_CONF_DIR_LEGACY
+ if os.path.exists(OOZIE_CONF_DIR):
+ oozie_config_directory = OOZIE_CONF_DIR
+
+ command = "source {0}/oozie-env.sh ; oozie admin -oozie {1} -status".format(
+ oozie_config_directory, oozie_url)
+
return (command, kerberos_env)
def execute(configurations={}, parameters={}, host_name=None):
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
index 34ac6e5..ef721ba 100644
--- a/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
+++ b/ambari-server/src/main/resources/common-services/OOZIE/4.0.0.2.0/package/scripts/oozie_service.py
@@ -44,6 +44,8 @@ def oozie_service(action = 'start', rolling_restart=False):
"""
import params
+ environment={'OOZIE_CONFIG': params.conf_dir}
+
if params.security_enabled:
if params.oozie_principal is None:
oozie_principal_with_host = 'missing_principal'
@@ -88,13 +90,16 @@ def oozie_service(action = 'start', rolling_restart=False):
params.oozie_user)
Execute( cmd2, user = params.oozie_user, not_if = not_if_command,
path = params.execute_path )
-
- Execute( start_cmd, user = params.oozie_user, not_if = no_op_test )
+
+ # start oozie
+ Execute( start_cmd, environment=environment, user = params.oozie_user,
+ not_if = no_op_test )
elif action == 'stop':
stop_cmd = format("cd {oozie_tmp_dir} && {oozie_home}/bin/oozie-stop.sh")
- Execute(stop_cmd, only_if = no_op_test, user = params.oozie_user)
- File(params.pid_file, action = "delete")
-
-
+ # stop oozie
+ Execute(stop_cmd, environment=environment, only_if = no_op_test,
+ user = params.oozie_user)
+
+ File(params.pid_file, action = "delete")
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml
new file mode 100644
index 0000000..ac12d7c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/OOZIE/5.0.0.2.3/configuration/oozie-env.xml
@@ -0,0 +1,142 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>oozie_user</name>
+ <value>oozie</value>
+ <property-type>USER</property-type>
+ <description>Oozie User.</description>
+ </property>
+ <property>
+ <name>oozie_database</name>
+ <value>New Derby Database</value>
+ <description>Oozie Server Database.</description>
+ </property>
+ <property>
+ <name>oozie_derby_database</name>
+ <value>Derby</value>
+ <description>Oozie Derby Database</description>
+ </property>
+ <property>
+ <name>oozie_data_dir</name>
+ <value>/hadoop/oozie/data</value>
+ <description>Data directory in which the Oozie DB exists</description>
+ </property>
+ <property>
+ <name>oozie_log_dir</name>
+ <value>/var/log/oozie</value>
+ <description>Directory for oozie logs</description>
+ </property>
+ <property>
+ <name>oozie_pid_dir</name>
+ <value>/var/run/oozie</value>
+ <description>Directory in which the pid files for oozie reside.</description>
+ </property>
+ <property>
+ <name>oozie_admin_port</name>
+ <value>11001</value>
+ <description>The admin port Oozie server runs.</description>
+ </property>
+ <property>
+ <name>oozie_heapsize</name>
+ <value>2048</value>
+ <description>Oozie heap size.</description>
+ </property>
+ <property>
+ <name>oozie_permsize</name>
+ <value>256</value>
+ <description>Oozie permanent generation size.</description>
+ </property>
+
+ <!-- oozie-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for oozie-env.sh file</description>
+ <value>
+#!/bin/bash
+
+if [ -d "/usr/lib/bigtop-tomcat" ]; then
+ export OOZIE_CONFIG=${OOZIE_CONFIG:-{{conf_dir}}}
+ export CATALINA_BASE=${CATALINA_BASE:-{{oozie_server_dir}}}
+ export CATALINA_TMPDIR=${CATALINA_TMPDIR:-/var/tmp/oozie}
+ export OOZIE_CATALINA_HOME=/usr/lib/bigtop-tomcat
+fi
+
+#Set JAVA HOME
+export JAVA_HOME={{java_home}}
+
+export JRE_HOME=${JAVA_HOME}
+
+# Set Oozie specific environment variables here.
+
+# Settings for the Embedded Tomcat that runs Oozie
+# Java System properties for Oozie should be specified in this variable
+#
+{% if java_version < 8 %}
+export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}} -XX:MaxPermSize={{oozie_permsize}}"
+{% else %}
+export CATALINA_OPTS="$CATALINA_OPTS -Xmx{{oozie_heapsize}}"
+{% endif %}
+# Oozie configuration file to load from Oozie configuration directory
+#
+# export OOZIE_CONFIG_FILE=oozie-site.xml
+
+# Oozie logs directory
+#
+export OOZIE_LOG={{oozie_log_dir}}
+
+# Oozie pid directory
+#
+export CATALINA_PID={{pid_file}}
+
+#Location of the data for oozie
+export OOZIE_DATA={{oozie_data_dir}}
+
+# Oozie Log4J configuration file to load from Oozie configuration directory
+#
+# export OOZIE_LOG4J_FILE=oozie-log4j.properties
+
+# Reload interval of the Log4J configuration file, in seconds
+#
+# export OOZIE_LOG4J_RELOAD=10
+
+# The port Oozie server runs
+#
+export OOZIE_HTTP_PORT={{oozie_server_port}}
+
+# The admin port Oozie server runs
+#
+export OOZIE_ADMIN_PORT={{oozie_server_admin_port}}
+
+# The host name Oozie server runs on
+#
+# export OOZIE_HTTP_HOSTNAME=`hostname -f`
+
+# The base URL for callback URLs to Oozie
+#
+# export OOZIE_BASE_URL="http://${OOZIE_HTTP_HOSTNAME}:${OOZIE_HTTP_PORT}/oozie"
+export JAVA_LIBRARY_PATH={{hadoop_lib_home}}/native/Linux-amd64-64
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
index 64a621e..eb89dbd 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.3.xml
@@ -21,7 +21,6 @@
<target>2.3.*.*</target>
<target-stack>HDP-2.3</target-stack>
<order>
-
<group xsi:type="cluster" name="PRE_CLUSTER" title="Prepare Backups">
<direction>UPGRADE</direction>
<execute-stage service="HDFS" component="NAMENODE" title="Pre Upgrade HDFS">
@@ -63,7 +62,7 @@
<component>ZOOKEEPER_SERVER</component>
</service>
</group>
-
+
<group name="RANGER" title="Ranger">
<skippable>true</skippable>
<service name="RANGER">
@@ -91,9 +90,9 @@
<service name="HBASE">
<component>HBASE_MASTER</component>
</service>
-
+
</group>
-
+
<group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
@@ -107,13 +106,13 @@
<service>AMBARI_METRICS</service>
</exclude>
</group>
-
+
<group name="CORE_SLAVES" title="Core Slaves" xsi:type="colocated">
<service-check>false</service-check>
<service name="HDFS">
<component>DATANODE</component>
</service>
-
+
<service name="HBASE">
<component>HBASE_REGIONSERVER</component>
</service>
@@ -128,7 +127,7 @@
<message>The initial batch of {{components}} hosts have been {{direction.past}}. You are advised to check the hosts and perform cluster/workload-specific tests against your cluster to ensure proper operation before proceeding with {{direction.text}} of the remaining services.</message>
</batch>
</group>
-
+
<group name="SERVICE_CHECK" title="All Service Checks" xsi:type="service-check">
<skippable>true</skippable>
<direction>UPGRADE</direction>
@@ -266,7 +265,6 @@
</service>
</group>
-
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize {{direction.text.proper}}">
<skippable>true</skippable>
<execute-stage title="Check Unhealthy Hosts" id="unhealthy-hosts">
@@ -274,7 +272,7 @@
<message>The following hosts were unhealthy and should be resolved before finalizing can be completed: {{hosts.unhealthy}}</message>
</task>
</execute-stage>
-
+
<execute-stage title="Confirm Finalize">
<direction>UPGRADE</direction>
<task xsi:type="manual">
@@ -577,6 +575,30 @@
<service name="OOZIE">
<component name="OOZIE_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure">
+ <summary>Updating oozie-site to remove redundant configurations</summary>
+ <type>oozie-site</type>
+ <transfer operation="DELETE" delete-key="*" preserve-edits="true">
+ <keep-key>oozie.base.url</keep-key>
+ <keep-key>oozie.services.ext</keep-key>
+ <keep-key>oozie.db.schema.name</keep-key>
+ <keep-key>oozie.service.JPAService.jdbc.username</keep-key>
+ <keep-key>oozie.service.JPAService.jdbc.password</keep-key>
+ <keep-key>oozie.service.JPAService.jdbc.driver</keep-key>
+ <keep-key>oozie.service.JPAService.jdbc.url</keep-key>
+ <keep-key>oozie.service.AuthorizationService.security.enabled</keep-key>
+ <keep-key>oozie.authentication.type</keep-key>
+ <keep-key>oozie.authentication.simple.anonymous.allowed</keep-key>
+ <keep-key>oozie.authentication.kerberos.name.rules</keep-key>
+ <keep-key>oozie.credentials.credentialclasses</keep-key>
+ <keep-key>oozie.service.AuthorizationService.authorization.enabled</keep-key>
+ <keep-key>oozie.service.HadoopAccessorService.hadoop.configurations</keep-key>
+ <keep-key>oozie.service.HadoopAccessorService.kerberos.enabled</keep-key>
+ <keep-key>oozie.service.URIHandlerService.uri.handlers</keep-key>
+ </transfer>
+ </task>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart" />
</upgrade>
@@ -590,6 +612,18 @@
<service name="FALCON">
<component name="FALCON_SERVER">
+ <pre-upgrade>
+ <task xsi:type="configure">
+ <type>oozie-site</type>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-job-submit-instances" to-key="oozie.service.ELService.ext.functions.coord-job-submit-instances"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-create-inst" to-key="oozie.service.ELService.ext.functions.coord-action-create-inst"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-create" to-key="oozie.service.ELService.ext.functions.coord-action-create"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-job-submit-data" to-key="oozie.service.ELService.ext.functions.coord-job-submit-data"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-action-start" to-key="oozie.service.ELService.ext.functions.coord-action-start"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-sla-submit" to-key="oozie.service.ELService.ext.functions.coord-sla-submit"/>
+ <transfer operation="COPY" from-key="oozie.service.ELService.ext.functions.coord-sla-create" to-key="oozie.service.ELService.ext.functions.coord-sla-create"/>
+ </task>
+ </pre-upgrade>
<upgrade>
<task xsi:type="restart" />
</upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
index bb1a037..2bf2af7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/OOZIE/test_oozie_server.py
@@ -255,6 +255,7 @@ class TestOozieServer(RMFTestCase):
path = ['/usr/bin:/usr/bin'],
)
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh',
+ environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
user = 'oozie',
)
@@ -270,6 +271,7 @@ class TestOozieServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh',
+ environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
user = 'oozie',
)
@@ -312,6 +314,7 @@ class TestOozieServer(RMFTestCase):
path = ['/usr/bin:/usr/bin'],
)
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-start.sh',
+ environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
not_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
user = 'oozie',
)
@@ -326,6 +329,7 @@ class TestOozieServer(RMFTestCase):
target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Execute', 'cd /var/tmp/oozie && /usr/lib/oozie/bin/oozie-stop.sh',
+ environment = {'OOZIE_CONFIG': '/etc/oozie/conf'},
only_if = 'ls /var/run/oozie/oozie.pid >/dev/null 2>&1 && ps -p `cat /var/run/oozie/oozie.pid` >/dev/null 2>&1',
user = 'oozie',
)
http://git-wip-us.apache.org/repos/asf/ambari/blob/f4a4d0d9/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index f2e4e50..bab4ed1 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -21,7 +21,9 @@ import json
from resource_management import *
from stacks.utils.RMFTestCase import *
from mock.mock import patch
+from mock.mock import MagicMock
+@patch("platform.linux_distribution", new = MagicMock(return_value="Linux"))
class TestKnoxGateway(RMFTestCase):
COMMON_SERVICES_PACKAGE_DIR = "KNOX/0.5.0.2.2/package"
STACK_VERSION = "2.2"