You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by nc...@apache.org on 2015/01/09 02:17:20 UTC
ambari git commit: AMBARI-9051. Rolling Upgrade: fix HBase tasks that
changed definition (ncole)
Repository: ambari
Updated Branches:
refs/heads/trunk 5350550c5 -> 942706041
AMBARI-9051. Rolling Upgrade: fix HBase tasks that changed definition (ncole)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/94270604
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/94270604
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/94270604
Branch: refs/heads/trunk
Commit: 942706041919a82d4c2752969a23e4cb3c7f97d9
Parents: 5350550
Author: Nate Cole <nc...@hortonworks.com>
Authored: Thu Jan 8 15:58:10 2015 -0500
Committer: Nate Cole <nc...@hortonworks.com>
Committed: Thu Jan 8 20:16:46 2015 -0500
----------------------------------------------------------------------
.../0.96.0.2.0/package/scripts/hbase_upgrade.py | 37 ++++
.../stacks/HDP/2.2/upgrades/upgrade-2.2.xml | 5 +-
.../stacks/2.0.6/HBASE/test_hbase_master.py | 15 +-
.../stacks/2.0.6/configs/hbase-preupgrade.json | 194 +++++++++++++++++++
4 files changed, 249 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
new file mode 100644
index 0000000..610f527
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HBASE/0.96.0.2.0/package/scripts/hbase_upgrade.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management.libraries.script import Script
+from resource_management.core.resources.system import Execute
+
+class HbaseMasterUpgrade(Script):
+
+ def snapshot(self, env):
+ import params
+
+ snap_cmd = "echo 'snapshot_all' | {0} shell".format(params.hbase_cmd)
+
+ exec_cmd = "{0} {1}".format(params.kinit_cmd, snap_cmd)
+
+ Execute(exec_cmd, user=params.hbase_user)
+
+if __name__ == "__main__":
+ HbaseMasterUpgrade().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
index 4549da0..ed6b50c 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/upgrades/upgrade-2.2.xml
@@ -85,6 +85,7 @@
</group>
<group name="KAFKA" title="Kafka">
+ <skippable>true</skippable>
<service name="KAFKA">
<component>KAFKA_BROKER</component>
</service>
@@ -122,6 +123,7 @@
</group>
<group xsi:type="cluster" name="POST_CLUSTER" title="Finalize Upgrade">
+ <skippable>true</skippable>
<execute-stage title="Confirm Finalize">
<task xsi:type="manual">
<message>Please confirm you are ready to finalize</message>
@@ -256,7 +258,8 @@
<component name="HBASE_MASTER">
<pre-upgrade>
<task xsi:type="execute" hosts="master">
- <command>su - {{hbase-env/hbase_user}} -c 'echo "snapshot_all" | hbase shell'</command>
+ <script>scripts/hbase_upgrade.py</script>
+ <function>snapshot</function>
</task>
</pre-upgrade>
<upgrade>
http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
index 0d04acf..c350a27 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HBASE/test_hbase_master.py
@@ -641,4 +641,17 @@ class TestHBaseMaster(RMFTestCase):
hdp_stack_version = self.STACK_VERSION,
target = RMFTestCase.TARGET_COMMON_SERVICES
)
- put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
\ No newline at end of file
+ put_structured_out_mock.assert_called_with({"securityState": "UNSECURED"})
+
+ def test_upgrade_backup(self):
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/hbase_upgrade.py",
+ classname = "HbaseMasterUpgrade",
+ command = "snapshot",
+ config_file="hbase-preupgrade.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES)
+
+ self.assertResourceCalled('Execute', " echo 'snapshot_all' | /usr/hdp/current/hbase-client/bin/hbase shell",
+ user = 'hbase')
+
+ self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/94270604/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
new file mode 100644
index 0000000..ee46527
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/hbase-preupgrade.json
@@ -0,0 +1,194 @@
+{
+ "configuration_attributes": {},
+ "roleCommand": "ACTIONEXECUTE",
+ "kerberosCommandParams": [],
+ "clusterName": "c1",
+ "componentName": "",
+ "hostname": "c6401.ambari.apache.org",
+ "hostLevelParams": {
+ "jdk_location": "http://10.0.0.15:8080/resources/",
+ "ambari_db_rca_password": "mapred",
+ "java_home": "/usr/jdk64/jdk1.7.0_45",
+ "ambari_db_rca_url": "jdbc:postgresql://10.0.0.15/ambarirca",
+ "jce_name": "UnlimitedJCEPolicyJDK7.zip",
+ "oracle_jdbc_url": "http://10.0.0.15:8080/resources//ojdbc6.jar",
+ "stack_version": "2.2",
+ "stack_name": "HDP",
+ "db_name": "ambari",
+ "ambari_db_rca_driver": "org.postgresql.Driver",
+ "jdk_name": "jdk-7u45-linux-x64.tar.gz",
+ "ambari_db_rca_username": "mapred",
+ "db_driver_filename": "mysql-connector-java.jar",
+ "agentCacheDir": "/var/lib/ambari-agent/cache",
+ "mysql_jdbc_url": "http://10.0.0.15:8080/resources//mysql-connector-java.jar"
+ },
+ "commandType": "EXECUTION_COMMAND",
+ "roleParams": {
+ "tasks1": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"command\":\"su - hbase -c \\u0027echo \\\"snapshot_all\\\" | hbase shell\\u0027\"}]",
+ "tasks": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"script\":\"scripts/hbase_upgrade.py\",\"function\": \"snapshot\"}]",
+ "service_package_folder": "common-services/HBASE/0.96.0.2.0/package",
+ "version": "2.2.0.0-2041",
+ "hooks_folder": "HDP/2.0.6/hooks"
+ },
+ "serviceName": "",
+ "role": "ru_execute_tasks",
+ "forceRefreshConfigTags": [],
+ "taskId": 402,
+ "public_hostname": "c6401.ambari.apache.org",
+ "configurations": {
+ "hbase-site": {
+ "hbase.master.info.bindAddress": "0.0.0.0",
+ "hbase.client.keyvalue.maxsize": "10485760",
+ "hbase.hstore.compactionThreshold": "3",
+ "hbase.hregion.majorcompaction.jitter": "0.50",
+ "hbase.rootdir": "hdfs://c6401.ambari.apache.org:8020/apps/hbase/data",
+ "hbase.regionserver.handler.count": "60",
+ "hbase.hregion.majorcompaction": "604800000",
+ "hbase.hregion.memstore.block.multiplier": "4",
+ "hbase.hregion.memstore.flush.size": "134217728",
+ "hbase.superuser": "hbase",
+ "hbase.regionserver.global.memstore.lowerLimit": "0.38",
+ "hbase.zookeeper.property.clientPort": "2181",
+ "hbase.regionserver.global.memstore.upperLimit": "0.4",
+ "zookeeper.session.timeout": "30000",
+ "hbase.client.scanner.caching": "100",
+ "hbase.tmp.dir": "/hadoop/hbase",
+ "hfile.block.cache.size": "0.40",
+ "hbase.hregion.max.filesize": "10737418240",
+ "hbase.security.authentication": "simple",
+ "hbase.defaults.for.version.skip": "true",
+ "hbase.master.info.port": "60010",
+ "hbase.zookeeper.quorum": "c6403.ambari.apache.org,c6401.ambari.apache.org,c6402.ambari.apache.org",
+ "hbase.regionserver.info.port": "60030",
+ "zookeeper.znode.parent": "/hbase-unsecure",
+ "hbase.zookeeper.useMulti": "true",
+ "hbase.hstore.blockingStoreFiles": "10",
+ "hbase.master.port": "60000",
+ "hbase.security.authorization": "false",
+ "hbase.local.dir": "${hbase.tmp.dir}/local",
+ "hbase.cluster.distributed": "true",
+ "hbase.hregion.memstore.mslab.enabled": "true",
+ "dfs.domain.socket.path": "/var/lib/hadoop-hdfs/dn_socket",
+ "hbase.coprocessor.region.classes": "",
+ "hbase.coprocessor.master.classes": ""
+ },
+ "hbase-env": {
+ "hbase_pid_dir": "/var/run/hbase",
+ "hbase_regionserver_xmn_max": "512",
+ "hbase_regionserver_xmn_ratio": "0.2",
+ "hbase_user": "hbase",
+ "hbase_master_heapsize": "1024m",
+ "content": "\n# Set environment variables here.\n\n# The java implementation to use. Java 1.6 required.\nexport JAVA_HOME={{java64_home}}\n\n# HBase Configuration directory\nexport HBASE_CONF_DIR=${HBASE_CONF_DIR:-{{hbase_conf_dir}}}\n\n# Extra Java CLASSPATH elements. Optional.\nexport HBASE_CLASSPATH=${HBASE_CLASSPATH}\n\n\n# The maximum amount of heap to use, in MB. Default is 1000.\n# export HBASE_HEAPSIZE=1000\n\n# Extra Java runtime options.\n# Below are what we set by default. May only work with SUN JVM.\n# For more on why as well as other possible settings,\n# see http://wiki.apache.org/hadoop/PerformanceTuning\nexport SERVER_GC_OPTS=\"-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:{{log_dir}}/gc.log-`date +'%Y%m%d%H%M'`\"\n# Uncomment below to enable java garbage collection logging.\n# export HBASE_OPTS=\"$HBASE_OPTS -verbose:gc -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:$HBASE_HOME/logs/gc-hbase.log\"\n\n# Uncomment and adjust to enable
JMX exporting\n# See jmxremote.password and jmxremote.access in $JRE_HOME/lib/management to configure remote password access.\n# More details at: http://java.sun.com/javase/6/docs/technotes/guides/management/agent.html\n#\n# export HBASE_JMX_BASE=\"-Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false\"\n# If you want to configure BucketCache, specify '-XX: MaxDirectMemorySize=' with proper direct memory size\n# export HBASE_THRIFT_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10103\"\n# export HBASE_ZOOKEEPER_OPTS=\"$HBASE_JMX_BASE -Dcom.sun.management.jmxremote.port=10104\"\n\n# File naming hosts on which HRegionServers will run. $HBASE_HOME/conf/regionservers by default.\nexport HBASE_REGIONSERVERS=${HBASE_CONF_DIR}/regionservers\n\n# Extra ssh options. Empty by default.\n# export HBASE_SSH_OPTS=\"-o ConnectTimeout=1 -o SendEnv=HBASE_CONF_DIR\"\n\n# Where log files are stored. $HBASE_HOME/logs by default.\nexport HBASE_LOG_DIR={{l
og_dir}}\n\n# A string representing this instance of hbase. $USER by default.\n# export HBASE_IDENT_STRING=$USER\n\n# The scheduling priority for daemon processes. See 'man nice'.\n# export HBASE_NICENESS=10\n\n# The directory where pid files are stored. /tmp by default.\nexport HBASE_PID_DIR={{pid_dir}}\n\n# Seconds to sleep between slave commands. Unset by default. This\n# can be useful in large clusters, where, e.g., slave rsyncs can\n# otherwise arrive faster than the master can service them.\n# export HBASE_SLAVE_SLEEP=0.1\n\n# Tell HBase whether it should manage it's own instance of Zookeeper or not.\nexport HBASE_MANAGES_ZK=false\n\n{% if security_enabled %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log -Djava.security.auth.login.config={{client_jaas_config_file}}\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}} -Djava.security.auth.login.config={{master_jaas_config_file}}\"\nexport HBASE_REGIONSERV
ER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}} -Djava.security.auth.login.config={{regionserver_jaas_config_file}}\"\n{% else %}\nexport HBASE_OPTS=\"$HBASE_OPTS -XX:+UseConcMarkSweepGC -XX:ErrorFile={{log_dir}}/hs_err_pid%p.log\"\nexport HBASE_MASTER_OPTS=\"$HBASE_MASTER_OPTS -Xmx{{master_heapsize}}\"\nexport HBASE_REGIONSERVER_OPTS=\"$HBASE_REGIONSERVER_OPTS -Xmn{{regionserver_xmn_size}} -XX:CMSInitiatingOccupancyFraction=70 -Xms{{regionserver_heapsize}} -Xmx{{regionserver_heapsize}}\"\n{% endif %}",
+ "hbase_regionserver_heapsize": "1024m",
+ "hbase_log_dir": "/var/log/hbase"
+ },
+ "cluster-env": {
+ "security_enabled": "false",
+ "hive_tar_source": "/usr/hdp/current/hive-client/hive.tar.gz",
+ "hadoop-streaming_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
+ "pig_tar_source": "/usr/hdp/current/pig-client/pig.tar.gz",
+ "tez_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/tez/",
+ "smokeuser": "ambari-qa",
+ "ignore_groupsusers_create": "false",
+ "hadoop-streaming_tar_source": "/usr/hdp/current/hadoop-mapreduce-client/hadoop-streaming.jar",
+ "kerberos_domain": "EXAMPLE.COM",
+ "sqoop_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/sqoop/",
+ "hive_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/hive/",
+ "mapreduce_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/mapreduce/",
+ "tez_tar_source": "/usr/hdp/current/tez-client/lib/tez.tar.gz",
+ "pig_tar_destination_folder": "hdfs:///hdp/apps/{{ hdp_stack_version }}/pig/",
+ "user_group": "hadoop",
+ "mapreduce_tar_source": "/usr/hdp/current/hadoop-client/mapreduce.tar.gz",
+ "sqoop_tar_source": "/usr/hdp/current/sqoop-client/sqoop.tar.gz",
+ "hadoop.user.password": "",
+ "hadoop.user.name": "hadoop"
+ }
+ },
+ "commandParams": {
+ "tasks": "[{\"type\":\"EXECUTE\",\"hosts\":\"master\",\"command\":\"su - hbase -c \\u0027echo \\\"snapshot_all\\\" | hbase shell\\u0027\"}]",
+ "service_package_folder": "common-services/HBASE/0.96.0.2.0/package",
+ "hooks_folder": "HDP/2.0.6/hooks",
+ "script": "ru_execute_tasks.py",
+ "version": "2.2.0.0-2041",
+ "command_timeout": "60",
+ "script_type": "PYTHON"
+ },
+ "commandId": "33-1",
+ "clusterHostInfo": {
+ "ganglia_monitor_hosts": [
+ "c6403.ambari.apache.org",
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "nm_hosts": [
+ "c6403.ambari.apache.org"
+ ],
+ "app_timeline_server_hosts": [
+ "c6402.ambari.apache.org"
+ ],
+ "hive_metastore_host": [
+ "c6402.ambari.apache.org"
+ ],
+ "zkfc_hosts": [
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "all_ping_ports": [
+ "8670",
+ "8670",
+ "8670"
+ ],
+ "journalnode_hosts": [
+ "c6403.ambari.apache.org",
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "rm_host": [
+ "c6402.ambari.apache.org"
+ ],
+ "all_hosts": [
+ "c6403.ambari.apache.org",
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "hbase_rs_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "slave_hosts": [
+ "c6403.ambari.apache.org"
+ ],
+ "namenode_host": [
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "ganglia_server_host": [
+ "c6401.ambari.apache.org"
+ ],
+ "hbase_master_hosts": [
+ "c6401.ambari.apache.org"
+ ],
+ "hive_mysql_host": [
+ "c6402.ambari.apache.org"
+ ],
+ "oozie_server": [
+ "c6403.ambari.apache.org"
+ ],
+ "webhcat_server_host": [
+ "c6402.ambari.apache.org"
+ ],
+ "ambari_server_host": [
+ "10.0.0.15"
+ ],
+ "zookeeper_hosts": [
+ "c6403.ambari.apache.org",
+ "c6401.ambari.apache.org",
+ "c6402.ambari.apache.org"
+ ],
+ "hs_host": [
+ "c6402.ambari.apache.org"
+ ],
+ "hive_server_host": [
+ "c6402.ambari.apache.org"
+ ]
+ }
+}