You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by vg...@apache.org on 2014/10/01 00:32:31 UTC
[22/22] git commit: AMBARI-7292. Create stack definitions for Bigtop
0.8 (Newton Alex via vgogate)
AMBARI-7292. Create stack definitions for Bigtop 0.8 (Newton Alex via vgogate)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/83efcfea
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/83efcfea
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/83efcfea
Branch: refs/heads/trunk
Commit: 83efcfeae5821f82f3a808276ca42f642b4b4e62
Parents: d17ede1
Author: root <ro...@c6401.ambari.apache.org>
Authored: Tue Sep 30 22:12:04 2014 +0000
Committer: root <ro...@c6401.ambari.apache.org>
Committed: Tue Sep 30 22:12:30 2014 +0000
----------------------------------------------------------------------
ambari-server/pom.xml | 2 +
.../0.8/blueprints/multinode-default.json | 182 +
.../0.8/blueprints/singlenode-default.json | 136 +
.../BIGTOP/0.8/configuration/cluster-env.xml | 56 +
.../0.8/hooks/after-INSTALL/scripts/hook.py | 35 +
.../0.8/hooks/after-INSTALL/scripts/params.py | 68 +
.../scripts/shared_initialization.py | 52 +
.../BIGTOP/0.8/hooks/before-ANY/scripts/hook.py | 33 +
.../0.8/hooks/before-ANY/scripts/params.py | 33 +
.../before-ANY/scripts/shared_initialization.py | 58 +
.../before-INSTALL/files/changeToSecureUid.sh | 50 +
.../0.8/hooks/before-INSTALL/scripts/hook.py | 39 +
.../0.8/hooks/before-INSTALL/scripts/params.py | 118 +
.../scripts/repo_initialization.py | 57 +
.../scripts/shared_initialization.py | 94 +
.../before-INSTALL/templates/repo_suse_rhel.j2 | 7 +
.../before-INSTALL/templates/repo_ubuntu.j2 | 1 +
.../0.8/hooks/before-RESTART/scripts/hook.py | 29 +
.../hooks/before-START/files/checkForFormat.sh | 62 +
.../before-START/files/task-log4j.properties | 134 +
.../0.8/hooks/before-START/scripts/hook.py | 38 +
.../0.8/hooks/before-START/scripts/params.py | 146 +
.../scripts/shared_initialization.py | 175 +
.../templates/commons-logging.properties.j2 | 43 +
.../templates/exclude_hosts_list.j2 | 21 +
.../templates/hadoop-metrics2.properties.j2 | 65 +
.../before-START/templates/health_check-v2.j2 | 81 +
.../before-START/templates/health_check.j2 | 109 +
.../templates/include_hosts_list.j2 | 21 +
.../resources/stacks/BIGTOP/0.8/metainfo.xml | 22 +
.../stacks/BIGTOP/0.8/repos/repoinfo.xml | 35 +
.../stacks/BIGTOP/0.8/role_command_order.json | 74 +
.../services/FLUME/configuration/flume-conf.xml | 31 +
.../services/FLUME/configuration/flume-env.xml | 40 +
.../FLUME/configuration/flume-log4j.xml | 31 +
.../BIGTOP/0.8/services/FLUME/metainfo.xml | 69 +
.../BIGTOP/0.8/services/FLUME/metrics.json | 716 +
.../0.8/services/FLUME/package/scripts/flume.py | 250 +
.../FLUME/package/scripts/flume_check.py | 40 +
.../FLUME/package/scripts/flume_handler.py | 121 +
.../services/FLUME/package/scripts/params.py | 60 +
.../FLUME/package/templates/flume.conf.j2 | 24 +
.../FLUME/package/templates/log4j.properties.j2 | 67 +
.../GANGLIA/configuration/ganglia-env.xml | 72 +
.../BIGTOP/0.8/services/GANGLIA/metainfo.xml | 127 +
.../GANGLIA/package/files/checkGmetad.sh | 37 +
.../GANGLIA/package/files/checkGmond.sh | 62 +
.../GANGLIA/package/files/checkRrdcached.sh | 34 +
.../services/GANGLIA/package/files/gmetad.init | 73 +
.../services/GANGLIA/package/files/gmetadLib.sh | 204 +
.../services/GANGLIA/package/files/gmond.init | 73 +
.../services/GANGLIA/package/files/gmondLib.sh | 539 +
.../GANGLIA/package/files/rrdcachedLib.sh | 47 +
.../GANGLIA/package/files/setupGanglia.sh | 141 +
.../GANGLIA/package/files/startGmetad.sh | 68 +
.../GANGLIA/package/files/startGmond.sh | 85 +
.../GANGLIA/package/files/startRrdcached.sh | 79 +
.../GANGLIA/package/files/stopGmetad.sh | 43 +
.../services/GANGLIA/package/files/stopGmond.sh | 54 +
.../GANGLIA/package/files/stopRrdcached.sh | 41 +
.../GANGLIA/package/files/teardownGanglia.sh | 28 +
.../GANGLIA/package/scripts/functions.py | 31 +
.../services/GANGLIA/package/scripts/ganglia.py | 97 +
.../GANGLIA/package/scripts/ganglia_monitor.py | 236 +
.../package/scripts/ganglia_monitor_service.py | 27 +
.../GANGLIA/package/scripts/ganglia_server.py | 119 +
.../package/scripts/ganglia_server_service.py | 27 +
.../services/GANGLIA/package/scripts/params.py | 155 +
.../GANGLIA/package/scripts/status_params.py | 25 +
.../GANGLIA/package/templates/ganglia.conf.j2 | 34 +
.../package/templates/gangliaClusters.conf.j2 | 43 +
.../GANGLIA/package/templates/gangliaEnv.sh.j2 | 46 +
.../GANGLIA/package/templates/gangliaLib.sh.j2 | 85 +
.../GANGLIA/package/templates/rrd.py.j2 | 361 +
.../services/HBASE/configuration/hbase-env.xml | 133 +
.../HBASE/configuration/hbase-log4j.xml | 143 +
.../HBASE/configuration/hbase-policy.xml | 53 +
.../services/HBASE/configuration/hbase-site.xml | 370 +
.../BIGTOP/0.8/services/HBASE/metainfo.xml | 139 +
.../BIGTOP/0.8/services/HBASE/metrics.json | 13635 +++++++++++++++++
.../HBASE/package/files/draining_servers.rb | 164 +
.../HBASE/package/files/hbaseSmokeVerify.sh | 33 +
.../services/HBASE/package/scripts/__init__.py | 19 +
.../services/HBASE/package/scripts/functions.py | 40 +
.../0.8/services/HBASE/package/scripts/hbase.py | 144 +
.../HBASE/package/scripts/hbase_client.py | 43 +
.../HBASE/package/scripts/hbase_decommission.py | 74 +
.../HBASE/package/scripts/hbase_master.py | 70 +
.../HBASE/package/scripts/hbase_regionserver.py | 66 +
.../HBASE/package/scripts/hbase_service.py | 51 +
.../services/HBASE/package/scripts/params.py | 123 +
.../HBASE/package/scripts/service_check.py | 79 +
.../HBASE/package/scripts/status_params.py | 26 +
...-metrics2-hbase.properties-GANGLIA-MASTER.j2 | 81 +
...doop-metrics2-hbase.properties-GANGLIA-RS.j2 | 80 +
.../HBASE/package/templates/hbase-smoke.sh.j2 | 44 +
.../package/templates/hbase_client_jaas.conf.j2 | 23 +
.../templates/hbase_grant_permissions.j2 | 39 +
.../package/templates/hbase_master_jaas.conf.j2 | 26 +
.../templates/hbase_regionserver_jaas.conf.j2 | 26 +
.../HBASE/package/templates/regionservers.j2 | 20 +
.../services/HDFS/configuration/core-site.xml | 180 +
.../services/HDFS/configuration/hadoop-env.xml | 198 +
.../HDFS/configuration/hadoop-policy.xml | 134 +
.../services/HDFS/configuration/hdfs-log4j.xml | 201 +
.../services/HDFS/configuration/hdfs-site.xml | 496 +
.../BIGTOP/0.8/services/HDFS/metainfo.xml | 225 +
.../BIGTOP/0.8/services/HDFS/metrics.json | 7840 ++++++++++
.../HDFS/package/files/checkForFormat.sh | 68 +
.../services/HDFS/package/files/checkWebUI.py | 53 +
.../scripts/balancer-emulator/balancer-err.log | 1032 ++
.../scripts/balancer-emulator/balancer.log | 29 +
.../scripts/balancer-emulator/hdfs-command.py | 45 +
.../services/HDFS/package/scripts/datanode.py | 59 +
.../0.8/services/HDFS/package/scripts/hdfs.py | 64 +
.../HDFS/package/scripts/hdfs_client.py | 53 +
.../HDFS/package/scripts/hdfs_datanode.py | 47 +
.../HDFS/package/scripts/hdfs_namenode.py | 157 +
.../HDFS/package/scripts/hdfs_rebalance.py | 130 +
.../HDFS/package/scripts/hdfs_snamenode.py | 45 +
.../HDFS/package/scripts/journalnode.py | 73 +
.../services/HDFS/package/scripts/namenode.py | 134 +
.../0.8/services/HDFS/package/scripts/params.py | 213 +
.../HDFS/package/scripts/service_check.py | 111 +
.../services/HDFS/package/scripts/snamenode.py | 65 +
.../HDFS/package/scripts/status_params.py | 31 +
.../0.8/services/HDFS/package/scripts/utils.py | 66 +
.../services/HDFS/package/scripts/zkfc_slave.py | 64 +
.../package/templates/exclude_hosts_list.j2 | 21 +
.../HDFS/package/templates/hdfs.conf.j2 | 35 +
.../services/HDFS/package/templates/slaves.j2 | 21 +
.../services/HIVE/configuration/hive-env.xml | 134 +
.../HIVE/configuration/hive-exec-log4j.xml | 111 +
.../services/HIVE/configuration/hive-log4j.xml | 120 +
.../services/HIVE/configuration/hive-site.xml | 314 +
.../HIVE/etc/hive-schema-0.12.0.mysql.sql | 777 +
.../HIVE/etc/hive-schema-0.12.0.oracle.sql | 718 +
.../HIVE/etc/hive-schema-0.12.0.postgres.sql | 1406 ++
.../BIGTOP/0.8/services/HIVE/metainfo.xml | 271 +
.../services/HIVE/package/files/addMysqlUser.sh | 41 +
.../HIVE/package/files/addPostgreSQLUser.sh | 44 +
.../services/HIVE/package/files/hcatSmoke.sh | 36 +
.../services/HIVE/package/files/hiveSmoke.sh | 24 +
.../services/HIVE/package/files/hiveserver2.sql | 23 +
.../HIVE/package/files/hiveserver2Smoke.sh | 32 +
.../0.8/services/HIVE/package/files/pigSmoke.sh | 18 +
.../HIVE/package/files/startMetastore.sh | 23 +
.../services/HIVE/package/scripts/__init__.py | 19 +
.../0.8/services/HIVE/package/scripts/hcat.py | 55 +
.../HIVE/package/scripts/hcat_client.py | 43 +
.../HIVE/package/scripts/hcat_service_check.py | 78 +
.../0.8/services/HIVE/package/scripts/hive.py | 213 +
.../HIVE/package/scripts/hive_client.py | 42 +
.../HIVE/package/scripts/hive_metastore.py | 64 +
.../HIVE/package/scripts/hive_server.py | 66 +
.../HIVE/package/scripts/hive_service.py | 110 +
.../HIVE/package/scripts/install_jars.py | 99 +
.../HIVE/package/scripts/mysql_server.py | 70 +
.../HIVE/package/scripts/mysql_service.py | 46 +
.../0.8/services/HIVE/package/scripts/params.py | 215 +
.../HIVE/package/scripts/postgresql_server.py | 109 +
.../HIVE/package/scripts/postgresql_service.py | 39 +
.../HIVE/package/scripts/service_check.py | 47 +
.../HIVE/package/scripts/status_params.py | 37 +
.../HIVE/package/templates/hcat-env.sh.j2 | 43 +
.../package/templates/startHiveserver2.sh.j2 | 29 +
.../NAGIOS/configuration/nagios-env.xml | 53 +
.../BIGTOP/0.8/services/NAGIOS/metainfo.xml | 160 +
.../NAGIOS/package/files/check_aggregate.php | 248 +
.../NAGIOS/package/files/check_ambari_alerts.py | 80 +
.../package/files/check_checkpoint_time.py | 112 +
.../services/NAGIOS/package/files/check_cpu.php | 109 +
.../services/NAGIOS/package/files/check_cpu.pl | 114 +
.../NAGIOS/package/files/check_cpu_ha.php | 116 +
.../package/files/check_datanode_storage.php | 100 +
.../NAGIOS/package/files/check_hdfs_blocks.php | 102 +
.../package/files/check_hdfs_capacity.php | 109 +
.../files/check_hive_metastore_status.sh | 45 +
.../NAGIOS/package/files/check_hue_status.sh | 31 +
.../files/check_mapred_local_dir_used.sh | 34 +
.../package/files/check_name_dir_status.php | 93 +
.../NAGIOS/package/files/check_namenodes_ha.sh | 83 +
.../package/files/check_nodemanager_health.sh | 45 +
.../NAGIOS/package/files/check_oozie_status.sh | 45 +
.../NAGIOS/package/files/check_rpcq_latency.php | 104 +
.../package/files/check_rpcq_latency_ha.php | 115 +
.../package/files/check_templeton_status.sh | 46 +
.../NAGIOS/package/files/check_webui.sh | 103 +
.../NAGIOS/package/files/check_webui_ha.sh | 64 +
.../NAGIOS/package/files/check_wrapper.sh | 94 +
.../package/files/hdp_mon_nagios_addons.conf | 24 +
.../NAGIOS/package/files/hdp_nagios_init.php | 81 +
.../NAGIOS/package/files/nagios_alerts.php | 513 +
.../services/NAGIOS/package/files/sys_logger.py | 186 +
.../NAGIOS/package/scripts/functions.py | 47 +
.../services/NAGIOS/package/scripts/nagios.py | 112 +
.../NAGIOS/package/scripts/nagios_server.py | 111 +
.../package/scripts/nagios_server_config.py | 98 +
.../NAGIOS/package/scripts/nagios_service.py | 103 +
.../services/NAGIOS/package/scripts/params.py | 287 +
.../NAGIOS/package/scripts/status_params.py | 29 +
.../NAGIOS/package/templates/contacts.cfg.j2 | 109 +
.../package/templates/hadoop-commands.cfg.j2 | 166 +
.../package/templates/hadoop-hostgroups.cfg.j2 | 33 +
.../package/templates/hadoop-hosts.cfg.j2 | 53 +
.../templates/hadoop-servicegroups.cfg.j2 | 119 +
.../package/templates/hadoop-services.cfg.j2 | 804 +
.../NAGIOS/package/templates/nagios.cfg.j2 | 1365 ++
.../NAGIOS/package/templates/nagios.conf.j2 | 84 +
.../services/NAGIOS/package/templates/nagios.j2 | 164 +
.../NAGIOS/package/templates/resource.cfg.j2 | 51 +
.../services/OOZIE/configuration/oozie-env.xml | 129 +
.../OOZIE/configuration/oozie-log4j.xml | 97 +
.../services/OOZIE/configuration/oozie-site.xml | 312 +
.../BIGTOP/0.8/services/OOZIE/metainfo.xml | 165 +
.../services/OOZIE/package/files/oozieSmoke2.sh | 107 +
.../OOZIE/package/files/wrap_ooziedb.sh | 31 +
.../0.8/services/OOZIE/package/scripts/oozie.py | 152 +
.../OOZIE/package/scripts/oozie_client.py | 43 +
.../OOZIE/package/scripts/oozie_server.py | 56 +
.../OOZIE/package/scripts/oozie_service.py | 75 +
.../services/OOZIE/package/scripts/params.py | 121 +
.../OOZIE/package/scripts/service_check.py | 59 +
.../OOZIE/package/scripts/status_params.py | 26 +
.../package/templates/oozie-log4j.properties.j2 | 92 +
.../0.8/services/PIG/configuration/pig-env.xml | 38 +
.../services/PIG/configuration/pig-log4j.xml | 62 +
.../PIG/configuration/pig-properties.xml | 92 +
.../stacks/BIGTOP/0.8/services/PIG/metainfo.xml | 85 +
.../0.8/services/PIG/package/files/pigSmoke.sh | 18 +
.../0.8/services/PIG/package/scripts/params.py | 45 +
.../0.8/services/PIG/package/scripts/pig.py | 58 +
.../services/PIG/package/scripts/pig_client.py | 41 +
.../PIG/package/scripts/service_check.py | 67 +
.../services/SQOOP/configuration/sqoop-env.xml | 54 +
.../BIGTOP/0.8/services/SQOOP/metainfo.xml | 92 +
.../services/SQOOP/package/scripts/__init__.py | 19 +
.../services/SQOOP/package/scripts/params.py | 37 +
.../SQOOP/package/scripts/service_check.py | 37 +
.../0.8/services/SQOOP/package/scripts/sqoop.py | 57 +
.../SQOOP/package/scripts/sqoop_client.py | 41 +
.../WEBHCAT/configuration/webhcat-env.xml | 54 +
.../WEBHCAT/configuration/webhcat-site.xml | 138 +
.../BIGTOP/0.8/services/WEBHCAT/metainfo.xml | 107 +
.../WEBHCAT/package/files/templetonSmoke.sh | 96 +
.../WEBHCAT/package/scripts/__init__.py | 20 +
.../services/WEBHCAT/package/scripts/params.py | 83 +
.../WEBHCAT/package/scripts/service_check.py | 45 +
.../WEBHCAT/package/scripts/status_params.py | 26 +
.../services/WEBHCAT/package/scripts/webhcat.py | 93 +
.../WEBHCAT/package/scripts/webhcat_server.py | 53 +
.../WEBHCAT/package/scripts/webhcat_service.py | 40 +
.../YARN/configuration-mapred/mapred-env.xml | 65 +
.../YARN/configuration-mapred/mapred-site.xml | 366 +
.../YARN/configuration/capacity-scheduler.xml | 124 +
.../services/YARN/configuration/yarn-env.xml | 184 +
.../services/YARN/configuration/yarn-log4j.xml | 71 +
.../services/YARN/configuration/yarn-site.xml | 413 +
.../BIGTOP/0.8/services/YARN/metainfo.xml | 250 +
.../BIGTOP/0.8/services/YARN/metrics.json | 5360 +++++++
.../files/validateYarnComponentStatus.py | 170 +
.../services/YARN/package/scripts/__init__.py | 20 +
.../scripts/application_timeline_server.py | 55 +
.../YARN/package/scripts/historyserver.py | 53 +
.../package/scripts/mapred_service_check.py | 76 +
.../YARN/package/scripts/mapreduce2_client.py | 42 +
.../YARN/package/scripts/nodemanager.py | 59 +
.../0.8/services/YARN/package/scripts/params.py | 157 +
.../YARN/package/scripts/resourcemanager.py | 100 +
.../services/YARN/package/scripts/service.py | 75 +
.../YARN/package/scripts/service_check.py | 67 +
.../YARN/package/scripts/status_params.py | 35 +
.../0.8/services/YARN/package/scripts/yarn.py | 231 +
.../YARN/package/scripts/yarn_client.py | 42 +
.../package/templates/container-executor.cfg.j2 | 40 +
.../package/templates/exclude_hosts_list.j2 | 21 +
.../YARN/package/templates/mapreduce.conf.j2 | 35 +
.../package/templates/taskcontroller.cfg.j2 | 38 +
.../YARN/package/templates/yarn.conf.j2 | 35 +
.../ZOOKEEPER/configuration/zookeeper-env.xml | 84 +
.../ZOOKEEPER/configuration/zookeeper-log4j.xml | 101 +
.../BIGTOP/0.8/services/ZOOKEEPER/metainfo.xml | 89 +
.../services/ZOOKEEPER/package/files/zkEnv.sh | 96 +
.../ZOOKEEPER/package/files/zkServer.sh | 120 +
.../ZOOKEEPER/package/files/zkService.sh | 26 +
.../services/ZOOKEEPER/package/files/zkSmoke.sh | 78 +
.../ZOOKEEPER/package/scripts/__init__.py | 20 +
.../ZOOKEEPER/package/scripts/params.py | 76 +
.../ZOOKEEPER/package/scripts/service_check.py | 46 +
.../ZOOKEEPER/package/scripts/status_params.py | 26 +
.../ZOOKEEPER/package/scripts/zookeeper.py | 110 +
.../package/scripts/zookeeper_client.py | 42 +
.../package/scripts/zookeeper_server.py | 54 +
.../package/scripts/zookeeper_service.py | 42 +
.../package/templates/configuration.xsl.j2 | 42 +
.../ZOOKEEPER/package/templates/zoo.cfg.j2 | 69 +
.../templates/zookeeper_client_jaas.conf.j2 | 24 +
.../package/templates/zookeeper_jaas.conf.j2 | 26 +
.../stacks/BIGTOP/0.8/services/stack_advisor.py | 439 +
299 files changed, 59369 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index 0fc41aa..8eeff3a 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -139,6 +139,8 @@
<exclude>src/main/resources/stacks/HDP/2.1.GlusterFS/services/YARN/package/templates/exclude_hosts_list.j2</exclude>
<exclude>src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer-err.log</exclude>
<exclude>src/main/resources/stacks/HDP/2.0.6/services/HDFS/package/scripts/balancer-emulator/balancer.log</exclude>
+ <exclude>src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/balancer-emulator/balancer.log</exclude>
+ <exclude>src/main/resources/stacks/BIGTOP/0.8/services/HDFS/package/scripts/balancer-emulator/balancer-err.log</exclude>
<exclude>conf/unix/ca.config</exclude>
<exclude>conf/unix/krb5JAASLogin.conf</exclude>
<exclude>**/*.json</exclude>
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/multinode-default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/multinode-default.json b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/multinode-default.json
new file mode 100644
index 0000000..642fcfa
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/multinode-default.json
@@ -0,0 +1,182 @@
+{
+ "configurations" : [
+ {
+ "nagios-env" : {
+ "nagios_contact" : "admin@localhost"
+ }
+ }
+ ],
+ "host_groups" : [
+ {
+ "name" : "master_1",
+ "components" : [
+ {
+ "name" : "NAMENODE"
+ },
+ {
+ "name" : "ZOOKEEPER_SERVER"
+ },
+ {
+ "name" : "HBASE_MASTER"
+ },
+ {
+ "name" : "GANGLIA_SERVER"
+ },
+ {
+ "name" : "HDFS_CLIENT"
+ },
+ {
+ "name" : "YARN_CLIENT"
+ },
+ {
+ "name" : "HCAT"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ }
+ ],
+ "cardinality" : "1"
+ },
+ {
+ "name" : "master_2",
+ "components" : [
+
+ {
+ "name" : "ZOOKEEPER_CLIENT"
+ },
+ {
+ "name" : "HISTORYSERVER"
+ },
+ {
+ "name" : "HIVE_SERVER"
+ },
+ {
+ "name" : "SECONDARY_NAMENODE"
+ },
+ {
+ "name" : "HIVE_METASTORE"
+ },
+ {
+ "name" : "HDFS_CLIENT"
+ },
+ {
+ "name" : "HIVE_CLIENT"
+ },
+ {
+ "name" : "YARN_CLIENT"
+ },
+ {
+ "name" : "MYSQL_SERVER"
+ },
+ {
+ "name" : "POSTGRESQL_SERVER"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ },
+ {
+ "name" : "WEBHCAT_SERVER"
+ }
+ ],
+ "cardinality" : "1"
+ },
+ {
+ "name" : "master_3",
+ "components" : [
+ {
+ "name" : "RESOURCEMANAGER"
+ },
+ {
+ "name" : "ZOOKEEPER_SERVER"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ }
+ ],
+ "cardinality" : "1"
+ },
+ {
+ "name" : "master_4",
+ "components" : [
+ {
+ "name" : "OOZIE_SERVER"
+ },
+ {
+ "name" : "ZOOKEEPER_SERVER"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ }
+ ],
+ "cardinality" : "1"
+ },
+ {
+ "name" : "slave",
+ "components" : [
+ {
+ "name" : "HBASE_REGIONSERVER"
+ },
+ {
+ "name" : "NODEMANAGER"
+ },
+ {
+ "name" : "DATANODE"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ }
+ ],
+ "cardinality" : "${slavesCount}"
+ },
+ {
+ "name" : "gateway",
+ "components" : [
+ {
+ "name" : "AMBARI_SERVER"
+ },
+ {
+ "name" : "NAGIOS_SERVER"
+ },
+ {
+ "name" : "ZOOKEEPER_CLIENT"
+ },
+ {
+ "name" : "PIG"
+ },
+ {
+ "name" : "OOZIE_CLIENT"
+ },
+ {
+ "name" : "HBASE_CLIENT"
+ },
+ {
+ "name" : "HCAT"
+ },
+ {
+ "name" : "SQOOP"
+ },
+ {
+ "name" : "HDFS_CLIENT"
+ },
+ {
+ "name" : "HIVE_CLIENT"
+ },
+ {
+ "name" : "YARN_CLIENT"
+ },
+ {
+ "name" : "MAPREDUCE2_CLIENT"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ }
+ ],
+ "cardinality" : "1"
+ }
+ ],
+ "Blueprints" : {
+ "blueprint_name" : "blueprint-multinode-default",
+ "stack_name" : "BIGTOP",
+ "stack_version" : "0.8"
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/singlenode-default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/singlenode-default.json b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/singlenode-default.json
new file mode 100644
index 0000000..3c769dd
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/blueprints/singlenode-default.json
@@ -0,0 +1,136 @@
+{
+ "configurations" : [
+ {
+ "nagios-env" : {
+ "nagios_contact" : "admin@localhost"
+ }
+ }
+ ],
+ "host_groups" : [
+ {
+ "name" : "host_group_1",
+ "components" : [
+ {
+ "name" : "STORM_REST_API"
+ },
+ {
+ "name" : "PIG"
+ },
+ {
+ "name" : "HISTORYSERVER"
+ },
+ {
+ "name" : "HBASE_REGIONSERVER"
+ },
+ {
+ "name" : "OOZIE_CLIENT"
+ },
+ {
+ "name" : "HBASE_CLIENT"
+ },
+ {
+ "name" : "NAMENODE"
+ },
+ {
+ "name" : "SUPERVISOR"
+ },
+ {
+ "name" : "FALCON_SERVER"
+ },
+ {
+ "name" : "HCAT"
+ },
+ {
+ "name" : "AMBARI_SERVER"
+ },
+ {
+ "name" : "APP_TIMELINE_SERVER"
+ },
+ {
+ "name" : "HDFS_CLIENT"
+ },
+ {
+ "name" : "HIVE_CLIENT"
+ },
+ {
+ "name" : "NODEMANAGER"
+ },
+ {
+ "name" : "DATANODE"
+ },
+ {
+ "name" : "WEBHCAT_SERVER"
+ },
+ {
+ "name" : "RESOURCEMANAGER"
+ },
+ {
+ "name" : "ZOOKEEPER_SERVER"
+ },
+ {
+ "name" : "ZOOKEEPER_CLIENT"
+ },
+ {
+ "name" : "STORM_UI_SERVER"
+ },
+ {
+ "name" : "HBASE_MASTER"
+ },
+ {
+ "name" : "HIVE_SERVER"
+ },
+ {
+ "name" : "OOZIE_SERVER"
+ },
+ {
+ "name" : "FALCON_CLIENT"
+ },
+ {
+ "name" : "NAGIOS_SERVER"
+ },
+ {
+ "name" : "SECONDARY_NAMENODE"
+ },
+ {
+ "name" : "TEZ_CLIENT"
+ },
+ {
+ "name" : "HIVE_METASTORE"
+ },
+ {
+ "name" : "GANGLIA_SERVER"
+ },
+ {
+ "name" : "SQOOP"
+ },
+ {
+ "name" : "YARN_CLIENT"
+ },
+ {
+ "name" : "MAPREDUCE2_CLIENT"
+ },
+ {
+ "name" : "MYSQL_SERVER"
+ },
+ {
+ "name" : "POSTGRESQL_SERVER"
+ },
+ {
+ "name" : "GANGLIA_MONITOR"
+ },
+ {
+ "name" : "DRPC_SERVER"
+ },
+ {
+ "name" : "NIMBUS"
+ }
+ ],
+ "cardinality" : "1"
+ }
+ ],
+ "Blueprints" : {
+ "blueprint_name" : "blueprint-singlenode-default",
+ "stack_name" : "BIGTOP",
+ "stack_version" : "0.8"
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/configuration/cluster-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/configuration/cluster-env.xml b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/configuration/cluster-env.xml
new file mode 100644
index 0000000..d41ff98
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/configuration/cluster-env.xml
@@ -0,0 +1,56 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <property>
+ <name>security_enabled</name>
+ <value>false</value>
+ <description>Hadoop Security</description>
+ </property>
+ <property>
+ <name>kerberos_domain</name>
+ <value>EXAMPLE.COM</value>
+ <description>Kerberos realm.</description>
+ </property>
+ <property>
+ <name>ignore_groupsusers_create</name>
+ <value>false</value>
+ <description>Whether to ignore failures on users and group creation</description>
+ </property>
+ <property>
+ <name>smokeuser</name>
+ <value>ambari-qa</value>
+ <property-type>USER</property-type>
+ <description>User executing service checks</description>
+ </property>
+ <property>
+ <name>smokeuser_keytab</name>
+ <value>/etc/security/keytabs/smokeuser.headless.keytab</value>
+ <description>Path to smoke test user keytab file</description>
+ </property>
+ <property>
+ <name>user_group</name>
+ <value>hadoop</value>
+ <property-type>GROUP</property-type>
+ <description>Hadoop user group.</description>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..cf83e19
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/hook.py
@@ -0,0 +1,35 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from shared_initialization import *
+
+#Hook for hosts with only client without other components
+class AfterInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ env.set_params(params)
+ setup_hadoop_env()
+ setup_config()
+
+if __name__ == "__main__":
+ AfterInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/params.py
new file mode 100644
index 0000000..d537199
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/params.py
@@ -0,0 +1,68 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.core.system import System
+import os
+
+config = Script.get_config()
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+#java params
+java_home = config['hostLevelParams']['java_home']
+#hadoop params
+hadoop_conf_dir = "/etc/hadoop/conf"
+hadoop_conf_empty_dir = "/etc/hadoop/conf.empty"
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger']
+hadoop_env_sh_template = config['configurations']['hadoop-env']['content']
+
+#hadoop-env.sh
+java_home = config['hostLevelParams']['java_home']
+
+if str(config['hostLevelParams']['stack_version']).startswith('2.0') and System.get_instance().os_family != "suse":
+ # deprecated rhel jsvc_path
+ jsvc_path = "/usr/libexec/bigtop-utils"
+else:
+ jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#users and groups
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
+
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+has_namenode = not len(namenode_host) == 0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..1ba0ae7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/after-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,52 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+import os
+from resource_management import *
+
+def setup_hadoop_env():
+ import params
+ if params.has_namenode:
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+ Directory(params.hadoop_conf_empty_dir,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+ Link(params.hadoop_conf_dir,
+ to=params.hadoop_conf_empty_dir,
+ not_if=format("ls {hadoop_conf_dir}")
+ )
+ File(os.path.join(params.hadoop_conf_dir, 'hadoop-env.sh'),
+ owner=tc_owner,
+ content=InlineTemplate(params.hadoop_env_sh_template)
+ )
+
+def setup_config():
+ import params
+ if params.has_namenode:
+ XmlConfig("core-site.xml",
+ conf_dir=params.hadoop_conf_dir,
+ configurations=params.config['configurations']['core-site'],
+ configuration_attributes=params.config['configuration_attributes']['core-site'],
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/hook.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/hook.py
new file mode 100644
index 0000000..0f97e28
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/hook.py
@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from shared_initialization import *
+
+class BeforeAnyHook(Hook):
+
+ def hook(self, env):
+ import params
+ env.set_params(params)
+
+ setup_jce()
+
+if __name__ == "__main__":
+ BeforeAnyHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/params.py
new file mode 100644
index 0000000..af67cb6
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/params.py
@@ -0,0 +1,33 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_name = default("/hostLevelParams/jdk_name", None)
+java_home = config['hostLevelParams']['java_home']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/shared_initialization.py
new file mode 100644
index 0000000..74bdce0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-ANY/scripts/shared_initialization.py
@@ -0,0 +1,58 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management import *
+
+
+
+def setup_jce():
+ import params
+
+ if not params.jdk_name:
+ return
+
+ environment = {
+ "no_proxy": format("{ambari_server_hostname}")
+ }
+
+ if params.jce_policy_zip is not None:
+ jce_curl_target = format("{artifact_dir}/{jce_policy_zip}")
+ download_jce = format("mkdir -p {artifact_dir}; \
+ curl -kf -x \"\" --retry 10 \
+ {jce_location}/{jce_policy_zip} -o {jce_curl_target}")
+ Execute( download_jce,
+ path = ["/bin","/usr/bin/"],
+ not_if =format("test -e {jce_curl_target}"),
+ ignore_failures = True,
+ environment = environment
+ )
+ elif params.security_enabled:
+ # Something weird is happening
+ raise Fail("Security is enabled, but JCE policy zip is not specified.")
+
+ if params.security_enabled:
+ security_dir = format("{java_home}/jre/lib/security")
+ extract_cmd = format("rm -f local_policy.jar; rm -f US_export_policy.jar; unzip -o -j -q {jce_curl_target}")
+ Execute(extract_cmd,
+ only_if = format("test -e {security_dir} && test -f {jce_curl_target}"),
+ cwd = security_dir,
+ path = ['/bin/','/usr/bin']
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/files/changeToSecureUid.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/files/changeToSecureUid.sh b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/files/changeToSecureUid.sh
new file mode 100644
index 0000000..154c1c0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/files/changeToSecureUid.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+username=$1
+directories=$2
+
+function find_available_uid() {
+ for ((i=1001; i<=2000; i++))
+ do
+ grep -q $i /etc/passwd
+ if [ "$?" -ne 0 ]
+ then
+ newUid=$i
+ break
+ fi
+ done
+}
+
+find_available_uid
+
+if [ $newUid -eq 0 ]
+then
+ echo "Failed to find Uid between 1000 and 2000"
+ exit 1
+fi
+
+dir_array=($(echo $directories | sed 's/,/\n/g'))
+old_uid=$(id -u $username)
+echo "Changing uid of $username from $old_uid to $newUid"
+echo "Changing directory permisions for ${dir_array[@]}"
+usermod -u $newUid $username && for dir in ${dir_array[@]} ; do chown -Rh $newUid $dir ; done
+exit 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/hook.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/hook.py
new file mode 100644
index 0000000..a8a5662
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/hook.py
@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from shared_initialization import *
+from repo_initialization import *
+
+class BeforeInstallHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ env.set_params(params)
+
+ install_repos()
+ install_packages()
+ setup_java()
+ setup_users()
+
+if __name__ == "__main__":
+ BeforeInstallHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py
new file mode 100644
index 0000000..01789a7
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/params.py
@@ -0,0 +1,118 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.core.system import System
+import os
+import json
+import collections
+
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+#users and groups
+hbase_user = config['configurations']['hbase-env']['hbase_user']
+nagios_user = config['configurations']['nagios-env']['nagios_user']
+smoke_user = config['configurations']['cluster-env']['smokeuser']
+gmetad_user = config['configurations']['ganglia-env']["gmetad_user"]
+gmond_user = config['configurations']['ganglia-env']["gmond_user"]
+tez_user = config['configurations']['tez-env']["tez_user"]
+
+user_group = config['configurations']['cluster-env']['user_group']
+proxyuser_group = default("/configurations/hadoop-env/proxyuser_group","users")
+nagios_group = config['configurations']['nagios-env']['nagios_group']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+hagios_server_hosts = default("/clusterHostInfo/nagios_server_host", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+storm_server_hosts = default("/clusterHostInfo/nimbus_hosts", [])
+falcon_host = default('/clusterHostInfo/falcon_server_hosts', [])
+
+has_sqoop_client = 'sqoop-env' in config['configurations']
+has_namenode = not len(namenode_host) == 0
+has_hs = not len(hs_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_nagios = not len(hagios_server_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+has_storm_server = not len(storm_server_hosts) == 0
+has_falcon_server = not len(falcon_host) == 0
+has_tez = 'tez-site' in config['configurations']
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+
+hbase_tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#java params
+java_home = config['hostLevelParams']['java_home']
+artifact_dir = format("{tmp_dir}/AMBARI-artifacts/")
+jdk_name = default("/hostLevelParams/jdk_name", None) # None when jdk is already installed by user
+jce_policy_zip = default("/hostLevelParams/jce_name", None) # None when jdk is already installed by user
+jce_location = config['hostLevelParams']['jdk_location']
+jdk_location = config['hostLevelParams']['jdk_location']
+ignore_groupsusers_create = default("/configurations/cluster-env/ignore_groupsusers_create", False)
+
+smoke_user_dirs = format("/tmp/hadoop-{smoke_user},/tmp/hsperfdata_{smoke_user},/home/{smoke_user},/tmp/{smoke_user},/tmp/sqoop-{smoke_user}")
+if has_hbase_masters:
+ hbase_user_dirs = format("/home/{hbase_user},/tmp/{hbase_user},/usr/bin/{hbase_user},/var/log/{hbase_user},{hbase_tmp_dir}")
+#repo params
+repo_info = config['hostLevelParams']['repo_info']
+service_repo_info = default("/hostLevelParams/service_repo_info",None)
+
+user_to_groups_dict = collections.defaultdict(lambda:[user_group])
+user_to_groups_dict[smoke_user] = [proxyuser_group]
+if has_ganglia_server:
+ user_to_groups_dict[gmond_user] = [gmond_user]
+ user_to_groups_dict[gmetad_user] = [gmetad_user]
+if has_tez:
+ user_to_groups_dict[tez_user] = [proxyuser_group]
+
+user_to_gid_dict = collections.defaultdict(lambda:user_group)
+if has_nagios:
+ user_to_gid_dict[nagios_user] = nagios_group
+
+user_list = json.loads(config['hostLevelParams']['user_list'])
+group_list = json.loads(config['hostLevelParams']['group_list'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/repo_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/repo_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/repo_initialization.py
new file mode 100644
index 0000000..87ce1f1
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/repo_initialization.py
@@ -0,0 +1,57 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from resource_management import *
+import json
+
+# components_lits = repoName + postfix
+_UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
+
+def _alter_repo(action, repo_string, repo_template):
+ """
+ @param action: "delete" or "create"
+ @param repo_string: e.g. "[{\"baseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\",\"osType\":\"centos6\",\"repoId\":\"HDP-2.0._\",\"repoName\":\"HDP\",\"defaultBaseUrl\":\"http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.0.6.0\"}]"
+ """
+ repo_dicts = json.loads(repo_string)
+
+ if not isinstance(repo_dicts, list):
+ repo_dicts = [repo_dicts]
+
+ for repo in repo_dicts:
+ if not 'baseUrl' in repo:
+ repo['baseUrl'] = None
+ if not 'mirrorsList' in repo:
+ repo['mirrorsList'] = None
+
+ ubuntu_components = [ repo['repoName'] ] + _UBUNTU_REPO_COMPONENTS_POSTFIX
+
+ Repository(repo['repoId'],
+ action = action,
+ base_url = repo['baseUrl'],
+ mirror_list = repo['mirrorsList'],
+ repo_file_name = repo['repoName'],
+ repo_template = repo_template,
+ components = ubuntu_components, # ubuntu specific
+ )
+
+def install_repos():
+ import params
+ template = "repo_suse_rhel.j2" if System.get_instance().os_family in ["suse", "redhat"] else "repo_ubuntu.j2"
+ _alter_repo("create", params.repo_info, template)
+ if params.service_repo_info:
+ _alter_repo("create", params.service_repo_info, template)
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py
new file mode 100644
index 0000000..720777f
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/scripts/shared_initialization.py
@@ -0,0 +1,94 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management import *
+
+def setup_users():
+ """
+ Creates users before cluster installation
+ """
+ import params
+
+ for group in params.group_list:
+ Group(group,
+ ignore_failures = params.ignore_groupsusers_create
+ )
+
+ for user in params.user_list:
+ User(user,
+ gid = params.user_to_gid_dict[user],
+ groups = params.user_to_groups_dict[user],
+ ignore_failures = params.ignore_groupsusers_create
+ )
+
+ set_uid(params.smoke_user, params.smoke_user_dirs)
+
+ if params.has_hbase_masters:
+ set_uid(params.hbase_user, params.hbase_user_dirs)
+
+def set_uid(user, user_dirs):
+ """
+ user_dirs - comma separated directories
+ """
+ import params
+
+ File(format("{tmp_dir}/changeUid.sh"),
+ content=StaticFile("changeToSecureUid.sh"),
+ mode=0555)
+ Execute(format("{tmp_dir}/changeUid.sh {user} {user_dirs} 2>/dev/null"),
+ not_if = format("test $(id -u {user}) -gt 1000"))
+
+def setup_java():
+ """
+ Installs jdk using specific params, that comes from ambari-server
+ """
+ import params
+
+ jdk_curl_target = format("{artifact_dir}/{jdk_name}")
+ java_dir = os.path.dirname(params.java_home)
+ java_exec = format("{java_home}/bin/java")
+
+ if not params.jdk_name:
+ return
+
+ environment = {
+ "no_proxy": format("{ambari_server_hostname}")
+ }
+
+ Execute(format("mkdir -p {artifact_dir} ; \
+ curl -kf -x \"\" \
+ --retry 10 {jdk_location}/{jdk_name} -o {jdk_curl_target}"),
+ path = ["/bin","/usr/bin/"],
+ not_if = format("test -e {java_exec}"),
+ environment = environment)
+
+ if params.jdk_name.endswith(".bin"):
+ install_cmd = format("mkdir -p {java_dir} ; chmod +x {jdk_curl_target}; cd {java_dir} ; echo A | {jdk_curl_target} -noregister > /dev/null 2>&1")
+ elif params.jdk_name.endswith(".gz"):
+ install_cmd = format("mkdir -p {java_dir} ; cd {java_dir} ; tar -xf {jdk_curl_target} > /dev/null 2>&1")
+
+ Execute(install_cmd,
+ path = ["/bin","/usr/bin/"],
+ not_if = format("test -e {java_exec}")
+ )
+
+def install_packages():
+ Package(['unzip', 'curl'])
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_suse_rhel.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_suse_rhel.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_suse_rhel.j2
new file mode 100644
index 0000000..d486f89
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_suse_rhel.j2
@@ -0,0 +1,7 @@
+[{{repo_id}}]
+name={{repo_file_name}}
+{% if mirror_list %}mirrorlist={{mirror_list}}{% else %}baseurl={{base_url}}{% endif %}
+
+path=/
+enabled=1
+gpgcheck=0
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_ubuntu.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_ubuntu.j2 b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_ubuntu.j2
new file mode 100644
index 0000000..52d4c9a
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-INSTALL/templates/repo_ubuntu.j2
@@ -0,0 +1 @@
+{{package_type}} {{base_url}} {{components}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-RESTART/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-RESTART/scripts/hook.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-RESTART/scripts/hook.py
new file mode 100644
index 0000000..14b9d99
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-RESTART/scripts/hook.py
@@ -0,0 +1,29 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+
+class BeforeRestartHook(Hook):
+
+ def hook(self, env):
+ self.run_custom_hook('before-START')
+
+if __name__ == "__main__":
+ BeforeRestartHook().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/checkForFormat.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/checkForFormat.sh b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/checkForFormat.sh
new file mode 100644
index 0000000..f92f613
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/checkForFormat.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+export hdfs_user=$1
+shift
+export conf_dir=$1
+shift
+export mark_dir=$1
+shift
+export name_dirs=$*
+
+export EXIT_CODE=0
+export command="namenode -format"
+export list_of_non_empty_dirs=""
+
+mark_file=/var/run/hadoop/hdfs/namenode-formatted
+if [[ -f ${mark_file} ]] ; then
+ rm -f ${mark_file}
+ mkdir -p ${mark_dir}
+fi
+
+if [[ ! -d $mark_dir ]] ; then
+ for dir in `echo $name_dirs | tr ',' ' '` ; do
+ echo "NameNode Dirname = $dir"
+ cmd="ls $dir | wc -l | grep -q ^0$"
+ eval $cmd
+ if [[ $? -ne 0 ]] ; then
+ (( EXIT_CODE = $EXIT_CODE + 1 ))
+ list_of_non_empty_dirs="$list_of_non_empty_dirs $dir"
+ fi
+ done
+
+ if [[ $EXIT_CODE == 0 ]] ; then
+ su - ${hdfs_user} -c "yes Y | hadoop --config ${conf_dir} ${command}"
+ else
+ echo "ERROR: Namenode directory(s) is non empty. Will not format the namenode. List of non-empty namenode dirs ${list_of_non_empty_dirs}"
+ fi
+else
+ echo "${mark_dir} exists. Namenode DFS already formatted"
+fi
+
+exit $EXIT_CODE
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/task-log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/task-log4j.properties b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/task-log4j.properties
new file mode 100644
index 0000000..7e12962
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/files/task-log4j.properties
@@ -0,0 +1,134 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+
+# Define some default values that can be overridden by system properties
+hadoop.root.logger=INFO,console
+hadoop.log.dir=.
+hadoop.log.file=hadoop.log
+
+#
+# Job Summary Appender
+#
+# Use following logger to send summary to separate file defined by
+# hadoop.mapreduce.jobsummary.log.file rolled daily:
+# hadoop.mapreduce.jobsummary.logger=INFO,JSA
+#
+hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger}
+hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log
+
+# Define the root logger to the system property "hadoop.root.logger".
+log4j.rootLogger=${hadoop.root.logger}, EventCounter
+
+# Logging Threshold
+log4j.threshhold=ALL
+
+#
+# Daily Rolling File Appender
+#
+
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+#
+# TaskLog Appender
+#
+
+#Default values
+hadoop.tasklog.taskid=null
+hadoop.tasklog.iscleanup=false
+hadoop.tasklog.noKeepSplits=4
+hadoop.tasklog.totalLogFileSize=100
+hadoop.tasklog.purgeLogSplits=true
+hadoop.tasklog.logsRetainHours=12
+
+log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender
+log4j.appender.TLA.taskId=${hadoop.tasklog.taskid}
+log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup}
+log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize}
+
+log4j.appender.TLA.layout=org.apache.log4j.PatternLayout
+log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+#
+# Rolling File Appender
+#
+
+#log4j.appender.RFA=org.apache.log4j.RollingFileAppender
+#log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file}
+
+# Logfile size and and 30-day backups
+#log4j.appender.RFA.MaxFileSize=1MB
+#log4j.appender.RFA.MaxBackupIndex=30
+
+#log4j.appender.RFA.layout=org.apache.log4j.PatternLayout
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} - %m%n
+#log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+
+# Custom Logging levels
+
+hadoop.metrics.log.level=INFO
+#log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG
+#log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+log4j.logger.org.apache.hadoop.metrics2=${hadoop.metrics.log.level}
+
+# Jets3t library
+log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR
+
+#
+# Null Appender
+# Trap security logger on the hadoop client side
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# Event Counter Appender
+# Sends counts of logging messages at different severity levels to Hadoop Metrics.
+#
+log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter
+
+# Removes "deprecated" messages
+log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/hook.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/hook.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/hook.py
new file mode 100644
index 0000000..a18a776
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/hook.py
@@ -0,0 +1,38 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+from shared_initialization import *
+
+class BeforeStartHook(Hook):
+
+ def hook(self, env):
+ import params
+
+ self.run_custom_hook('before-ANY')
+ self.run_custom_hook('after-INSTALL')
+ env.set_params(params)
+
+ setup_hadoop()
+ setup_configs()
+ create_javahome_symlink()
+
+if __name__ == "__main__":
+ BeforeStartHook().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/params.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/params.py
new file mode 100644
index 0000000..fc525a6
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/params.py
@@ -0,0 +1,146 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+from resource_management.core.system import System
+import os
+
+config = Script.get_config()
+
+#security params
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+
+#users and groups
+mapred_user = config['configurations']['mapred-env']['mapred_user']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+yarn_user = config['configurations']['yarn-env']['yarn_user']
+
+user_group = config['configurations']['cluster-env']['user_group']
+
+#hosts
+hostname = config["hostname"]
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+rm_host = default("/clusterHostInfo/rm_host", [])
+slave_hosts = default("/clusterHostInfo/slave_hosts", [])
+hagios_server_hosts = default("/clusterHostInfo/nagios_server_host", [])
+oozie_servers = default("/clusterHostInfo/oozie_server", [])
+hcat_server_hosts = default("/clusterHostInfo/webhcat_server_host", [])
+hive_server_host = default("/clusterHostInfo/hive_server_host", [])
+hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
+hs_host = default("/clusterHostInfo/hs_host", [])
+jtnode_host = default("/clusterHostInfo/jtnode_host", [])
+namenode_host = default("/clusterHostInfo/namenode_host", [])
+zk_hosts = default("/clusterHostInfo/zookeeper_hosts", [])
+ganglia_server_hosts = default("/clusterHostInfo/ganglia_server_host", [])
+
+has_namenode = not len(namenode_host) == 0
+has_resourcemanager = not len(rm_host) == 0
+has_slaves = not len(slave_hosts) == 0
+has_nagios = not len(hagios_server_hosts) == 0
+has_oozie_server = not len(oozie_servers) == 0
+has_hcat_server_host = not len(hcat_server_hosts) == 0
+has_hive_server_host = not len(hive_server_host) == 0
+has_hbase_masters = not len(hbase_master_hosts) == 0
+has_zk_host = not len(zk_hosts) == 0
+has_ganglia_server = not len(ganglia_server_hosts) == 0
+
+is_namenode_master = hostname in namenode_host
+is_jtnode_master = hostname in jtnode_host
+is_rmnode_master = hostname in rm_host
+is_hsnode_master = hostname in hs_host
+is_hbase_master = hostname in hbase_master_hosts
+is_slave = hostname in slave_hosts
+if has_ganglia_server:
+ ganglia_server_host = ganglia_server_hosts[0]
+#hadoop params
+
+if has_namenode:
+ hadoop_tmp_dir = format("/tmp/hadoop-{hdfs_user}")
+hadoop_lib_home = "/usr/lib/hadoop/lib"
+hadoop_conf_dir = "/etc/hadoop/conf"
+hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix']
+hadoop_home = "/usr"
+hadoop_bin = "/usr/lib/hadoop/sbin"
+
+task_log4j_properties_location = os.path.join(hadoop_conf_dir, "task-log4j.properties")
+
+hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix']
+hbase_tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir']
+#db params
+server_db_name = config['hostLevelParams']['db_name']
+db_driver_filename = config['hostLevelParams']['db_driver_filename']
+oracle_driver_url = config['hostLevelParams']['oracle_jdbc_url']
+mysql_driver_url = config['hostLevelParams']['mysql_jdbc_url']
+ambari_server_resources = config['hostLevelParams']['jdk_location']
+oracle_driver_symlink_url = format("{ambari_server_resources}oracle-jdbc-driver.jar")
+mysql_driver_symlink_url = format("{ambari_server_resources}mysql-jdbc-driver.jar")
+
+ambari_db_rca_url = config['hostLevelParams']['ambari_db_rca_url'][0]
+ambari_db_rca_driver = config['hostLevelParams']['ambari_db_rca_driver'][0]
+ambari_db_rca_username = config['hostLevelParams']['ambari_db_rca_username'][0]
+ambari_db_rca_password = config['hostLevelParams']['ambari_db_rca_password'][0]
+
+if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']:
+ rca_enabled = config['configurations']['hadoop-env']['rca_enabled']
+else:
+ rca_enabled = False
+rca_disabled_prefix = "###"
+if rca_enabled == True:
+ rca_prefix = ""
+else:
+ rca_prefix = rca_disabled_prefix
+
+#hadoop-env.sh
+java_home = config['hostLevelParams']['java_home']
+
+if str(config['hostLevelParams']['stack_version']).startswith('2.0') and System.get_instance().os_family != "suse":
+ # deprecated rhel jsvc_path
+ jsvc_path = "/usr/libexec/bigtop-utils"
+else:
+ jsvc_path = "/usr/lib/bigtop-utils"
+
+hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize']
+namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize']
+namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize']
+namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize']
+
+jtnode_opt_newsize = "200m"
+jtnode_opt_maxnewsize = "200m"
+jtnode_heapsize = "1024m"
+ttnode_heapsize = "1024m"
+
+dtnode_heapsize = config['configurations']['hadoop-env']['dtnode_heapsize']
+mapred_pid_dir_prefix = default("/configurations/mapred-env/mapred_pid_dir_prefix","/var/run/hadoop-mapreduce")
+mapreduce_libs_path = "/usr/lib/hadoop-mapreduce/*"
+hadoop_libexec_dir = "/usr/lib/hadoop/libexec"
+mapred_log_dir_prefix = default("/configurations/mapred-env/mapred_log_dir_prefix","/var/log/hadoop-mapreduce")
+
+#log4j.properties
+
+yarn_log_dir_prefix = default("/configurations/yarn-env/yarn_log_dir_prefix","/var/log/hadoop-yarn")
+
+dfs_hosts = default('/configurations/hdfs-site/dfs.hosts', None)
+
+#log4j.properties
+if (('hdfs-log4j' in config['configurations']) and ('content' in config['configurations']['hdfs-log4j'])):
+ log4j_props = config['configurations']['hdfs-log4j']['content']
+ if (('yarn-log4j' in config['configurations']) and ('content' in config['configurations']['yarn-log4j'])):
+ log4j_props += config['configurations']['yarn-log4j']['content']
+else:
+ log4j_props = None
http://git-wip-us.apache.org/repos/asf/ambari/blob/83efcfea/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/shared_initialization.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/shared_initialization.py b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/shared_initialization.py
new file mode 100644
index 0000000..0d00aca
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/BIGTOP/0.8/hooks/before-START/scripts/shared_initialization.py
@@ -0,0 +1,175 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management import *
+
+def setup_hadoop():
+ """
+ Setup hadoop files and directories
+ """
+ import params
+
+ Execute("/bin/echo 0 > /selinux/enforce",
+ only_if="test -f /selinux/enforce"
+ )
+
+ install_snappy()
+
+ #directories
+ if params.has_namenode:
+ Directory(params.hdfs_log_dir_prefix,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+ Directory(params.hadoop_pid_dir_prefix,
+ recursive=True,
+ owner='root',
+ group='root'
+ )
+ #this doesn't needed with stack 1
+ Directory(params.hadoop_tmp_dir,
+ recursive=True,
+ owner=params.hdfs_user,
+ )
+ #files
+ if params.security_enabled:
+ tc_owner = "root"
+ else:
+ tc_owner = params.hdfs_user
+
+ File(os.path.join(params.hadoop_conf_dir, 'commons-logging.properties'),
+ owner=tc_owner,
+ content=Template('commons-logging.properties.j2')
+ )
+
+ health_check_template = "health_check-v2" #for stack 1 use 'health_check'
+ File(os.path.join(params.hadoop_conf_dir, "health_check"),
+ owner=tc_owner,
+ content=Template(health_check_template + ".j2")
+ )
+
+ log4j_filename = os.path.join(params.hadoop_conf_dir, "log4j.properties")
+ if (params.log4j_props != None):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.hadoop_conf_dir}/log4j.properties"))):
+ File(log4j_filename,
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ )
+
+ File(os.path.join(params.hadoop_conf_dir, "hadoop-metrics2.properties"),
+ owner=params.hdfs_user,
+ content=Template("hadoop-metrics2.properties.j2")
+ )
+
+def setup_database():
+ """
+ Load DB
+ """
+ import params
+ db_driver_dload_cmd = ""
+ environment = {
+ "no_proxy": format("{ambari_server_hostname}")
+ }
+ if params.server_db_name == 'oracle' and params.oracle_driver_url != "":
+ db_driver_dload_cmd = format(
+ "curl -kf -x \"\" \
+ --retry 5 {oracle_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}",)
+ elif params.server_db_name == 'mysql' and params.mysql_driver_url != "":
+ db_driver_dload_cmd = format(
+ "curl -kf -x \"\" \
+ --retry 5 {mysql_driver_symlink_url} -o {hadoop_lib_home}/{db_driver_filename}")
+
+ if db_driver_dload_cmd:
+ Execute(db_driver_dload_cmd,
+ not_if =format("test -e {hadoop_lib_home}/{db_driver_filename}"),
+ environment = environment
+ )
+
+
+def setup_configs():
+ """
+ Creates configs for services HDFS mapred
+ """
+ import params
+
+ if params.has_namenode:
+ File(params.task_log4j_properties_location,
+ content=StaticFile("task-log4j.properties"),
+ mode=0755
+ )
+
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'configuration.xsl')):
+ File(os.path.join(params.hadoop_conf_dir, 'configuration.xsl'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+ if os.path.exists(os.path.join(params.hadoop_conf_dir, 'masters')):
+ File(os.path.join(params.hadoop_conf_dir, 'masters'),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+ generate_include_file()
+
+
+def generate_include_file():
+ import params
+
+ if params.has_namenode and params.dfs_hosts and params.has_slaves:
+ include_hosts_list = params.slave_hosts
+ File(params.dfs_hosts,
+ content=Template("include_hosts_list.j2"),
+ owner=params.hdfs_user,
+ group=params.user_group
+ )
+
+
+def install_snappy():
+ import params
+
+ snappy_so = "libsnappy.so"
+ so_target_dir_x86 = format("{hadoop_lib_home}/native/Linux-i386-32")
+ so_target_dir_x64 = format("{hadoop_lib_home}/native/Linux-amd64-64")
+ so_target_x86 = format("{so_target_dir_x86}/{snappy_so}")
+ so_target_x64 = format("{so_target_dir_x64}/{snappy_so}")
+ so_src_dir_x86 = format("{hadoop_home}/lib")
+ so_src_dir_x64 = format("{hadoop_home}/lib64")
+ so_src_x86 = format("{so_src_dir_x86}/{snappy_so}")
+ so_src_x64 = format("{so_src_dir_x64}/{snappy_so}")
+ Execute(
+ format("mkdir -p {so_target_dir_x86}; ln -sf {so_src_x86} {so_target_x86}"))
+ Execute(
+ format("mkdir -p {so_target_dir_x64}; ln -sf {so_src_x64} {so_target_x64}"))
+
+
+def create_javahome_symlink():
+ if os.path.exists("/usr/jdk/jdk1.6.0_31") and not os.path.exists("/usr/jdk64/jdk1.6.0_31"):
+ Execute("mkdir -p /usr/jdk64/")
+ Execute("ln -s /usr/jdk/jdk1.6.0_31 /usr/jdk64/jdk1.6.0_31")
+