You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ja...@apache.org on 2015/11/06 22:50:00 UTC
[2/3] ambari git commit: AMBARI-13725: HAWQ and PXF to support 3
digit versioning instead of 4. (jaoki)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
new file mode 100644
index 0000000..41b10dc
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-site.xml
@@ -0,0 +1,178 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>hawq_master_address_host</name>
+ <display-name>HAWQ Master</display-name>
+ <value>localhost</value>
+ <description>The host name of HAWQ master.</description>
+ </property>
+
+ <property>
+ <name>hawq_standby_address_host</name>
+ <display-name>HAWQ Standby Master</display-name>
+ <value>localhost</value>
+ <description>The host name of HAWQ standby.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_address_port</name>
+ <display-name>HAWQ Master Port</display-name>
+ <value>5432</value>
+ <description>The port of HAWQ master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_address_port</name>
+ <display-name>HAWQ Segment Port</display-name>
+ <value>40000</value>
+ <description>The port of HAWQ segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_dfs_url</name>
+ <display-name>HAWQ DFS URL</display-name>
+ <value>localhost:8020/hawq_default</value>
+ <description>URL for accessing HDFS.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_directory</name>
+ <display-name>HAWQ Master Directory</display-name>
+ <value>/data/hawq/master</value>
+ <description>The directory of HAWQ master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_directory</name>
+ <display-name>HAWQ Segment Directory</display-name>
+ <value>/data/hawq/segment</value>
+ <description>The directory of HAWQ segment.</description>
+ </property>
+
+ <property>
+ <name>hawq_master_temp_directory</name>
+ <display-name>HAWQ Master Temp Directory</display-name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for HAWQ master.</description>
+ </property>
+
+ <property>
+ <name>hawq_segment_temp_directory</name>
+ <display-name>HAWQ Segment Temp Directory</display-name>
+ <value>/tmp</value>
+ <description>The temporary directory reserved for HAWQ segment.</description>
+ </property>
+
+ <!-- HAWQ resource manager parameters -->
+ <property>
+ <name>hawq_global_rm_type</name>
+ <value>none</value>
+ <description>The resource manager type to start for allocating resource.
+ 'none' means HAWQ resource manager exclusively uses whole
+ cluster; 'yarn' means HAWQ resource manager contacts YARN
+ resource manager to negotiate resource.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_memory_limit_perseg</name>
+ <value>64GB</value>
+ <description>The limit of memory usage in a HAWQ segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_nvcore_limit_perseg</name>
+ <value>16</value>
+ <description>The limit of virtual core usage in a HAWQ segment when
+ hawq_global_rm_type is set 'none'.
+ </description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_address</name>
+ <value>localhost:8032</value>
+ <description>The address of YARN resource manager server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_scheduler_address</name>
+ <value>localhost:8030</value>
+ <description>The address of YARN scheduler server.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_queue_name</name>
+ <value>default</value>
+ <description>The YARN queue name to register HAWQ resource manager.</description>
+ </property>
+
+ <property>
+ <name>hawq_rm_yarn_app_name</name>
+ <value>hawq</value>
+ <description>The application name to register HAWQ resource manager in YARN.</description>
+ </property>
+
+ <property>
+ <name>hawq_re_cpu_enable</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_mount_point</name>
+ <value>/sys/fs/cgroup</value>
+ </property>
+
+ <property>
+ <name>hawq_re_cgroup_hierarchy_name</name>
+ <value>hadoop-yarn</value>
+ </property>
+
+ <property>
+ <name>hawq_re_cleanup_period</name>
+ <value>180</value>
+ </property>
+
+ <property>
+ <name>hawq_re_cpu_weight</name>
+ <value>1024.0</value>
+ </property>
+
+ <property>
+ <name>hawq_re_vcore_pcore_ratio</name>
+ <value>1.0</value>
+ </property>
+
+ <property>
+ <name>hawq_resourcemanager_master_address_domainsocket_port</name>
+ <value>5436</value>
+ </property>
+
+ <property>
+ <name>hawq_rm_master_port</name>
+ <value>5437</value>
+ </property>
+
+ <property>
+ <name>hawq_rm_segment_port</name>
+ <value>5438</value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
new file mode 100644
index 0000000..32ae5a5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-sysctl-env.xml
@@ -0,0 +1,247 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>kernel.shmmax</name>
+ <value>500000000</value>
+ <description>Maximum size in bytes of a single shared memory segment that a Linux process can allocate in its
+ virtual address space</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.shmmni</name>
+ <value>4096</value>
+ <description>System wide maximum number of shared memory segments</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.shmall</name>
+ <value>4000000000</value>
+ <description>Total amount of shared memory pages that can be used system wide</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.sem</name>
+ <value>250 512000 100 2048</value>
+ <description>Parameter to define semaphore related values</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.sysrq</name>
+ <value>1</value>
+ <description>Enable(1)/Disable(0) functions of sysrq</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.core_uses_pid</name>
+ <value>1</value>
+ <description>Enable appending process id to the name of core dump file. Ex: core.PID</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.msgmnb</name>
+ <value>65536</value>
+ <description>Default maximum size of a message in bytes</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.msgmax</name>
+ <value>65536</value>
+ <description>Default maxmimum size of a mesage queue</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.msgmni</name>
+ <value>2048</value>
+ <description>Number of message queue identifiers</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.tcp_syncookies</name>
+ <value>0</value>
+ <description>Enable(1)/Disable(0) SYN cookie protection</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.ip_forward</name>
+ <value>0</value>
+ <description>Enable(1)/Disable(0) IP forwarding</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.conf.default.accept_source_route</name>
+ <value>0</value>
+ <description>Enable(1)/Disable(0) IP source routing</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.tcp_tw_recycle</name>
+ <value>1</value>
+ <description>Enable(1)/Disable(0) fast recycling of TIME_WAIT sockets</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.tcp_max_syn_backlog</name>
+ <value>200000</value>
+ <description>Number of outstanding syn requests allowed</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.conf.all.arp_filter</name>
+ <value>1</value>
+ <description>Enable(1)/Disable(0) route lookup to determine the interface through which to send the reply</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.ipv4.ip_local_port_range</name>
+ <value>1281 65535</value>
+ <description>Local port range that is used by TCP and UDP traffic to choose the local port</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.core.netdev_max_backlog</name>
+ <value>200000</value>
+ <description>Maximum number of packets allowed to queue when a particular interface receives packets faster than the kernel can process them</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>fs.nr_open</name>
+ <value>3000000</value>
+ <description>Maximum number of file-handles a process can allocate</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.threads-max</name>
+ <value>798720</value>
+ <description>System-wide maximum number of threads to be used by the kernel at one time</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>kernel.pid_max</name>
+ <value>798720</value>
+ <description>Maximum number of unique process identifiers</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.core.rmem_max</name>
+ <value>2097152</value>
+ <description>Maximum receive socket buffer size in bytes</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>net.core.wmem_max</name>
+ <value>2097152</value>
+ <description>Maximum send socket buffer size in bytes</description>
+ <value-attributes>
+ <overridable>false</overridable>
+ <editable-only-at-install>true</editable-only-at-install>
+ </value-attributes>
+ </property>
+
+ <property>
+ <name>vm.overcommit_memory</name>
+ <value>1</value>
+ <description>Defines the conditions that determine whether a large memory request is accepted or denied. There are
+ three possible values for this parameter: 0, 1 or 2. For production environment, value of 2 is recommended</description>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
new file mode 100644
index 0000000..2ceff96
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/metainfo.xml
@@ -0,0 +1,129 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>HAWQ</name>
+ <displayName>HAWQ</displayName>
+ <comment>Apache HAWQ - Apache Hadoop Native SQL</comment>
+ <version>2.0</version>
+ <components>
+
+ <component>
+ <name>HAWQMASTER</name>
+ <displayName>HAWQ Master</displayName>
+ <category>MASTER</category>
+ <cardinality>1</cardinality>
+ <commandScript>
+ <script>scripts/hawqmaster.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <dependencies>
+ <dependency>
+ <name>HDFS/NAMENODE</name>
+ <scope>cluster</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+
+ <component>
+ <name>HAWQSTANDBY</name>
+ <displayName>HAWQ Standby Master</displayName>
+ <category>MASTER</category>
+ <cardinality>0-1</cardinality>
+ <commandScript>
+ <script>scripts/hawqstandby.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+
+ <component>
+ <name>HAWQSEGMENT</name>
+ <displayName>HAWQ Segment</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/hawqsegment.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <dependencies>
+ <dependency>
+ <name>HDFS/DATANODE</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>false</enabled>
+ <co-locate>HDFS/DATANODE</co-locate>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ </component>
+ </components>
+ <requiredServices>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>hawq</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <configuration-dependencies>
+ <config-type>hawq-site</config-type>
+ <config-type>gpcheck-env</config-type>
+ </configuration-dependencies>
+ </service>
+
+ </services>
+</metainfo>
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
new file mode 100644
index 0000000..31ef2fa
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
@@ -0,0 +1,283 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import re
+import os
+import time
+import crypt
+import filecmp
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.resources.system import Execute, Directory, File
+from resource_management.core.logger import Logger
+from resource_management.core.system import System
+from resource_management.core.exceptions import Fail
+from resource_management.core.resources.accounts import Group, User
+from resource_management.core.source import Template
+import xml.etree.ElementTree as ET
+
+import utils
+import constants
+
+
+def update_bashrc(source_file, target_file):
+ """
+ Updates the hawq_user's .bashrc file with HAWQ env variables like
+ MASTER_DATA_DIRECTORY, PGHOST, PGPORT and PGUSER.
+ And sources the greenplum_path file.
+ """
+ append_src_cmd = "echo 'source {0}' >> {1}".format(source_file, target_file)
+ src_cmd_exists = "grep 'source {0}' {1}".format(source_file, target_file)
+ Execute(append_src_cmd, user=constants.hawq_user, timeout=constants.default_exec_timeout, not_if=src_cmd_exists)
+
+
+def setup_user():
+ """
+ Creates HAWQ user home directory and sets up the correct ownership.
+ """
+ __create_hawq_user()
+ __set_home_dir_ownership()
+
+
+def __create_hawq_user():
+ """
+ Creates HAWQ user with default password and group.
+ """
+ import params
+ Group(constants.hawq_group, ignore_failures=True)
+
+ User(constants.hawq_user,
+ gid=constants.hawq_group,
+ password=crypt.crypt(constants.hawq_password, "salt"),
+ groups=[constants.hawq_group, params.user_group],
+ ignore_failures=True)
+
+
+def __set_home_dir_ownership():
+ """
+ Updates the HAWQ user home directory to be owned by gpadmin:gpadmin.
+ """
+ command = "chown -R {0}:{1} {2}".format(constants.hawq_user, constants.hawq_group, constants.hawq_home_dir)
+ Execute(command, timeout=constants.default_exec_timeout)
+
+
+def setup_common_configurations():
+ """
+ Sets up the config files common to master, standby and segment nodes.
+ """
+ import params
+
+ substituted_conf_dict = __substitute_hostnames_in_hawq_site()
+ XmlConfig("hawq-site.xml",
+ conf_dir=constants.hawq_config_dir,
+ configurations=substituted_conf_dict,
+ configuration_attributes=params.config['configuration_attributes']['hawq-site'],
+ owner=constants.hawq_user,
+ group=constants.hawq_group,
+ mode=0644)
+ __set_osparams()
+
+
+def __substitute_hostnames_in_hawq_site():
+ """
+ Temporary function to replace localhost with actual HAWQ component hostnames.
+ This function will be in place till the entire HAWQ plugin code along with the UI
+ changes are submitted to the trunk.
+ """
+ import params
+
+ LOCALHOST = "localhost"
+
+ # in case there is no standby
+ hawqstandby_host_desired_value = params.hawqstandby_host if params.hawqstandby_host is not None else 'none'
+
+ substituted_hawq_site = params.hawq_site.copy()
+ hawq_site_property_map = {"hawq_master_address_host": params.hawqmaster_host,
+ "hawq_standby_address_host": hawqstandby_host_desired_value,
+ "hawq_rm_yarn_address": params.rm_host,
+ "hawq_rm_yarn_scheduler_address": params.rm_host,
+ "hawq_dfs_url": params.namenode_host
+ }
+
+ for property, desired_value in hawq_site_property_map.iteritems():
+ if desired_value is not None:
+ # Replace localhost with required component hostname
+ substituted_hawq_site[property] = re.sub(LOCALHOST, desired_value, substituted_hawq_site[property])
+
+ return substituted_hawq_site
+
+
+def __set_osparams():
+ """
+ Updates parameters in sysctl.conf and limits.conf required by HAWQ.
+ """
+ # Create a temp scratchpad directory
+ utils.create_dir_as_hawq_user(constants.hawq_tmp_dir)
+
+ # Suse doesn't supports loading values from files in /etc/sysctl.d
+ # So we will have to directly edit the sysctl file
+ if System.get_instance().os_family == "suse":
+ # Update /etc/sysctl.conf
+ __update_sysctl_file_suse()
+ else:
+ # Update /etc/sysctl.d/hawq.conf
+ __update_sysctl_file()
+
+ __update_limits_file()
+
+
+def __update_limits_file():
+ """
+ Updates /etc/security/limits.d/hawq.conf file with the HAWQ parameters.
+ """
+ import params
+ # Ensure limits directory exists
+ Directory(constants.limits_conf_dir, recursive=True, owner=constants.root_user, group=constants.root_user)
+
+ # Generate limits for hawq user
+ limits_file_content = "#### HAWQ Limits Parameters ###########\n"
+ for key, value in params.hawq_limits.iteritems():
+ if not __valid_input(value):
+ raise Exception("Value {0} for parameter {1} contains non-numeric characters which are not allowed (except whitespace), please fix the value and retry".format(value, key))
+ """
+ Content of the file to be written should be of the format
+ gpadmin soft nofile 290000
+ gpadmin hard nofile 290000
+ key used in the configuration is of the format soft_nofile, thus strip '_' and replace with 'space'
+ """
+ limits_file_content += "{0} {1} {2}\n".format(constants.hawq_user, re.sub("_", " ", key), value.strip())
+ File('{0}/{1}.conf'.format(constants.limits_conf_dir, constants.hawq_user), content=limits_file_content,
+ owner=constants.hawq_user, group=constants.hawq_group)
+
+
+def __valid_input(value):
+ """
+ Validate if input value contains number (whitespaces allowed), return true if found else false
+ """
+ return re.search("^ *[0-9][0-9 ]*$", value)
+
+
+def __convert_sysctl_dict_to_text():
+ """
+ Convert sysctl configuration dict to text with each property value pair separated on new line
+ """
+ import params
+ sysctl_file_content = "### HAWQ System Parameters ###########\n"
+ for key, value in params.hawq_sysctl.iteritems():
+ if not __valid_input(value):
+ raise Exception("Value {0} for parameter {1} contains non-numeric characters which are not allowed (except whitespace), please fix the value and retry".format(value, key))
+ sysctl_file_content += "{0} = {1}\n".format(key, value)
+ return sysctl_file_content
+
+
+def __update_sysctl_file():
+ """
+ Updates /etc/sysctl.d/hawq_sysctl.conf file with the HAWQ parameters on CentOS/RHEL.
+ """
+ # Ensure sys ctl sub-directory exists
+ Directory(constants.sysctl_conf_dir, recursive=True, owner=constants.root_user, group=constants.root_user)
+
+ # Generate temporary file with kernel parameters needed by hawq
+ File(constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=constants.hawq_user,
+ group=constants.hawq_group)
+
+ is_changed = True
+ if os.path.exists(constants.hawq_sysctl_tmp_file) and os.path.exists(constants.hawq_sysctl_file):
+ is_changed = not filecmp.cmp(constants.hawq_sysctl_file, constants.hawq_sysctl_tmp_file)
+
+ if is_changed:
+ # Generate file with kernel parameters needed by hawq, only if something
+ # has been changed by user
+ Execute("cp -p {0} {1}".format(constants.hawq_sysctl_tmp_file, constants.hawq_sysctl_file))
+
+ # Reload kernel sysctl parameters from hawq file.
+ Execute("sysctl -e -p {0}".format(constants.hawq_sysctl_file), timeout=constants.default_exec_timeout)
+
+ # Wipe out temp file
+ File(constants.hawq_sysctl_tmp_file, action='delete')
+
+
+def __update_sysctl_file_suse():
+ """
+ Updates /etc/sysctl.conf file with the HAWQ parameters on SUSE.
+ """
+ # Backup file
+ backup_file_name = constants.sysctl_backup_file.format(str(int(time.time())))
+ try:
+ # Generate file with kernel parameters needed by hawq to temp file
+ File(constants.hawq_sysctl_tmp_file, content=__convert_sysctl_dict_to_text(), owner=constants.hawq_user,
+ group=constants.hawq_group)
+
+ sysctl_file_dict = utils.read_file_to_dict(constants.sysctl_suse_file)
+ sysctl_file_dict_original = sysctl_file_dict.copy()
+ hawq_sysctl_dict = utils.read_file_to_dict(constants.hawq_sysctl_tmp_file)
+
+ # Merge common system file with hawq specific file
+ sysctl_file_dict.update(hawq_sysctl_dict)
+
+ if sysctl_file_dict_original != sysctl_file_dict:
+ # Backup file
+ Execute("cp {0} {1}".format(constants.sysctl_suse_file, backup_file_name), timeout=constants.default_exec_timeout)
+ # Write merged properties to file
+ utils.write_dict_to_file(sysctl_file_dict, constants.sysctl_suse_file)
+ # Reload kernel sysctl parameters from /etc/sysctl.conf
+ Execute("sysctl -e -p", timeout=constants.default_exec_timeout)
+
+ except Exception as e:
+ Logger.error("Error occurred while updating sysctl.conf file, reverting the contents" + str(e))
+ Execute("cp {0} {1}".format(constants.sysctl_suse_file, constants.hawq_sysctl_tmp_file))
+ Execute("mv {0} {1}".format(backup_file_name, constants.sysctl_suse_file), timeout=constants.default_exec_timeout)
+ Logger.error("Please execute `sysctl -e -p` on the command line manually to reload the contents of file {0}".format(
+ constants.hawq_sysctl_tmp_file))
+ raise Fail("Failed to update sysctl.conf file ")
+
+
+def get_local_hawq_site_property(property_name):
+ """
+ Fetches the value of the property specified, from the local hawq-site.xml.
+ """
+ hawq_site_path = None
+ try:
+ hawq_site_path = os.path.join(constants.hawq_config_dir, "hawq-site.xml")
+ hawq_site_root = ET.parse(hawq_site_path).getroot()
+ for property in hawq_site_root.findall("property"):
+ for item in property:
+ if item.tag == 'name':
+ current_property_name = item.text.strip() if item and item.text else item.text
+ elif item.tag == 'value':
+ current_property_value = item.text.strip() if item and item.text else item.text
+ if property_name == current_property_name:
+ return current_property_value
+ raise #If property has not been found
+ except Exception:
+ raise Fail("Unable to read property {0} from local {1}".format(property_name, hawq_site_path))
+
+def validate_configuration():
+ """
+ Validates if YARN is present in the configuration when the user specifies YARN as HAWQ's resource manager.
+ """
+ import params
+
+ # At this point, hawq should be included.
+ if 'hawq-site' not in params.config['configurations']:
+ raise Fail("Configurations does not contain hawq-site. Please include HAWQ")
+
+ # If HAWQ is set to use YARN and YARN is not configured, error.
+ rm_type = params.config["configurations"]["hawq-site"].get("hawq_global_rm_type")
+ if rm_type == "yarn" and "yarn-site" not in params.config["configurations"]:
+ raise Fail("HAWQ is set to use YARN but YARN is not deployed. " +
+ "hawq_global_rm_type property in hawq-site is set to 'yarn' but YARN is not configured. " +
+ "Please deploy YARN before starting HAWQ or change the value of hawq_global_rm_type property to 'none'")
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
new file mode 100644
index 0000000..b56fd59
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
@@ -0,0 +1,61 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+
+MASTER = "master"
+STANDBY = "standby"
+SEGMENT = "segment"
+START = "start"
+INIT = "init"
+STOP = "stop"
+
+# Users
+root_user = "root"
+hawq_user = "gpadmin"
+hawq_group = hawq_user
+hawq_password = hawq_user
+
+# Directories
+hawq_home_dir = "/usr/local/hawq/"
+hawq_config_dir = "/usr/local/hawq/etc/"
+hawq_pid_dir = "/var/run/hawq/"
+hawq_tmp_dir = '/tmp/hawq/'
+hawq_user_home_dir = os.path.expanduser("~{0}".format(hawq_user))
+limits_conf_dir = "/etc/security/limits.d"
+sysctl_conf_dir = "/etc/sysctl.d"
+
+# Files
+hawq_slaves_file = os.path.join(hawq_config_dir, "slaves")
+hawq_user_bashrc_file = os.path.join(hawq_user_home_dir, ".bashrc")
+hawq_greenplum_path_file = os.path.join(hawq_home_dir, "greenplum_path.sh")
+hawq_hosts_file = "/tmp/hawq_hosts"
+hawq_check_file = os.path.join(hawq_config_dir, "gpcheck.cnf")
+sysctl_suse_file = "/etc/sysctl.conf"
+sysctl_backup_file = "/etc/sysctl.conf.backup.{0}"
+hawq_sysctl_filename = "hawq_sysctl.conf"
+hawq_sysctl_tmp_file = os.path.join(hawq_tmp_dir, hawq_sysctl_filename)
+hawq_sysctl_file = os.path.join(sysctl_conf_dir, hawq_sysctl_filename)
+postmaster_opts_filename = "postmaster.opts"
+postmaster_pid_filename = "postmaster.pid"
+
+# Smoke check table
+smoke_check_table_name = "ambari_hawq_smoke_test"
+
+# Timeouts
+default_exec_timeout = 600
+hawq_operation_exec_timeout = 900
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
new file mode 100644
index 0000000..4443264
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
@@ -0,0 +1,55 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from resource_management import Script
+from resource_management.libraries.functions.check_process_status import check_process_status
+
+import master_helper
+import common
+import constants
+
+class HawqMaster(Script):
+ """
+ Contains the interface definitions for methods like install,
+ start, stop, status, etc. for the HAWQ Master
+ """
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ env.set_params(constants)
+ master_helper.configure_master()
+
+ def start(self, env):
+ self.configure(env)
+ common.validate_configuration()
+ master_helper.start_master()
+
+ def stop(self, env):
+ master_helper.stop_master()
+
+ def status(self, env):
+ from hawqstatus import get_pid_file
+ check_process_status(get_pid_file())
+
+if __name__ == "__main__":
+ HawqMaster().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py
new file mode 100644
index 0000000..b4be502
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqsegment.py
@@ -0,0 +1,102 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+from resource_management import Script
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.check_process_status import check_process_status
+
+import utils
+import common
+import constants
+
+class HawqSegment(Script):
+ """
+ Contains the interface definitions for methods like install,
+ start, stop, status, etc. for the HAWQ Segment
+ """
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+
+ env.set_params(params)
+ env.set_params(constants)
+ common.setup_user()
+ common.setup_common_configurations()
+ common.update_bashrc(constants.hawq_greenplum_path_file, constants.hawq_user_bashrc_file)
+
+
+ def __start_segment(self):
+ import params
+ return utils.exec_hawq_operation(
+ constants.START,
+ "{0} -a".format(constants.SEGMENT),
+ not_if=utils.chk_postgres_status_cmd(params.hawq_segment_address_port))
+
+ def start(self, env):
+ self.configure(env)
+ common.validate_configuration()
+
+ if self.__is_segment_initialized():
+ self.__start_segment()
+ return
+
+ # Initialization also starts process.
+ self.__init_segment()
+
+
+ def stop(self, env):
+ import params
+
+ utils.exec_hawq_operation(constants.STOP, "{0} -a".format(constants.SEGMENT), only_if=utils.chk_postgres_status_cmd(
+ params.hawq_segment_address_port))
+
+
+ def status(self, env):
+ from hawqstatus import get_pid_file
+ check_process_status(get_pid_file())
+
+
+ @staticmethod
+ def __init_segment():
+ import params
+
+ # Create segment directories
+ utils.create_dir_as_hawq_user(params.hawq_segment_dir)
+ utils.create_dir_as_hawq_user(params.hawq_segment_temp_dir.split(','))
+
+ Execute("chown {0}:{1} {2}".format(constants.hawq_user, constants.hawq_group, os.path.dirname(params.hawq_segment_dir)),
+ user=constants.root_user, timeout=constants.default_exec_timeout)
+
+ # Initialize hawq segment
+ utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.SEGMENT))
+
+ def __is_segment_initialized(self):
+ """
+ Check whether the HAWQ Segment is initialized
+ """
+ import params
+ return os.path.exists(os.path.join(params.hawq_segment_dir, constants.postmaster_opts_filename))
+
+
+if __name__ == "__main__":
+ HawqSegment().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py
new file mode 100644
index 0000000..d8254f9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstandby.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+from resource_management import Script
+from resource_management.libraries.functions.check_process_status import check_process_status
+
+import master_helper
+import common
+import constants
+
+class HawqStandby(Script):
+ """
+ Contains the interface definitions for methods like install,
+ start, stop, status, etc. for the HAWQ Standby Master
+ """
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ env.set_params(constants)
+ master_helper.configure_master()
+
+ def start(self, env):
+ self.configure(env)
+ common.validate_configuration()
+ master_helper.start_master()
+
+ def stop(self, env):
+ master_helper.stop_master()
+
+ def status(self, env):
+ from hawqstatus import get_pid_file
+ check_process_status(get_pid_file())
+
+ def activatestandby(self, env):
+ pass
+
+if __name__ == "__main__":
+ HawqStandby().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py
new file mode 100644
index 0000000..26dfdd0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqstatus.py
@@ -0,0 +1,64 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import os
+
+from resource_management import Script
+from resource_management.core.resources.system import File
+from resource_management.core.exceptions import Fail
+
+import utils
+import common
+import constants
+
+
+def get_pid_file():
+ """
+ Fetches the pid file, which will be used to get the status of the HAWQ Master, Standby
+ or Segments
+ """
+
+ config = Script.get_config()
+
+ component_name = config['componentName']
+ component = "master" if component_name in ["HAWQMASTER", "HAWQSTANDBY"] else "segment"
+ hawq_pid_file = os.path.join(constants.hawq_pid_dir, "hawq-{0}.pid".format(component))
+
+ File(hawq_pid_file, action='delete')
+ utils.create_dir_as_hawq_user(constants.hawq_pid_dir)
+
+ #Get hawq_master_directory or hawq_segment_directory value from hawq-site.xml depending
+ #on the component
+ hawq_site_directory_property = "hawq_{0}_directory".format(component)
+
+ #hawq-site content from Ambari server will not be available when the
+ #command type is STATUS_COMMAND. Hence, reading it directly from the local file
+ postmaster_pid_file = os.path.join(common.get_local_hawq_site_property(
+ hawq_site_directory_property), constants.postmaster_pid_filename)
+
+ pid = ""
+ if os.path.exists(postmaster_pid_file):
+ with open(postmaster_pid_file, 'r') as fh:
+ pid = fh.readline().strip()
+
+ if not pid:
+ raise Fail("Failed to fetch pid from {0}".format(postmaster_pid_file))
+
+ File(hawq_pid_file, content=pid, owner=constants.hawq_user, group=constants.hawq_user)
+
+ return hawq_pid_file
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
new file mode 100644
index 0000000..35f5112
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
@@ -0,0 +1,194 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import os
+from resource_management.core.resources.system import File, Execute
+from resource_management.core.source import Template
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+
+import utils
+import common
+import constants
+
+def __setup_master_specific_conf_files():
+ """
+ Sets up config files only applicable for HAWQ Master and Standby nodes
+ """
+ import params
+
+ File(constants.hawq_check_file, content=params.gpcheck_content, owner=constants.hawq_user, group=constants.hawq_group,
+ mode=0644)
+
+ File(constants.hawq_slaves_file, content=Template("slaves.j2"), owner=constants.hawq_user, group=constants.hawq_group,
+ mode=0644)
+
+ File(constants.hawq_hosts_file, content=Template("hawq-hosts.j2"), owner=constants.hawq_user, group=constants.hawq_group,
+ mode=0644)
+
+
+def __setup_passwordless_ssh():
+ """
+ Exchanges ssh keys to setup passwordless ssh for the hawq_user between the HAWQ Master and the HAWQ Segment nodes
+ """
+ utils.exec_hawq_operation("ssh-exkeys", "-f {0} -p {1}".format(constants.hawq_hosts_file, constants.hawq_password))
+
+ File(constants.hawq_hosts_file, action='delete')
+
+
+def __setup_hawq_user_profile():
+ """
+ Sets up the ENV variables for hawq_user as a convenience for the command line users
+ """
+ hawq_profile_file = os.path.join(os.path.expanduser("~{0}".format(constants.hawq_user)), ".hawq-profile.sh")
+ File(hawq_profile_file, content=Template("hawq-profile.sh.j2"), owner=constants.hawq_user, group=constants.hawq_group)
+ common.update_bashrc(hawq_profile_file, constants.hawq_user_bashrc_file)
+
+
+def configure_master():
+ """
+ Configures the master node after rpm install
+ """
+ common.setup_user()
+ common.setup_common_configurations()
+ __setup_master_specific_conf_files()
+ __setup_passwordless_ssh()
+ __setup_hawq_user_profile()
+ __create_local_dirs()
+
+
+def __create_local_dirs():
+ """
+ Creates the required local directories for HAWQ
+ """
+ import params
+ # Create Master directories
+ utils.create_dir_as_hawq_user(params.hawq_master_dir)
+ utils.create_dir_as_hawq_user(params.hawq_master_temp_dir.split(','))
+
+ Execute("chown {0}:{1} {2}".format(constants.hawq_user, constants.hawq_group, os.path.dirname(params.hawq_master_dir)),
+ user=constants.root_user, timeout=constants.default_exec_timeout)
+
+ Execute("chmod 700 {0}".format(params.hawq_master_dir), user=constants.root_user, timeout=constants.default_exec_timeout)
+
+
+def __create_hdfs_dirs():
+ """
+ Creates the required HDFS directories for HAWQ
+ """
+ import params
+ params.HdfsResource(params.hawq_hdfs_data_dir, type="directory", action="create_on_execute", owner=constants.hawq_user, group=constants.hawq_group, mode=0755)
+ params.HdfsResource(None, action="execute")
+
+
+def __init_active():
+ """
+ Initializes the active master
+ """
+ __create_hdfs_dirs()
+ utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.MASTER))
+
+
+def __init_standby():
+ """
+ Initializes the HAWQ Standby Master
+ """
+ utils.exec_hawq_operation(constants.INIT, "{0} -a -v".format(constants.STANDBY))
+
+
+def __get_component_name():
+ """
+ Identifies current node as either HAWQ Master or HAWQ Standby Master
+ """
+ return constants.MASTER if __is_active_master() else constants.STANDBY
+
+
+def __start_local_master():
+ """
+ Starts HAWQ Master or HAWQ Standby Master component on the host
+ """
+ import params
+ utils.exec_hawq_operation(
+ constants.START,
+ "{0} -a".format(__get_component_name()),
+ not_if=utils.chk_postgres_status_cmd(params.hawq_master_address_port))
+
+
+def __is_local_initialized():
+ """
+ Checks if the local node has been initialized
+ """
+ import params
+ return os.path.exists(os.path.join(params.hawq_master_dir, constants.postmaster_opts_filename))
+
+
+def __get_standby_host():
+ """
+ Returns the name of the HAWQ Standby Master host from hawq-site.xml, or None if no standby is configured
+ """
+ standby_host = common.get_local_hawq_site_property("hawq_standby_address_host")
+ return None if standby_host is None or standby_host.lower() == 'none' else standby_host
+
+
+def __is_standby_initialized():
+ """
+ Returns True if HAWQ Standby Master is initialized, False otherwise
+ """
+ import params
+
+ file_path = os.path.join(params.hawq_master_dir, constants.postmaster_opts_filename)
+ (retcode, _, _) = utils.exec_ssh_cmd(__get_standby_host(), "[ -f {0} ]".format(file_path))
+ return retcode == 0
+
+
+def start_master():
+ """
+ Initializes HAWQ Master/Standby if not already done and starts them
+ """
+ import params
+
+ if not params.hostname in [params.hawqmaster_host, params.hawqstandby_host]:
+ Fail("Host should be either active Hawq master or Hawq standby.")
+
+ is_active_master = __is_active_master()
+ if __is_local_initialized():
+ __start_local_master()
+
+ elif is_active_master:
+ __init_active()
+
+ if is_active_master and __get_standby_host() is not None and not __is_standby_initialized():
+ __init_standby()
+
+
+def stop_master():
+ """
+ Stops the HAWQ Master/Standby
+ """
+ import params
+ utils.exec_hawq_operation(
+ constants.STOP,
+ "{0} -a".format(__get_component_name()),
+ only_if=utils.chk_postgres_status_cmd(params.hawq_master_address_port))
+
+
+def __is_active_master():
+ """
+ Finds if this node is the active master
+ """
+ import params
+ return params.hostname == common.get_local_hawq_site_property("hawq_master_address_host")
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
new file mode 100644
index 0000000..fb449b9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
@@ -0,0 +1,92 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import functools
+from resource_management import Script
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+
+config = Script.get_config()
+
+
+def __get_component_host(component):
+ """
+ Returns the first host where the given component is deployed, None if the component is not deployed
+ """
+ component_host = None
+ if component in config['clusterHostInfo'] and len(config['clusterHostInfo'][component]) > 0:
+ component_host = config['clusterHostInfo'][component][0]
+ return component_host
+
+
+def __get_namenode_host():
+ """
+ Gets the namenode host; active namenode in case of HA
+ """
+ namenode_host = __get_component_host('namenode_host')
+
+ # hostname of the active HDFS HA Namenode (only used when HA is enabled)
+ dfs_ha_namenode_active = default('/configurations/hadoop-env/dfs_ha_initial_namenode_active', None)
+ if dfs_ha_namenode_active is not None:
+ namenode_host = dfs_ha_namenode_active
+ return namenode_host
+
+
+hostname = config['hostname']
+
+# Users and Groups
+hdfs_superuser = config['configurations']['hadoop-env']['hdfs_user']
+user_group = config['configurations']['cluster-env']['user_group']
+
+# HAWQ Hostnames
+hawqmaster_host = __get_component_host('hawqmaster_hosts')
+hawqstandby_host = __get_component_host('hawqstandby_hosts')
+hawqsegment_hosts = default('/clusterHostInfo/hawqsegment_hosts', [])
+
+# HDFS
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+# HDFSResource partial function
+HdfsResource = functools.partial(HdfsResource, user=hdfs_superuser, hdfs_site=hdfs_site, default_fs=default_fs)
+
+namenode_host= __get_namenode_host()
+
+# YARN
+# Note: YARN is not mandatory for HAWQ. It is required only when the users set HAWQ to use YARN as resource manager
+rm_host = __get_component_host('rm_host')
+
+# Config files
+gpcheck_content = config['configurations']['gpcheck-env']['content']
+# database user limits
+hawq_limits = config['configurations']['hawq-limits-env']
+# sysctl parameters
+hawq_sysctl = config['configurations']['hawq-sysctl-env']
+
+hawq_site = config['configurations']['hawq-site']
+hawq_master_dir = hawq_site.get('hawq_master_directory')
+hawq_segment_dir = hawq_site.get('hawq_segment_directory')
+hawq_master_temp_dir = hawq_site.get('hawq_master_temp_directory')
+hawq_segment_temp_dir = hawq_site.get('hawq_segment_temp_directory')
+# Extract hawq hdfs directory from hdfs url. Ex: /hawq/hawq_default from
+# host:8080/hawq/hawq_default
+hawq_hdfs_data_dir = "/{0}".format(hawq_site.get('hawq_dfs_url').split('/', 1)[1])
+hawq_master_address_port = hawq_site.get('hawq_master_address_port')
+hawq_segment_address_port = hawq_site.get('hawq_segment_address_port')
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py
new file mode 100644
index 0000000..7a99f5e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/service_check.py
@@ -0,0 +1,102 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import common
+import constants
+from utils import exec_psql_cmd, exec_ssh_cmd
+from resource_management.libraries.script import Script
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+
+import sys
+
+class HAWQServiceCheck(Script):
+ """
+ Runs a set of simple HAWQ tests to verify if the service has been setup correctly
+ """
+
+ def __init__(self):
+ self.active_master_host = common.get_local_hawq_site_property("hawq_master_address_host")
+
+
+ def service_check(self, env):
+ Logger.info("Starting HAWQ service checks..")
+ # All the tests are run on the active_master_host using ssh irrespective of the node on which service check
+ # is executed by Ambari
+ try:
+ self.check_state()
+ self.drop_table()
+ self.create_table()
+ self.insert_data()
+ self.query_data()
+ self.check_data_correctness()
+ except:
+ Logger.error("Service check failed")
+ sys.exit(1)
+ finally:
+ self.drop_table()
+
+ Logger.info("Service check completed successfully")
+
+
+ def drop_table(self):
+ Logger.info("Dropping {0} table if exists".format(constants.smoke_check_table_name))
+ sql_cmd = "drop table if exists {0}".format(constants.smoke_check_table_name)
+ exec_psql_cmd(sql_cmd, self.active_master_host)
+
+
+ def create_table(self):
+ Logger.info("Creating table {0}".format(constants.smoke_check_table_name))
+ sql_cmd = "create table {0} (col1 int) distributed randomly".format(constants.smoke_check_table_name)
+ exec_psql_cmd(sql_cmd, self.active_master_host)
+
+
+ def insert_data(self):
+ Logger.info("Inserting data to table {0}".format(constants.smoke_check_table_name))
+ sql_cmd = "insert into {0} select * from generate_series(1,10)".format(constants.smoke_check_table_name)
+ exec_psql_cmd(sql_cmd, self.active_master_host)
+
+
+ def query_data(self):
+ Logger.info("Querying data from table {0}".format(constants.smoke_check_table_name))
+ sql_cmd = "select * from {0}".format(constants.smoke_check_table_name)
+ exec_psql_cmd(sql_cmd, self.active_master_host)
+
+
+ def check_data_correctness(self):
+ expected_data = "55"
+ Logger.info("Validating data inserted, finding sum of all the inserted entries. Expected output: {0}".format(expected_data))
+ sql_cmd = "select sum(col1) from {0}".format(constants.smoke_check_table_name)
+ _, stdout, _ = exec_psql_cmd(sql_cmd, self.active_master_host, tuples_only=False)
+ if expected_data != stdout.strip():
+ Logger.error("Incorrect data returned. Expected Data: {0} Actual Data: {1}".format(expected_data, stdout))
+ raise Fail("Incorrect data returned.")
+
+
+ def check_state(self):
+ import params
+ command = "source {0} && hawq state -d {1}".format(constants.hawq_greenplum_path_file, params.hawq_master_dir)
+ Logger.info("Executing hawq status check..")
+ (retcode, out, err) = exec_ssh_cmd(self.active_master_host, command)
+ if retcode:
+ Logger.error("hawq state command returned non-zero result: {0}. Out: {1} Error: {2}".format(retcode, out, err))
+ raise Fail("Unexpected result of hawq state command.")
+ Logger.info("Output of command:\n{0}".format(str(out) + "\n"))
+
+
+if __name__ == "__main__":
+ HAWQServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py
new file mode 100644
index 0000000..da51c19
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/utils.py
@@ -0,0 +1,108 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+import subprocess
+
+from resource_management.core.resources.system import Execute, Directory
+from resource_management.core.exceptions import Fail
+from resource_management.core.logger import Logger
+
+import constants
+
+def chk_postgres_status_cmd(port):
+ """
+ Checks if the postgres port is occupied
+ """
+ return "netstat -tupln | egrep ':{0}\s' | egrep postgres".format(port)
+
+
+def create_dir_as_hawq_user(directory):
+ """
+ Creates directories with hawq_user and hawq_group (defaults to gpadmin:gpadmin)
+ """
+ Directory(directory, recursive=True, owner=constants.hawq_user, group=constants.hawq_group)
+
+
+def exec_hawq_operation(operation, option, not_if=None, only_if=None, logoutput=True):
+ """
+ Sets up execution environment and runs a given command as HAWQ user
+ """
+ hawq_cmd = "source {0} && hawq {1} {2}".format(constants.hawq_greenplum_path_file, operation, option)
+ Execute(
+ hawq_cmd,
+ user=constants.hawq_user,
+ timeout=constants.hawq_operation_exec_timeout,
+ not_if=not_if,
+ only_if=only_if,
+ logoutput=logoutput)
+
+
+def read_file_to_dict(file_name):
+ """
+ Converts a file with key=value format to dictionary
+ """
+ with open(file_name, "r") as fh:
+ lines = fh.readlines()
+ lines = [item for item in lines if '=' in item]
+ result_dict = dict(item.split("=") for item in lines)
+ return result_dict
+
+
+def write_dict_to_file(source_dict, dest_file):
+ """
+ Writes a dictionary into a file with key=value format
+ """
+ with open(dest_file, "w") as fh:
+ for property_key, property_value in source_dict.items():
+ if property_value is None:
+ fh.write(property_key + "\n")
+ else:
+ fh.write("{0}={1}\n".format(property_key, property_value))
+
+
+def exec_ssh_cmd(hostname, cmd):
+ """
+ Runs the command on the remote host as gpadmin user
+ """
+ import params
+ # Only gpadmin should be allowed to run command via ssh, thus not exposing user as a parameter
+ if params.hostname != hostname:
+ cmd = "su - {0} -c 'ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {1} \"{2} \" '".format(constants.hawq_user, hostname, cmd)
+ else:
+ cmd = "su - {0} -c \"{1}\"".format(constants.hawq_user, cmd)
+ Logger.info("Command executed: {0}".format(cmd))
+ process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
+ (stdout, stderr) = process.communicate()
+ return process.returncode, stdout, stderr
+
+
+def exec_psql_cmd(command, host, db="template1", tuples_only=True):
+ """
+ Sets up execution environment and runs the HAWQ queries
+ """
+ src_cmd = "source {0}".format(constants.hawq_greenplum_path_file)
+ if tuples_only:
+ cmd = src_cmd + " && psql -d {0} -c \\\"{1};\\\"".format(db, command)
+ else:
+ cmd = src_cmd + " && psql -t -d {0} -c \\\"{1};\\\"".format(db, command)
+ retcode, out, err = exec_ssh_cmd(host, cmd)
+ if retcode:
+ Logger.error("SQL command executed failed: {0}\nReturncode: {1}\nStdout: {2}\nStderr: {3}".format(cmd, retcode, out, err))
+ raise Fail("SQL command executed failed.")
+
+ Logger.info("Output:\n{0}".format(out))
+ return retcode, out, err
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
new file mode 100644
index 0000000..9bdaa75
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-hosts.j2
@@ -0,0 +1,5 @@
+{{hawqmaster_host}}
+{{hawqstandby_host}}
+{% for host in hawqsegment_hosts %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-profile.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-profile.sh.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-profile.sh.j2
new file mode 100644
index 0000000..e28e416
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/hawq-profile.sh.j2
@@ -0,0 +1,8 @@
+######################################
+##### HAWQ env #######################
+######################################
+source {{hawq_greenplum_path_file}}
+export MASTER_DATA_DIRECTORY={{hawq_master_dir}}
+export PGPORT={{hawq_master_address_port}}
+export PGHOST={{hawqmaster_host}}
+export PGUSER={{hawq_user}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2 b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
new file mode 100644
index 0000000..591d92b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/templates/slaves.j2
@@ -0,0 +1,3 @@
+{% for host in hawqsegment_hosts %}
+{{host}}
+{% endfor %}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/configuration/pxf-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/configuration/pxf-site.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/configuration/pxf-site.xml
deleted file mode 100644
index 0b3a36e..0000000
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/configuration/pxf-site.xml
+++ /dev/null
@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<configuration>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/metainfo.xml
deleted file mode 100644
index f578d64..0000000
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/metainfo.xml
+++ /dev/null
@@ -1,71 +0,0 @@
-<?xml version="1.0"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-
-<metainfo>
- <schemaVersion>2.0</schemaVersion>
- <services>
- <service>
- <name>PXF</name>
- <displayName>PXF</displayName>
- <comment>HAWQ Extension Framework</comment>
- <version>3.0</version>
- <components>
-
- <component>
- <name>PXF</name>
- <displayName>PXF</displayName>
- <category>SLAVE</category>
- <cardinality>1+</cardinality>
- <commandScript>
- <script>scripts/pxf.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- </component>
- </components>
-
- <requiredServices>
- <service>HAWQ</service>
- </requiredServices>
-
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>pxf-service</name>
- </package>
- <package>
- <name>apache-tomcat</name>
- </package>
- <package>
- <name>pxf-hive</name>
- </package>
- <package>
- <name>pxf-hdfs</name>
- </package>
- <package>
- <name>pxf-hbase</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- </service>
- </services>
-</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/params.py
deleted file mode 100644
index a4986c9..0000000
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/params.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management import Script
-
-config = Script.get_config()
-
-
-pxf_service_name = "pxf-service"
-stack_name = str(config["hostLevelParams"]["stack_name"])
-
-# Users and Groups
-pxf_user = "pxf"
-pxf_group = pxf_user
-hdfs_superuser_group = config["configurations"]["hdfs-site"]["dfs.permissions.superusergroup"]
-user_group = config["configurations"]["cluster-env"]["user_group"]
-tomcat_group = "tomcat"
-
-# Directories
-pxf_conf_dir = "/etc/pxf/conf"
-pxf_instance_dir = "/var/pxf"
-
-# Java home path
-java_home = config["hostLevelParams"]["java_home"] if "java_home" in config["hostLevelParams"] else None
-
-# Timeouts
-default_exec_timeout = 600
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/pxf.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/pxf.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/pxf.py
deleted file mode 100644
index dd0031c..0000000
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/scripts/pxf.py
+++ /dev/null
@@ -1,120 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
-
-from resource_management import Script
-
-from resource_management.libraries.resources.xml_config import XmlConfig
-from resource_management.core.resources.accounts import User
-from resource_management.core.resources.system import Directory, File, Execute
-from resource_management.core.source import Template
-
-
-
-class Pxf(Script):
- """
- Contains the interface definitions for methods like install,
- start, stop, status, etc. for the PXF
- """
-
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
-
- def configure(self, env):
- import params
- env.set_params(params)
- self.__setup_user_group()
- self.__generate_config_files()
- # pxf-service init exits safely when it is already initialized
- self.__execute_service_command("init")
-
-
- def start(self, env):
- self.configure(env)
- self.__grant_permissions()
- self.__execute_service_command("restart")
-
-
- def stop(self, env):
- self.__execute_service_command("stop")
-
-
- def status(self, env):
- try:
- self.__execute_service_command("status")
- except Exception:
- raise ComponentIsNotRunning()
-
-
- def __execute_service_command(self, command):
- import params
- Execute("service {0} {1}".format(params.pxf_service_name, command),
- timeout=params.default_exec_timeout,
- logoutput=True)
-
-
- def __setup_user_group(self):
- """
- Creates PXF user with the required groups and bash as default shell
- """
- import params
- User(params.pxf_user,
- groups=[params.hdfs_superuser_group, params.user_group, params.tomcat_group],
- shell="/bin/bash")
-
-
- def __generate_config_files(self):
- """
- Generates pxf-env.sh file from jinja template and sets the classpath for HDP
- """
- import params
- import shutil
-
- hdp_stack = "HDP"
-
- # Create file pxf-env.sh from jinja template
- File("{0}/pxf-env.sh".format(params.pxf_conf_dir),
- content = Template("pxf-env.j2"))
-
- # Classpath is set for PHD by default. If stack is HDP, set classpath for HDP
- if(params.stack_name == hdp_stack):
- shutil.copy2("{0}/pxf-privatehdp.classpath".format(params.pxf_conf_dir),
- "{0}/pxf-private.classpath".format(params.pxf_conf_dir))
-
- XmlConfig("pxf-site.xml",
- conf_dir=params.pxf_conf_dir,
- configurations=params.config['configurations']['pxf-site'],
- configuration_attributes=params.config['configuration_attributes']['pxf-site'])
-
-
- def __grant_permissions(self):
- """
- Grants permission to pxf:pxf for PXF instance directory
- """
- import params
- Directory(params.pxf_instance_dir,
- owner=params.pxf_user,
- group=params.pxf_group,
- recursive=True)
-
-
-if __name__ == "__main__":
- Pxf().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/templates/pxf-env.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/templates/pxf-env.j2 b/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/templates/pxf-env.j2
deleted file mode 100644
index 03f2420..0000000
--- a/ambari-server/src/main/resources/common-services/PXF/3.0.0.0/package/templates/pxf-env.j2
+++ /dev/null
@@ -1,34 +0,0 @@
-#!/bin/sh
-
-#Licensed to the Apache Software Foundation (ASF) under one
-#or more contributor license agreements. See the NOTICE file
-#distributed with this work for additional information
-#regarding copyright ownership. The ASF licenses this file
-#to you under the Apache License, Version 2.0 (the
-#"License"); you may not use this file except in compliance
-#with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-
-
-# THIS FILE SHOULD MATCH WITH https://github.com/apache/incubator-hawq/blob/master/pxf/pxf-service/src/scripts/pxf-env.sh
-
-# Path to HDFS native libraries
-export LD_LIBRARY_PATH=/usr/lib/hadoop/lib/native:${LD_LIBRARY_PATH}
-
-# Path to JAVA
-export JAVA_HOME={{java_home}}
-
-# Path to Log directory
-export PXF_LOGDIR=/var/log/pxf
-export CATALINA_OUT=${PXF_LOGDIR}/catalina.out
-
-# Path to Run directory
-export PXF_RUNDIR=/var/run/pxf
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-site.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-site.xml
new file mode 100644
index 0000000..0b3a36e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/configuration/pxf-site.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml
new file mode 100644
index 0000000..f578d64
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/metainfo.xml
@@ -0,0 +1,71 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PXF</name>
+ <displayName>PXF</displayName>
+ <comment>HAWQ Extension Framework</comment>
+ <version>3.0</version>
+ <components>
+
+ <component>
+ <name>PXF</name>
+ <displayName>PXF</displayName>
+ <category>SLAVE</category>
+ <cardinality>1+</cardinality>
+ <commandScript>
+ <script>scripts/pxf.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ </component>
+ </components>
+
+ <requiredServices>
+ <service>HAWQ</service>
+ </requiredServices>
+
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>pxf-service</name>
+ </package>
+ <package>
+ <name>apache-tomcat</name>
+ </package>
+ <package>
+ <name>pxf-hive</name>
+ </package>
+ <package>
+ <name>pxf-hdfs</name>
+ </package>
+ <package>
+ <name>pxf-hbase</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5dac27be/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
new file mode 100644
index 0000000..a4986c9
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PXF/3.0.0/package/scripts/params.py
@@ -0,0 +1,42 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from resource_management import Script
+
+config = Script.get_config()
+
+
+pxf_service_name = "pxf-service"
+stack_name = str(config["hostLevelParams"]["stack_name"])
+
+# Users and Groups
+pxf_user = "pxf"
+pxf_group = pxf_user
+hdfs_superuser_group = config["configurations"]["hdfs-site"]["dfs.permissions.superusergroup"]
+user_group = config["configurations"]["cluster-env"]["user_group"]
+tomcat_group = "tomcat"
+
+# Directories
+pxf_conf_dir = "/etc/pxf/conf"
+pxf_instance_dir = "/var/pxf"
+
+# Java home path
+java_home = config["hostLevelParams"]["java_home"] if "java_home" in config["hostLevelParams"] else None
+
+# Timeouts
+default_exec_timeout = 600