You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by od...@apache.org on 2015/12/24 21:13:17 UTC
ambari git commit: AMBARI-14165. Populate hawq's yarn-client xml
under /usr/local/hawq/etc (bhuvnesh chaudhary via odiachenko).
Repository: ambari
Updated Branches:
refs/heads/branch-2.2 d59600e62 -> 246dc6800
AMBARI-14165. Populate hawq's yarn-client xml under /usr/local/hawq/etc (bhuvnesh chaudhary via odiachenko).
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/246dc680
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/246dc680
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/246dc680
Branch: refs/heads/branch-2.2
Commit: 246dc6800b95baf45df93cba205889eca1c38c9e
Parents: d59600e
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Thu Dec 24 12:12:57 2015 -0800
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Thu Dec 24 12:12:57 2015 -0800
----------------------------------------------------------------------
.../HAWQ/2.0.0/configuration/yarn-client.xml | 99 ++++++++++++++++++++
.../HAWQ/2.0.0/package/scripts/common.py | 55 +++++++++++
.../HAWQ/2.0.0/package/scripts/constants.py | 1 +
.../HAWQ/2.0.0/package/scripts/params.py | 4 +-
4 files changed, 157 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
new file mode 100755
index 0000000..a2296a1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/yarn-client.xml
@@ -0,0 +1,99 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- RPC client configuration -->
+ <property>
+ <name>rpc.client.timeout</name>
+ <value>3600000</value>
+ <description>
+ Timeout interval of a RPC invocation in millisecond. Default is 3600000.
+ </description>
+ </property>
+ <property>
+ <name>rpc.client.connect.tcpnodelay</name>
+ <value>true</value>
+ <description>
+ Whether to set socket TCP_NODELAY to true when connecting to RPC server. Default is true.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.max.idle</name>
+ <value>10000</value>
+ <description>
+ Max idle time of a RPC connection in milliseconds. Default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.ping.interval</name>
+ <value>10000</value>
+ <description>
+ Interval at which the RPC client sends a heart beat to server. 0 means disable, default is 10000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.timeout</name>
+ <value>600000</value>
+ <description>
+ Timeout interval in milliseconds for which RPC client tries to setup the connection. Default is 600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.connect.retry</name>
+ <value>10</value>
+ <description>
+ Max retry attempts if the RPC client fails to setup the connection to server. Default is 10.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.read.timeout</name>
+ <value>3600000</value>
+ <description>
+ Timeout interval in milliseconds when the RPC client is trying to read from server. Default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.write.timeout</name>
+ <value>3600000</value>
+ <description>
+ Timeout interval in milliseconds when the RPC client is trying to write to server. Default is 3600000.
+ </description>
+ </property>
+
+ <property>
+ <name>rpc.client.socket.linger.timeout</name>
+ <value>-1</value>
+ <description>
+ Set value to socket SO_LINGER when connecting to RPC server. -1 means default OS value. Default is -1.
+ </description>
+ </property>
+
+ <property>
+ <name>yarn.client.failover.max.attempts</name>
+ <value>15</value>
+ <description>
+ If multiple resource managers are configured, it is the max retry attempts for which yarn client tries to issue a RPC call. Default is 15.
+ </description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
index 31ef2fa..c62ac12 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
@@ -22,6 +22,7 @@ import crypt
import filecmp
from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.core.resources.system import Execute, Directory, File
+from resource_management.libraries.script.config_dictionary import ConfigDictionary
from resource_management.core.logger import Logger
from resource_management.core.system import System
from resource_management.core.exceptions import Fail
@@ -88,9 +89,63 @@ def setup_common_configurations():
owner=constants.hawq_user,
group=constants.hawq_group,
mode=0644)
+ if "yarn-site" in params.config["configurations"]:
+ __update_yarn_client()
__set_osparams()
+def __update_yarn_client():
+ """
+ Writes yarn-client.xml on the local filesystem on hawq nodes.
+ If yarn ha is enabled, appends related parameters to yarn-client.xml
+ """
+ import params
+
+ yarn_client_dict = params.yarn_client.copy()
+ if params.yarn_ha_enabled:
+ # Temporary logic, this logic will be moved in ambari-web to expose these parameters on UI once Yarn HA is enabled
+ rm_ids = [rm_id.strip() for rm_id in params.config['configurations']['yarn-site']['yarn.resourcemanager.ha.rm-ids'].split(',')]
+ rm_id1 = rm_ids[0]
+ rm_id2 = rm_ids[1]
+ # Identify the hostname for yarn resource managers
+ rm_host1= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id1)]
+ rm_host2= params.config['configurations']['yarn-site']['yarn.resourcemanager.hostname.{0}'.format(rm_id2)]
+ # Ambari does not update yarn.resourcemanager.address.${rm_id} and yarn.resourcemanager.scheduler.address.${rm_id}
+ # property as its derived automatically at yarn.
+ # Hawq uses these properties to use yarn ha. If these properties are defined at Ambari use them, else derive them.
+ # Use port 8032 to derive hawq.resourcemanager.address.${rm_id}:port value if needed
+ rm_default_port = 8032
+ # Use port 8030 to derive hawq.resourcemanager.scheduler.address.${rm_id}:port value if needed
+ rm_scheduler_default_port = 8030
+
+ rm_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id1))
+ if rm_address_host1 is None:
+ rm_address_host1 = "{0}:{1}".format(rm_host1, rm_default_port)
+
+ rm_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.address.{0}'.format(rm_id2))
+ if rm_address_host2 is None:
+ rm_address_host2 = "{0}:{1}".format(rm_host2, rm_default_port)
+
+ rm_scheduler_address_host1 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id1))
+ if rm_scheduler_address_host1 is None:
+ rm_scheduler_address_host1 = "{0}:{1}".format(rm_host1, rm_scheduler_default_port)
+
+ rm_scheduler_address_host2 = params.config['configurations']['yarn-site'].get('yarn.resourcemanager.scheduler.address.{0}'.format(rm_id2))
+ if rm_scheduler_address_host2 is None:
+ rm_scheduler_address_host2 = "{0}:{1}".format(rm_host2, rm_scheduler_default_port)
+
+ yarn_client_dict['yarn.resourcemanager.ha'] = "{0},{1}".format(rm_address_host1, rm_address_host2)
+ yarn_client_dict['yarn.resourcemanager.scheduler.ha'] = "{0},{1}".format(rm_scheduler_address_host1, rm_scheduler_address_host2)
+
+ XmlConfig("yarn-client.xml",
+ conf_dir=constants.hawq_config_dir,
+ configurations=ConfigDictionary(yarn_client_dict),
+ configuration_attributes=params.config['configuration_attributes']['yarn-client'],
+ owner=constants.hawq_user,
+ group=constants.hawq_group,
+ mode=0644)
+
+
def __substitute_hostnames_in_hawq_site():
"""
Temporary function to replace localhost with actual HAWQ component hostnames.
http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
index b56fd59..ff8f4be 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/constants.py
@@ -23,6 +23,7 @@ SEGMENT = "segment"
START = "start"
INIT = "init"
STOP = "stop"
+YARN = "yarn"
# Users
root_user = "root"
http://git-wip-us.apache.org/repos/asf/ambari/blob/246dc680/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
index fb449b9..c8e97b8 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/params.py
@@ -70,6 +70,7 @@ namenode_host= __get_namenode_host()
# YARN
# Note: YARN is not mandatory for HAWQ. It is required only when the users set HAWQ to use YARN as resource manager
rm_host = __get_component_host('rm_host')
+yarn_ha_enabled = default('/configurations/yarn-site/yarn.resourcemanager.ha.enabled', False)
# Config files
gpcheck_content = config['configurations']['gpcheck-env']['content']
@@ -79,6 +80,7 @@ hawq_limits = config['configurations']['hawq-limits-env']
hawq_sysctl = config['configurations']['hawq-sysctl-env']
hawq_site = config['configurations']['hawq-site']
+yarn_client = config['configurations']['yarn-client']
hawq_master_dir = hawq_site.get('hawq_master_directory')
hawq_segment_dir = hawq_site.get('hawq_segment_directory')
hawq_master_temp_dir = hawq_site.get('hawq_master_temp_directory')
@@ -88,5 +90,3 @@ hawq_segment_temp_dir = hawq_site.get('hawq_segment_temp_directory')
hawq_hdfs_data_dir = "/{0}".format(hawq_site.get('hawq_dfs_url').split('/', 1)[1])
hawq_master_address_port = hawq_site.get('hawq_master_address_port')
hawq_segment_address_port = hawq_site.get('hawq_segment_address_port')
-
-