You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2014/02/20 20:03:17 UTC
[1/2] git commit: AMBARI-4754. Changes to default configs. (swagle)
Repository: ambari
Updated Branches:
refs/heads/trunk f72be08ba -> e44bbb73c
AMBARI-4754. Changes to default configs. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e44bbb73
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e44bbb73
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e44bbb73
Branch: refs/heads/trunk
Commit: e44bbb73c111a492daba4414f12a0a09bd859c1f
Parents: 22fa3e0
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Feb 20 11:02:14 2014 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Feb 20 11:03:07 2014 -0800
----------------------------------------------------------------------
.../services/HIVE/configuration/hive-site.xml | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e44bbb73/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
index bfdc8ac..07f37ca 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/HIVE/configuration/hive-site.xml
@@ -282,4 +282,22 @@ limitations under the License.
</description>
</property>
+ <property>
+ <name>hive.execution.engine</name>
+ <value>mr</value>
+ <description>Whether to use MR or Tez</description>
+ </property>
+
+ <property>
+ <name>hive.exec.post.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ <description>Post Execute Hook for Tests</description>
+ </property>
+
+ <property>
+ <name>hive.exec.pre.hooks</name>
+ <value>org.apache.hadoop.hive.ql.hooks.ATSHook</value>
+ <description>Pre Execute Hook for Tests</description>
+ </property>
+
</configuration>
[2/2] git commit: AMBARI-4754. Upload tez libraries to HDFS on Tez
install. (swagle)
Posted by sw...@apache.org.
AMBARI-4754. Upload tez libraries to HDFS on Tez install. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/22fa3e03
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/22fa3e03
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/22fa3e03
Branch: refs/heads/trunk
Commit: 22fa3e0338c710683a6444f8294ab3c2e8c537fc
Parents: f72be08
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Thu Feb 20 10:34:45 2014 -0800
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Thu Feb 20 11:03:07 2014 -0800
----------------------------------------------------------------------
.../libraries/script/config_dictionary.py | 4 +-
.../services/TEZ/configuration/tez-site.xml | 2 +-
.../services/TEZ/package/scripts/params.py | 29 ++++-
.../2.1.1/services/TEZ/package/scripts/tez.py | 123 ++++++++++++++++++-
.../TEZ/package/templates/tez-env.sh.j2 | 2 +-
.../python/stacks/2.1.1/TEZ/test_tez_client.py | 51 +++++++-
.../python/stacks/2.1.1/configs/default.json | 2 +-
7 files changed, 203 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py b/ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
index 453c546..ab94d60 100644
--- a/ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
+++ b/ambari-agent/src/main/python/resource_management/libraries/script/config_dictionary.py
@@ -72,7 +72,9 @@ class UnknownConfiguration():
self.name = name
def __getattr__(self, name):
- raise Fail("Configuration parameter '"+self.name+"' was not found in configurations dictionary!")
+# raise Fail("Configuration parameter '"+self.name+"' was not found in
+# configurations dictionary!")
+ pass
def __getitem__(self, name):
"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
index 5d78c08..333a8ac 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/configuration/tez-site.xml
@@ -21,7 +21,7 @@
<property>
<name>tez.lib.uris</name>
- <value>file:///usr/lib/tez/,file:///usr/lib/tez/lib/</value>
+ <value>hdfs:///apps/tez/,hdfs:///apps/tez/lib/</value>
<description>The location of the Tez libraries which will be localized for DAGs</description>
</property>
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py
index c35f7be..2bc4540 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/params.py
@@ -23,10 +23,35 @@ from resource_management import *
# server configurations
config = Script.get_config()
-conf_dir = "/etc/tez/conf"
+config_dir = "/etc/tez/conf"
hadoop_home = '/usr'
java64_home = config['hostLevelParams']['java_home']
tez_user = 'tez'
-user_group = config['configurations']['global']['user_group']
\ No newline at end of file
+user_group = config['configurations']['global']['user_group']
+
+tez_lib_uris = config['configurations']['tez-site']['tez.lib.uris']
+tez_local_api_jars = '/usr/lib/tez/tez*.jar'
+tez_local_lib_jars = '/usr/lib/tez/lib/*.jar'
+
+stub_path = '/tmp/tez_jars_copied'
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+_authentication = config['configurations']['core-site']['hadoop.security.authentication']
+security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
+hdfs_user_keytab = config['configurations']['global']['hdfs_user_keytab']
+hdfs_user = config['configurations']['global']['hdfs_user']
+kinit_path_local = functions.get_kinit_path([default("kinit_path_local",None), "/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py
index b23a356..757fe73 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/scripts/tez.py
@@ -21,18 +21,19 @@ Ambari Agent
"""
from resource_management import *
+import os
def tez():
import params
- Directory( params.conf_dir,
+ Directory(params.config_dir,
owner = params.tez_user,
group = params.user_group,
recursive = True
)
XmlConfig( "tez-site.xml",
- conf_dir = params.conf_dir,
+ conf_dir = params.config_dir,
configurations = params.config['configurations']['tez-site'],
owner = params.tez_user,
group = params.user_group,
@@ -41,6 +42,68 @@ def tez():
tez_TemplateConfig( ['tez-env.sh'])
+ destination_hdfs_dirs = get_tez_hdfs_dir_paths(params.tez_lib_uris)
+
+ # If tez libraries are to be stored in hdfs
+ if destination_hdfs_dirs:
+ for hdfs_dir in destination_hdfs_dirs:
+ params.HdfsDirectory(hdfs_dir,
+ action="create_delayed",
+ owner=params.tez_user,
+ mode=0755
+ )
+ pass
+ params.HdfsDirectory(None, action="create")
+
+ if params.security_enabled:
+ kinit_if_needed = format("{kinit_path_local} -kt {hdfs_user_keytab} {hdfs_user};")
+ else:
+ kinit_if_needed = ""
+
+ if kinit_if_needed:
+ Execute(kinit_if_needed,
+ user=params.tez_user,
+ path='/bin'
+ )
+
+ app_dir_path = None
+ lib_dir_path = None
+
+ if len(destination_hdfs_dirs) > 1:
+ for path in destination_hdfs_dirs:
+ if 'lib' in path:
+ lib_dir_path = path
+ else:
+ app_dir_path = path
+ pass
+ pass
+ pass
+
+ if app_dir_path:
+ copyFromLocal(path=params.tez_local_api_jars,
+ mode=0655,
+ owner=params.tez_user,
+ dest_dir=app_dir_path,
+ kinnit_if_needed=kinit_if_needed,
+ stub_path=params.stub_path
+ )
+ pass
+
+ if lib_dir_path:
+ copyFromLocal(path=params.tez_local_lib_jars,
+ mode=0655,
+ owner=params.tez_user,
+ dest_dir=lib_dir_path,
+ kinnit_if_needed=kinit_if_needed,
+ stub_path=params.stub_path
+ )
+ pass
+
+ ExecuteHadoop(format('dfs -touchz {stub_path}'),
+ user = params.tez_user,
+ conf_dir=params.hadoop_conf_dir
+ )
+
def tez_TemplateConfig(name):
import params
@@ -49,6 +112,60 @@ def tez_TemplateConfig(name):
name = [name]
for x in name:
- TemplateConfig( format("{conf_dir}/{x}"),
+ TemplateConfig(format("{config_dir}/{x}"),
owner = params.tez_user
)
+
+def get_tez_hdfs_dir_paths(tez_lib_uris = None):
+ hdfs_path_prefix = 'hdfs://'
+ lib_dir_paths = []
+ if tez_lib_uris and tez_lib_uris.strip().find(hdfs_path_prefix, 0) != -1:
+ dir_paths = tez_lib_uris.split(',')
+ for path in dir_paths:
+ lib_dir_path = path.replace(hdfs_path_prefix, '')
+ lib_dir_path = lib_dir_path if lib_dir_path.endswith(os.sep) else lib_dir_path + os.sep
+ lib_dir_paths.append(lib_dir_path)
+ pass
+ pass
+
+ return lib_dir_paths
+
+
+def copyFromLocal(path=None, owner=None, group=None, mode=None,
+ dest_dir=None, kinnit_if_needed="", stub_path=None):
+ import params
+
+ if not stub_path:
+ stub_path = params.stub_path
+ pass
+
+ copy_cmd = format("fs -copyFromLocal {path} {dest_dir}")
+ unless_cmd = format("{kinnit_if_needed} hadoop fs -ls {stub_path} >/dev/null 2>&1")
+
+ ExecuteHadoop(copy_cmd,
+ not_if=unless_cmd,
+ user=owner,
+ conf_dir=params.hadoop_conf_dir,
+ ignore_failures = True)
+
+ if not owner:
+ chown = None
+ else:
+ if not group:
+ chown = owner
+ else:
+ chown = format('{owner}:{group}')
+
+ if not chown:
+ chown_cmd = format("fs -chown {chown} {dest_dir}")
+
+ ExecuteHadoop(copy_cmd,
+ user=owner,
+ conf_dir=params.hadoop_conf_dir)
+
+ if not mode:
+ chmod_cmd = format('fs -chmod {mode} {dest_dir}')
+
+ ExecuteHadoop(chmod_cmd,
+ user=owner,
+ conf_dir=params.hadoop_conf_dir)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2
index 49a40e1..b82329f 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2
+++ b/ambari-server/src/main/resources/stacks/HDP/2.1.1/services/TEZ/package/templates/tez-env.sh.j2
@@ -14,7 +14,7 @@
# limitations under the License.
# Tez specific configuration
-export TEZ_CONF_DIR={{conf_dir}}
+export TEZ_CONF_DIR={{config_dir}}
# Set HADOOP_HOME to point to a specific hadoop install directory
export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py b/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py
index 05224fc..b3227ed 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py
+++ b/ambari-server/src/test/python/stacks/2.1.1/TEZ/test_tez_client.py
@@ -22,7 +22,7 @@ from stacks.utils.RMFTestCase import *
class TestTezClient(RMFTestCase):
- def test_configure_defualt(self):
+ def test_configure_default(self):
self.executeScript("2.1.1/services/TEZ/package/scripts/tez_client.py",
classname = "TezClient",
command = "configure",
@@ -47,6 +47,55 @@ class TestTezClient(RMFTestCase):
owner = 'tez'
)
+ self.assertResourceCalled('HdfsDirectory', '/apps/tez/',
+ action = ['create_delayed'],
+ mode = 0755,
+ owner = 'tez',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = "/usr/bin/kinit"
+ )
+
+ self.assertResourceCalled('HdfsDirectory', '/apps/tez/lib/',
+ action = ['create_delayed'],
+ mode = 0755,
+ owner = 'tez',
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = "/usr/bin/kinit"
+ )
+ self.assertResourceCalled('HdfsDirectory', None,
+ security_enabled = False,
+ keytab = UnknownConfigurationMock(),
+ conf_dir = '/etc/hadoop/conf',
+ hdfs_user = 'hdfs',
+ kinit_path_local = '/usr/bin/kinit',
+ action = ['create']
+ )
+
+ self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/tez/tez*.jar /apps/tez/',
+ not_if = ' hadoop fs -ls /tmp/tez_jars_copied >/dev/null 2>&1',
+ user = 'tez',
+ conf_dir = '/etc/hadoop/conf',
+ ignore_failures=True
+ )
+
+ self.assertResourceCalled('ExecuteHadoop', 'fs -copyFromLocal /usr/lib/tez/lib/*.jar /apps/tez/lib/',
+ not_if = ' hadoop fs -ls /tmp/tez_jars_copied >/dev/null 2>&1',
+ user = 'tez',
+ conf_dir = '/etc/hadoop/conf',
+ ignore_failures=True
+ )
+
+ self.assertResourceCalled('ExecuteHadoop', 'dfs -touchz /tmp/tez_jars_copied',
+ user = 'tez',
+ conf_dir = '/etc/hadoop/conf'
+ )
+
self.assertNoMoreResources()
http://git-wip-us.apache.org/repos/asf/ambari/blob/22fa3e03/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
index ebb0742..89c03ab 100644
--- a/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.1.1/configs/default.json
@@ -485,7 +485,7 @@
},
"tez-site": {
"tez.am.log.level": "WARN",
- "tez.lib.uris": "file:///usr/lib/tez/,file:///usr/lib/tez/lib/",
+ "tez.lib.uris": "hdfs:///apps/tez/,hdfs:///apps/tez/lib/",
"tez.staging-dir": "/tmp/${user.name}/staging",
"tez.am.am-rm.heartbeat.interval-ms.max": "250"
},