You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by fb...@apache.org on 2015/05/29 18:43:49 UTC
ambari git commit: AMBARI-11513 [WinTP2] HDP 2.3 stack: Tez service
fails to start
Repository: ambari
Updated Branches:
refs/heads/trunk 2831bdb3f -> 2dbb7b284
AMBARI-11513 [WinTP2] HDP 2.3 stack: Tez service fails to start
+Fixed Tez settings generation
+Fixed Tez service check dependency order
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/2dbb7b28
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/2dbb7b28
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/2dbb7b28
Branch: refs/heads/trunk
Commit: 2dbb7b2842807359937b5d15fb4a35635333622f
Parents: 2831bdb
Author: Florian Barca <fb...@hortonworks.com>
Authored: Fri May 29 09:43:46 2015 -0700
Committer: Florian Barca <fb...@hortonworks.com>
Committed: Fri May 29 09:43:46 2015 -0700
----------------------------------------------------------------------
.../src/main/python/ambari_commons/os_utils.py | 21 ++++++++-
.../libraries/functions/get_hdp_version.py | 32 +++++++++++++-
.../0.4.0.2.1/package/scripts/params_windows.py | 22 +++++++++-
.../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 46 +++++++++++++++++---
.../stacks/HDPWIN/2.1/role_command_order.json | 1 +
.../2.1/services/TEZ/configuration/tez-env.xml | 3 +-
.../2.2/services/TEZ/configuration/tez-site.xml | 11 +----
.../services/HIVE/configuration/hive-site.xml | 9 ----
.../2.3/services/TEZ/configuration/tez-site.xml | 15 ++++++-
9 files changed, 130 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-common/src/main/python/ambari_commons/os_utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_commons/os_utils.py b/ambari-common/src/main/python/ambari_commons/os_utils.py
index 5ee34a8..d496530 100644
--- a/ambari-common/src/main/python/ambari_commons/os_utils.py
+++ b/ambari-common/src/main/python/ambari_commons/os_utils.py
@@ -123,4 +123,23 @@ def find_in_path(file):
full_path = _search_file(file, os.environ["PATH"], os.pathsep)
if full_path is None:
raise Exception("File {0} not found in PATH".format(file))
- return full_path
\ No newline at end of file
+ return full_path
+
+def extract_path_component(path, path_fragment):
+ iFragment = path.find(path_fragment)
+ if iFragment != -1:
+ iComponentStart = 0
+ while iComponentStart < iFragment:
+ iComponentStartTemp = path.find(os.pathsep, iComponentStart)
+ if iComponentStartTemp == -1 or iComponentStartTemp > iFragment:
+ break
+ iComponentStart = iComponentStartTemp
+
+ iComponentEnd = path.find(os.pathsep, iFragment)
+ if iComponentEnd == -1:
+ iComponentEnd = len(path)
+
+ path_component = path[iComponentStart:iComponentEnd]
+ return path_component
+ else:
+ return None
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
index ec30dc4..e8fdbb6 100644
--- a/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
+++ b/ambari-common/src/main/python/resource_management/libraries/functions/get_hdp_version.py
@@ -19,15 +19,45 @@ limitations under the License.
Ambari Agent
"""
-
__all__ = ["get_hdp_version"]
+
import os
import re
+
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
from resource_management.core.logger import Logger
from resource_management.core.exceptions import Fail
from resource_management.core import shell
+@OsFamilyFuncImpl(OSConst.WINSRV_FAMILY)
+def get_hdp_version(package_name):
+ """
+ @param package_name, name of the package, from which, function will try to get hdp version
+ """
+ try:
+ component_home_dir = os.environ[package_name.upper() + "_HOME"]
+ except KeyError:
+ Logger.info('Skipping get_hdp_version since the component {0} is not yet available'.format(package_name))
+ return None # lazy fail
+
+ #As a rule, component_home_dir is of the form <hdp_root_dir>\[\]<component_versioned_subdir>[\]
+ home_dir_split = os.path.split(component_home_dir)
+ iSubdir = len(home_dir_split) - 1
+ while not home_dir_split[iSubdir]:
+ iSubdir -= 1
+
+ #The component subdir is expected to be of the form <package_name>-<package_version>.<hdp_stack_version>
+ # with package_version = #.#.# and hdp_stack_version=#.#.#.#-<build_number>
+ match = re.findall('[0-9]+.[0-9]+.[0-9]+.[0-9]+-[0-9]+', home_dir_split[iSubdir])
+ if not match:
+ Logger.info('Failed to get extracted version for component {0}. Home dir not in expected format.'.format(package_name))
+ return None # lazy fail
+
+ return match[0]
+@OsFamilyFuncImpl(OsFamilyImpl.DEFAULT)
def get_hdp_version(package_name):
"""
@param package_name, name of the package, from which, function will try to get hdp version
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
index 2a7bd8b..2f9ee30 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
@@ -17,17 +17,35 @@ limitations under the License.
"""
-from resource_management import *
import os
+from resource_management.libraries.functions.get_hdp_version import get_hdp_version
+from resource_management.libraries.script.script import Script
+
config = Script.get_config()
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+user_group = config["configurations"]["cluster-env"]["user_group"]
tez_user = hadoop_user
tez_home_dir = None
tez_conf_dir = "conf"
+try:
+ hadoop_classpath_prefix_template = config["configurations"]["tez-site"]["tez.cluster.additional.classpath.prefix"]
+except KeyError:
+ hadoop_classpath_prefix_template = ""
+
+hdp_stack_version = ""
+
hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
-if os.environ.has_key("TEZ_HOME"):
+
+def refresh_tez_state_dependent_params():
+ global tez_home_dir, tez_conf_dir, hdp_stack_version
tez_home_dir = os.environ["TEZ_HOME"]
tez_conf_dir = os.path.join(tez_home_dir, "conf")
+ # this is not available on INSTALL action because hdp-select is not available
+ hdp_stack_version = get_hdp_version("tez")
+
+
+if os.environ.has_key("TEZ_HOME"):
+ refresh_tez_state_dependent_params()
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
index e6c92e1..1bd82ae 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
@@ -18,14 +18,23 @@ limitations under the License.
Ambari Agent
"""
+import os
+import urlparse
-import sys
-from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.inet_utils import download_file
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons.os_utils import copy_file, extract_path_component
+
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.source import InlineTemplate
from resource_management.libraries.functions import conf_select
from resource_management.libraries.functions import hdp_select
+from resource_management.libraries.functions.get_hdp_version import get_hdp_version
+from resource_management.libraries.functions.version import compare_versions, format_hdp_stack_version
+from resource_management.libraries.script.script import Script
+
from tez import tez
-from ambari_commons import OSConst
-from ambari_commons.os_family_impl import OsFamilyImpl
class TezClient(Script):
def configure(self, env):
@@ -56,12 +65,39 @@ class TezClientLinux(TezClient):
@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
class TezClientWindows(TezClient):
-
def install(self, env):
import params
if params.tez_home_dir is None:
self.install_packages(env)
+ params.refresh_tez_state_dependent_params()
+ env.set_params(params)
+ self._install_lzo_support_if_needed(params)
self.configure(env)
+ def _install_lzo_support_if_needed(self, params):
+ hadoop_classpath_prefix = self._expand_hadoop_classpath_prefix(params.hadoop_classpath_prefix_template, params.config['configurations']['tez-site'])
+
+ hadoop_lzo_dest_path = extract_path_component(hadoop_classpath_prefix, "hadoop-lzo-")
+ if hadoop_lzo_dest_path:
+ hadoop_lzo_file = os.path.split(hadoop_lzo_dest_path)[1]
+
+ config = Script.get_config()
+ file_url = urlparse.urljoin(config['hostLevelParams']['jdk_location'], hadoop_lzo_file)
+ hadoop_lzo_dl_path = os.path.join(config["hostLevelParams"]["agentCacheDir"], hadoop_lzo_file)
+ download_file(file_url, hadoop_lzo_dl_path)
+ #This is for protection against configuration changes. It will infect every new destination with the lzo jar,
+ # but since the classpath points to the jar directly we're getting away with it.
+ if not os.path.exists(hadoop_lzo_dest_path):
+ copy_file(hadoop_lzo_dl_path, hadoop_lzo_dest_path)
+
+ def _expand_hadoop_classpath_prefix(self, hadoop_classpath_prefix_template, configurations):
+ import resource_management
+
+ hadoop_classpath_prefix_obj = InlineTemplate(hadoop_classpath_prefix_template, configurations_dict=configurations,
+ extra_imports=[resource_management, resource_management.core,
+ resource_management.core.source])
+ hadoop_classpath_prefix = hadoop_classpath_prefix_obj.get_content()
+ return hadoop_classpath_prefix
+
if __name__ == "__main__":
TezClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json
index dc55533..daa7f80 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/role_command_order.json
@@ -72,6 +72,7 @@
"YARN_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"],
"RESOURCEMANAGER_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START"],
"PIG_SERVICE_CHECK-SERVICE_CHECK": ["RESOURCEMANAGER-START", "NODEMANAGER-START"],
+ "TEZ_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START", "HISTORYSERVER-START"],
"NAMENODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP",
"NODEMANAGER-STOP", "HISTORYSERVER-STOP", "HBASE_MASTER-STOP"],
"DATANODE-STOP": ["JOBTRACKER-STOP", "TASKTRACKER-STOP", "RESOURCEMANAGER-STOP",
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml
index fd7e7b0..55f3eaf 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/configuration/tez-env.xml
@@ -30,7 +30,6 @@
<property>
<name>content</name>
<description>This is the jinja template for tez-env.cmd file</description>
- <value>
- </value>
+ <value></value>
</property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml
index 50345fd..3f8c368 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.2/services/TEZ/configuration/tez-site.xml
@@ -18,7 +18,6 @@
-->
<configuration supports_final="true" supports_do_not_extend="true">
-
<property>
<name>tez.lib.uris</name>
<value>/hdp/apps/${hdp.version}/tez/tez.tar.gz</value>
@@ -47,12 +46,6 @@
</property>
<property>
- <name>tez.staging-dir</name>
- <value>/tmp/${user.name}/staging</value>
- <description>The staging dir used while submitting DAGs</description>
- </property>
-
- <property>
<name>tez.am.resource.memory.mb</name>
<value>1536</value>
<description>The amount of memory to be used by the AppMaster.
@@ -76,7 +69,7 @@
<property>
<name>tez.am.launch.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ <value>LD_LIBRARY_PATH=C:\hdp\hadoop\bin;C:\hdp\hadoop\share\hadoop\common\lib</value>
<description>
Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if
you want to have access to native libraries.
@@ -108,7 +101,7 @@
<property>
<name>tez.task.launch.env</name>
- <value>LD_LIBRARY_PATH=/usr/hdp/${hdp.version}/hadoop/lib/native:/usr/hdp/${hdp.version}/hadoop/lib/native/Linux-amd64-64</value>
+ <value>LD_LIBRARY_PATH=C:\hdp\hadoop\bin;C:\hdp\hadoop\share\hadoop\common\lib</value>
<description>
Additional execution environment entries for tez. This is not an additive property. You must preserve the original value if
you want to have access to native libraries.
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml
index e55b1de..a16994a 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/HIVE/configuration/hive-site.xml
@@ -205,15 +205,6 @@ limitations under the License.
</property>
<property>
- <name>hive.execution.engine</name>
- <value>mr</value>
- <description>
- Expects one of [mr, tez].
- Chooses execution engine. Options are: mr (Map reduce, default) or tez (hadoop 2 only)
- </description>
- </property>
-
- <property>
<name>hive.compute.query.using.stats</name>
<value>true</value>
<description>
http://git-wip-us.apache.org/repos/asf/ambari/blob/2dbb7b28/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml
index be534f1..8f4ff63 100644
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml
+++ b/ambari-server/src/main/resources/stacks/HDPWIN/2.3/services/TEZ/configuration/tez-site.xml
@@ -27,6 +27,20 @@
</property>
<property>
+ <name>tez.cluster.additional.classpath.prefix</name>
+ <value>C:\hdp\hadoop\share\hadoop\common\lib\hadoop-lzo-0.4.19.{{hdp_stack_version}}.jar</value>
+ </property>
+
+ <property>
+ <name>tez.lib.uris</name>
+ <value>file:///C:/hdp/tez-0.7.0.{{hdp_stack_version}}/tez-0.7.0.{{hdp_stack_version}}.tar.gz</value>
+ <description>Comma-delimited list of the location of the Tez libraries which will be localized for DAGs.
+ Specifying a single .tar.gz or .tgz assumes that a compressed version of the tez libs is being used. This is uncompressed into a tezlibs directory when running containers, and tezlibs/;tezlibs/lib/ are added to the classpath (after . and .*).
+ If multiple files are specified - files are localized as regular files, contents of directories are localized as regular files (non-recursive).
+ </description>
+ </property>
+
+ <property>
<name>tez.runtime.optimize.local.fetch</name>
<value>true</value>
<description>If the shuffle input is on the local host bypass the http fetch and access the files directly.</description>
@@ -94,5 +108,4 @@
</property>
</depends-on>
</property>
-
</configuration>