You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2015/03/27 17:03:08 UTC
[2/4] ambari git commit: AMBARI-10225: [WinTP2] Merge HDPWIN TEZ
package scripts to common services (jluniya)
AMBARI-10225: [WinTP2] Merge HDPWIN TEZ package scripts to common services (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/194df087
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/194df087
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/194df087
Branch: refs/heads/trunk
Commit: 194df087131fe9a254e6c0e2c47499305bc72075
Parents: e4686db
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Fri Mar 27 08:54:00 2015 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Fri Mar 27 08:54:00 2015 -0700
----------------------------------------------------------------------
.../TEZ/0.4.0.2.1/package/scripts/params.py | 58 ++-------------
.../0.4.0.2.1/package/scripts/params_linux.py | 76 ++++++++++++++++++++
.../0.4.0.2.1/package/scripts/params_windows.py | 30 ++++++++
.../0.4.0.2.1/package/scripts/service_check.py | 14 ++++
.../TEZ/0.4.0.2.1/package/scripts/tez.py | 21 +++---
.../TEZ/0.4.0.2.1/package/scripts/tez_client.py | 26 +++++--
.../2.1/services/TEZ/package/scripts/params.py | 30 --------
.../2.1/services/TEZ/package/scripts/tez.py | 31 --------
.../services/TEZ/package/scripts/tez_client.py | 42 -----------
9 files changed, 155 insertions(+), 173 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py
index b58bcd6..a10c1d4 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params.py
@@ -17,59 +17,9 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+from ambari_commons import OSCheck
-from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
-from resource_management import *
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-stack_name = default("/hostLevelParams/stack_name", None)
-
-# This is expected to be of the form #.#.#.#
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
-
-# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
-version = default("/commandParams/version", None)
-
-if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
- path_to_tez_examples_jar = "/usr/hdp/{hdp_version}/tez/tez-examples*.jar"
+if OSCheck.is_windows_family():
+ from params_windows import *
else:
- hadoop_bin_dir = "/usr/bin"
- path_to_tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar"
-hadoop_conf_dir = "/etc/hadoop/conf"
-
-kinit_path_local = functions.get_kinit_path()
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-
-config_dir_prefix = "/etc/tez"
-config_dir = format("{config_dir_prefix}/conf")
-
-hadoop_home = '/usr'
-java64_home = config['hostLevelParams']['java_home']
-
-tez_user = config['configurations']['tez-env']['tez_user']
-user_group = config['configurations']['cluster-env']['user_group']
-tez_env_sh_template = config['configurations']['tez-env']['content']
-
-import functools
-# Create partial functions with common arguments for every HdfsDirectory call
-# to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled=security_enabled,
- keytab=hdfs_user_keytab,
- kinit_path_local=kinit_path_local,
- bin_dir=hadoop_bin_dir
-)
+ from params_linux import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
new file mode 100644
index 0000000..3245ff0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_linux.py
@@ -0,0 +1,76 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+
+# This is expected to be of the form #.#.#.#
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+ hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+ path_to_tez_examples_jar = "/usr/hdp/{hdp_version}/tez/tez-examples*.jar"
+else:
+ hadoop_bin_dir = "/usr/bin"
+ path_to_tez_examples_jar = "/usr/lib/tez/tez-mapreduce-examples*.jar"
+hadoop_conf_dir = "/etc/hadoop/conf"
+
+kinit_path_local = functions.get_kinit_path()
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+
+config_dir_prefix = "/etc/tez"
+config_dir = format("{config_dir_prefix}/conf")
+
+hadoop_home = '/usr'
+java64_home = config['hostLevelParams']['java_home']
+
+tez_user = config['configurations']['tez-env']['tez_user']
+user_group = config['configurations']['cluster-env']['user_group']
+tez_env_sh_template = config['configurations']['tez-env']['content']
+
+import functools
+# Create partial functions with common arguments for every HdfsDirectory call
+# to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_user,
+ security_enabled=security_enabled,
+ keytab=hdfs_user_keytab,
+ kinit_path_local=kinit_path_local,
+ bin_dir=hadoop_bin_dir
+)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
new file mode 100644
index 0000000..db34c9f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/params_windows.py
@@ -0,0 +1,30 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import os
+
+config = Script.get_config()
+tez_user = "hadoop"
+tez_home_dir = None
+tez_conf_dir = "conf"
+
+if os.environ.has_key("TEZ_HOME"):
+ tez_home_dir = os.environ["TEZ_HOME"]
+ tez_conf_dir = os.path.join(tez_home_dir, "conf")
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
index 434b505..35ae5b6 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/service_check.py
@@ -22,8 +22,14 @@ Ambari Agent
from resource_management import *
from resource_management.libraries.functions.version import compare_versions
from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl
class TezServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TezServiceCheckLinux(TezServiceCheck):
def service_check(self, env):
import params
env.set_params(params)
@@ -91,6 +97,14 @@ class TezServiceCheck(Script):
)
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class TezServiceCheckWindows(TezServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd")
+ service = "TEZ"
+ Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.tez_user)
if __name__ == "__main__":
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
index bb8b7c2..ceafa56 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez.py
@@ -20,7 +20,10 @@ Ambari Agent
"""
from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def tez():
import params
@@ -48,14 +51,14 @@ def tez():
)
-def tez_TemplateConfig(name):
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def tez():
import params
-
- if not isinstance(name, list):
- name = [name]
-
- for x in name:
- TemplateConfig(format("{config_dir}/{x}"),
- owner = params.tez_user
- )
+ XmlConfig("tez-site.xml",
+ conf_dir=params.tez_conf_dir,
+ configurations=params.config['configurations']['tez-site'],
+ owner=params.tez_user,
+ mode="f",
+ configuration_attributes=params.config['configuration_attributes']['tez-site']
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
index 00375d7..9746d13 100644
--- a/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
+++ b/ambari-server/src/main/resources/common-services/TEZ/0.4.0.2.1/package/scripts/tez_client.py
@@ -22,8 +22,20 @@ Ambari Agent
import sys
from resource_management import *
from tez import tez
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyImpl
class TezClient(Script):
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ tez()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class TezClientLinux(TezClient):
def get_stack_to_component(self):
return {"HDP": "hadoop-client"}
@@ -39,14 +51,14 @@ class TezClient(Script):
self.install_packages(env)
self.configure(env)
- def configure(self, env):
- import params
- env.set_params(params)
- tez()
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class TezClientWindows(TezClient):
+ def install(self, env):
+ import params
+ if params.tez_home_dir is None:
+ self.install_packages(env)
+ self.configure(env)
if __name__ == "__main__":
TezClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py
deleted file mode 100644
index db34c9f..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/params.py
+++ /dev/null
@@ -1,30 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import os
-
-config = Script.get_config()
-tez_user = "hadoop"
-tez_home_dir = None
-tez_conf_dir = "conf"
-
-if os.environ.has_key("TEZ_HOME"):
- tez_home_dir = os.environ["TEZ_HOME"]
- tez_conf_dir = os.path.join(tez_home_dir, "conf")
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py
deleted file mode 100644
index d9f5a38..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import sys
-
-def tez():
- import params
- XmlConfig("tez-site.xml",
- conf_dir=params.tez_conf_dir,
- configurations=params.config['configurations']['tez-site'],
- owner=params.tez_user,
- mode="f",
- configuration_attributes=params.config['configuration_attributes']['tez-site']
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/194df087/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
deleted file mode 100644
index 89dd6bb..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/TEZ/package/scripts/tez_client.py
+++ /dev/null
@@ -1,42 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import sys
-from resource_management import *
-from tez import tez
-
-class TezClient(Script):
- def install(self, env):
- import params
- if params.tez_home_dir is None:
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- tez()
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- TezClient().execute()