You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2015/03/27 17:03:09 UTC
[3/4] ambari git commit: AMBARI-10226: [WinTP2] Merge HDPWIN PIG
package scripts to common services (jluniya)
AMBARI-10226: [WinTP2] Merge HDPWIN PIG package scripts to common services (jluniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/626ef03d
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/626ef03d
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/626ef03d
Branch: refs/heads/trunk
Commit: 626ef03daf847ad4e480ce5eb27fca7f6f9fcb81
Parents: 194df08
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Fri Mar 27 08:58:57 2015 -0700
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Fri Mar 27 08:58:57 2015 -0700
----------------------------------------------------------------------
.../PIG/0.12.0.2.0/package/scripts/params.py | 60 +--------------
.../0.12.0.2.0/package/scripts/params_linux.py | 78 ++++++++++++++++++++
.../package/scripts/params_windows.py | 36 +++++++++
.../PIG/0.12.0.2.0/package/scripts/pig.py | 20 ++++-
.../0.12.0.2.0/package/scripts/pig_client.py | 26 +++++--
.../0.12.0.2.0/package/scripts/service_check.py | 16 ++++
.../2.1/services/PIG/package/scripts/params.py | 37 ----------
.../2.1/services/PIG/package/scripts/pig.py | 49 ------------
.../services/PIG/package/scripts/pig_client.py | 41 ----------
.../PIG/package/scripts/service_check.py | 34 ---------
10 files changed, 172 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
index 2be49fa..9e3148e 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
@@ -18,61 +18,9 @@ limitations under the License.
Ambari Agent
"""
+from ambari_commons import OSCheck
-from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
-from resource_management import *
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-stack_name = default("/hostLevelParams/stack_name", None)
-
-stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
-
-# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
-version = default("/commandParams/version", None)
-
-#hadoop params
-if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
- hadoop_home = '/usr/hdp/current/hadoop-client'
- pig_bin_dir = '/usr/hdp/current/pig-client/bin'
+if OSCheck.is_windows_family():
+ from params_windows import *
else:
- hadoop_bin_dir = "/usr/bin"
- hadoop_home = '/usr'
- pig_bin_dir = ""
-
-hadoop_conf_dir = "/etc/hadoop/conf"
-pig_conf_dir = "/etc/pig/conf"
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
-user_group = config['configurations']['cluster-env']['user_group']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-kinit_path_local = functions.get_kinit_path()
-pig_env_sh_template = config['configurations']['pig-env']['content']
-
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
-
-pig_properties = config['configurations']['pig-properties']['content']
-
-log4j_props = config['configurations']['pig-log4j']['content']
-
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- bin_dir = hadoop_bin_dir
-)
+ from params_linux import *
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..2be49fa
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_linux.py
@@ -0,0 +1,78 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+
+stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+#hadoop params
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+ hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+ hadoop_home = '/usr/hdp/current/hadoop-client'
+ pig_bin_dir = '/usr/hdp/current/pig-client/bin'
+else:
+ hadoop_bin_dir = "/usr/bin"
+ hadoop_home = '/usr'
+ pig_bin_dir = ""
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+pig_conf_dir = "/etc/pig/conf"
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+user_group = config['configurations']['cluster-env']['user_group']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path()
+pig_env_sh_template = config['configurations']['pig-env']['content']
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+log4j_props = config['configurations']['pig-log4j']['content']
+
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ bin_dir = hadoop_bin_dir
+)
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..3e63083
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params_windows.py
@@ -0,0 +1,36 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+pig_home = os.environ['PIG_HOME']
+pig_conf_dir = os.path.join(pig_home,'conf')
+pig_properties = config['configurations']['pig-properties']['content']
+
+if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
+ log4j_props = config['configurations']['pig-log4j']['content']
+else:
+ log4j_props = None
+
+pig_user = "hadoop"
+hdfs_user = "hadoop"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
index 036b930..c5f095f 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
@@ -19,9 +19,11 @@ Ambari Agent
"""
import os
-
from resource_management import *
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def pig():
import params
@@ -58,3 +60,19 @@ def pig():
group=params.user_group,
owner=params.hdfs_user
)
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def pig():
+ import params
+ File(os.path.join(params.pig_conf_dir, "pig.properties"),
+ mode="f",
+ owner=params.pig_user,
+ content=params.pig_properties
+ )
+
+ if (params.log4j_props != None):
+ File(os.path.join(params.pig_conf_dir, "log4j.properties"),
+ mode='f',
+ owner=params.pig_user,
+ content=params.log4j_props
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
index eec9304..5b81b1f 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
@@ -20,12 +20,23 @@ Ambari Agent
"""
import sys
+import os
from resource_management import *
from pig import pig
-
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
class PigClient(Script):
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ pig()
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigClientLinux(PigClient):
def get_stack_to_component(self):
return {"HDP": "hadoop-client"}
@@ -40,13 +51,14 @@ class PigClient(Script):
self.install_packages(env)
self.configure(env)
- def configure(self, env):
- import params
- env.set_params(params)
- pig()
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigClientWindows(PigClient):
- def status(self, env):
- raise ClientComponentHasNoStatus()
+ def install(self, env):
+ import params
+ if params.pig_home is None:
+ self.install_packages(env)
+ self.configure(env)
if __name__ == "__main__":
PigClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
index 4fbfeb4..9de30ed 100644
--- a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
@@ -21,8 +21,15 @@ Ambari Agent
from resource_management import *
from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+from resource_management.libraries import functions
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
class PigServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigServiceCheckLinux(PigServiceCheck):
def service_check(self, env):
import params
env.set_params(params)
@@ -104,6 +111,15 @@ class PigServiceCheck(Script):
bin_dir = params.hadoop_bin_dir
)
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigServiceCheckWindows(PigServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd")
+ service = "PIG"
+ Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user)
+
if __name__ == "__main__":
PigServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py
deleted file mode 100644
index 854d577..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/params.py
+++ /dev/null
@@ -1,37 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-
-# server configurations
-config = Script.get_config()
-hdp_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
-pig_home = os.environ['PIG_HOME']
-pig_conf_dir = os.path.join(pig_home,'conf')
-pig_properties = config['configurations']['pig-properties']['pig-content']
-
-if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
- log4j_props = config['configurations']['pig-log4j']['content']
-else:
- log4j_props = None
-
-pig_user = "hadoop"
-hdfs_user = "hadoop"
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
deleted file mode 100644
index 47bb40f..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig.py
+++ /dev/null
@@ -1,49 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import os
-from resource_management import *
-
-def pig():
- import params
- File(os.path.join(params.pig_conf_dir, "pig.properties"),
- mode="f",
- owner=params.pig_user,
- content=params.pig_properties
- )
-
- if (params.log4j_props != None):
- File(os.path.join(params.pig_conf_dir, "log4j.properties"),
- mode='f',
- owner=params.pig_user,
- content=params.log4j_props
- )
-
-def pig_TemplateConfig(name):
- import params
-
- if not isinstance(name, list):
- name = [name]
-
- for x in name:
- TemplateConfig(os.path.join(params.pig_conf_dir,x),
- owner = params.hdfs_user
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py
deleted file mode 100644
index 381bfe9..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/pig_client.py
+++ /dev/null
@@ -1,41 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import os
-from resource_management import *
-
-
-class PigClient(Script):
- def install(self, env):
- import params
- if params.pig_home is None:
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- PigClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/626ef03d/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py
deleted file mode 100644
index 9e519c5..0000000
--- a/ambari-server/src/main/resources/stacks/HDPWIN/2.1/services/PIG/package/scripts/service_check.py
+++ /dev/null
@@ -1,34 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-from resource_management.libraries import functions
-
-class PigServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
- smoke_cmd = os.path.join(params.hdp_root,"Run-SmokeTests.cmd")
- service = "PIG"
- Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True, user=params.hdfs_user)
-
-if __name__ == "__main__":
- PigServiceCheck().execute()