You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ao...@apache.org on 2017/04/06 09:33:28 UTC
[07/41] ambari git commit: AMBARI-20652. HDP 3.0 TP - add Pig client:
configs, metainfo, descriptors, service advisor and etc.(vbrodetskyi)
AMBARI-20652. HDP 3.0 TP - add Pig client: configs, metainfo, descriptors, service advisor and etc.(vbrodetskyi)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/27da4c8f
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/27da4c8f
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/27da4c8f
Branch: refs/heads/branch-3.0-perf
Commit: 27da4c8f2ac1d6b5ae93091bbdc04603bb8d4244
Parents: e0412e8
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Mon Apr 3 15:37:51 2017 +0300
Committer: Andrew Onishuk <ao...@hortonworks.com>
Committed: Thu Apr 6 12:32:56 2017 +0300
----------------------------------------------------------------------
.../PIG/0.16.1.3.0/configuration/pig-env.xml | 41 +++++
.../PIG/0.16.1.3.0/configuration/pig-log4j.xml | 65 +++++++
.../0.16.1.3.0/configuration/pig-properties.xml | 93 ++++++++++
.../PIG/0.16.1.3.0/kerberos.json | 17 ++
.../common-services/PIG/0.16.1.3.0/metainfo.xml | 101 +++++++++++
.../PIG/0.16.1.3.0/package/files/pigSmoke.sh | 18 ++
.../PIG/0.16.1.3.0/package/scripts/params.py | 31 ++++
.../0.16.1.3.0/package/scripts/params_linux.py | 104 +++++++++++
.../package/scripts/params_windows.py | 45 +++++
.../PIG/0.16.1.3.0/package/scripts/pig.py | 80 +++++++++
.../0.16.1.3.0/package/scripts/pig_client.py | 71 ++++++++
.../0.16.1.3.0/package/scripts/service_check.py | 142 +++++++++++++++
.../PIG/0.16.1.3.0/service_advisor.py | 171 +++++++++++++++++++
.../stacks/HDP/3.0/services/PIG/metainfo.xml | 27 +++
14 files changed, 1006 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
new file mode 100644
index 0000000..11afa11
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-env.xml
@@ -0,0 +1,41 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <!-- pig-env.sh -->
+ <property>
+ <name>content</name>
+ <display-name>pig-env template</display-name>
+ <description>This is the jinja template for pig-env.sh file</description>
+ <value>
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+if [ -d "/usr/lib/tez" ]; then
+ PIG_OPTS="$PIG_OPTS -Dmapreduce.framework.name=yarn"
+fi
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="fallse"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
new file mode 100644
index 0000000..e5ff9b6
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-log4j.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>pig-log4j template</display-name>
+ <description>Custom log4j.properties</description>
+ <value>
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
new file mode 100644
index 0000000..fb43f6e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/configuration/pig-properties.xml
@@ -0,0 +1,93 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="false" supports_adding_forbidden="true">
+ <property>
+ <name>content</name>
+ <display-name>pig-properties template</display-name>
+ <description>Describe all the Pig agent configurations</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce|tez, mapreduce is default
+exectype=tez
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+ </value>
+ <value-attributes>
+ <type>content</type>
+ <empty-value-valid>true</empty-value-valid>
+ <show-property-name>false</show-property-name>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
new file mode 100644
index 0000000..22dd6cb
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/kerberos.json
@@ -0,0 +1,17 @@
+{
+ "services": [
+ {
+ "name": "PIG",
+ "components": [
+ {
+ "name": "PIG",
+ "identities": [
+ {
+ "name": "/HDFS/NAMENODE/hdfs"
+ }
+ ]
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
new file mode 100644
index 0000000..041255a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/metainfo.xml
@@ -0,0 +1,101 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PIG</name>
+ <displayName>Pig</displayName>
+ <comment>Scripting platform for analyzing large datasets</comment>
+ <version>0.16.1.3.0</version>
+ <components>
+ <component>
+ <name>PIG</name>
+ <displayName>Pig Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>0+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <commandScript>
+ <script>scripts/pig_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>1200</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>env</type>
+ <fileName>pig-env.sh</fileName>
+ <dictionaryName>pig-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>log4j.properties</fileName>
+ <dictionaryName>pig-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>pig.properties</fileName>
+ <dictionaryName>pig-properties</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>pig_${stack_version}</name>
+ </package>
+ <package>
+ <name>datafu_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>pig-${stack_version}</name>
+ </package>
+ <package>
+ <name>datafu-${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ <service>TEZ</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>pig-env</config-type>
+ <config-type>pig-log4j</config-type>
+ <config-type>pig-properties</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
new file mode 100644
index 0000000..a22456e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/files/pigSmoke.sh
@@ -0,0 +1,18 @@
+/*Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License */
+
+A = load 'passwd' using PigStorage(':');
+B = foreach A generate \$0 as id;
+store B into 'pigsmoke.out';
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
new file mode 100644
index 0000000..3aebda0
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params.py
@@ -0,0 +1,31 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.copy_tarball import get_sysprep_skip_copy_tarballs_hdfs
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+sysprep_skip_copy_tarballs_hdfs = get_sysprep_skip_copy_tarballs_hdfs()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..5af1274
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_linux.py
@@ -0,0 +1,104 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import get_kinit_path
+from resource_management.libraries.functions.get_not_managed_resources import get_not_managed_resources
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+stack_name = default("/hostLevelParams/stack_name", None)
+stack_root = Script.get_stack_root()
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+# hadoop default parameters
+hadoop_conf_dir = conf_select.get_hadoop_conf_dir()
+hadoop_bin_dir = stack_select.get_hadoop_dir("bin")
+pig_conf_dir = "/etc/pig/conf"
+hadoop_home = '/usr'
+pig_bin_dir = ""
+
+# hadoop parameters for stack versions supporting rolling_upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ pig_conf_dir = format("{stack_root}/current/pig-client/conf")
+ hadoop_home = stack_select.get_hadoop_dir("home")
+ pig_bin_dir = format("{stack_root}/current/pig-client/bin")
+
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+smoke_hdfs_user_mode = 0770
+user_group = config['configurations']['cluster-env']['user_group']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+pig_env_sh_template = config['configurations']['pig-env']['content']
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+log4j_props = config['configurations']['pig-log4j']['content']
+
+
+
+hdfs_site = config['configurations']['hdfs-site']
+default_fs = config['configurations']['core-site']['fs.defaultFS']
+
+dfs_type = default("/commandParams/dfs_type", "")
+
+import functools
+#create partial functions with common arguments for every HdfsResource call
+#to create hdfs directory we need to call params.HdfsResource in code
+HdfsResource = functools.partial(
+ HdfsResource,
+ user=hdfs_user,
+ hdfs_resource_ignore_file = "/var/lib/ambari-agent/data/.hdfs_resource_ignore",
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ hadoop_bin_dir = hadoop_bin_dir,
+ hadoop_conf_dir = hadoop_conf_dir,
+ principal_name = hdfs_principal_name,
+ hdfs_site = hdfs_site,
+ default_fs = default_fs,
+ immutable_paths = get_not_managed_resources(),
+ dfs_type = dfs_type
+ )
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..bd9654f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/params_windows.py
@@ -0,0 +1,45 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+from resource_management.libraries.script.script import Script
+
+# server configurations
+config = Script.get_config()
+
+stack_root = None
+pig_home = None
+pig_conf_dir = None
+try:
+ stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"],".."))
+ pig_home = os.environ['PIG_HOME']
+ pig_conf_dir = os.path.join(pig_home,'conf')
+except:
+ pass
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+if (('pig-log4j' in config['configurations']) and ('content' in config['configurations']['pig-log4j'])):
+ log4j_props = config['configurations']['pig-log4j']['content']
+else:
+ log4j_props = None
+
+hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
+pig_user = hadoop_user
+hdfs_user = hadoop_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
new file mode 100644
index 0000000..b6825d5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig.py
@@ -0,0 +1,80 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+from resource_management.core.resources.system import Directory, File
+from resource_management.core.source import InlineTemplate
+from resource_management.libraries.functions.format import format
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def pig():
+ import params
+
+ Directory( params.pig_conf_dir,
+ create_parents = True,
+ owner = params.hdfs_user,
+ group = params.user_group
+ )
+
+ File(format("{pig_conf_dir}/pig-env.sh"),
+ owner=params.hdfs_user,
+ mode=0755,
+ content=InlineTemplate(params.pig_env_sh_template)
+ )
+
+ # pig_properties is always set to a default even if it's not in the payload
+ File(format("{params.pig_conf_dir}/pig.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.pig_properties
+ )
+
+ if (params.log4j_props != None):
+ File(format("{params.pig_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
+ File(format("{params.pig_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user
+ )
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def pig():
+ import params
+ File(os.path.join(params.pig_conf_dir, "pig.properties"),
+ mode="f",
+ owner=params.pig_user,
+ content=params.pig_properties
+ )
+
+ if (params.log4j_props != None):
+ File(os.path.join(params.pig_conf_dir, "log4j.properties"),
+ mode='f',
+ owner=params.pig_user,
+ content=params.log4j_props
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
new file mode 100644
index 0000000..6fdcef8
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/pig_client.py
@@ -0,0 +1,71 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+import os
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select, stack_select
+from resource_management.libraries.functions.constants import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from pig import pig
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from resource_management.libraries.functions.default import default
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+
+class PigClient(Script):
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ pig()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigClientLinux(PigClient):
+ def get_component_name(self):
+ return "hadoop-client"
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+ conf_select.select(params.stack_name, "pig", params.version)
+ conf_select.select(params.stack_name, "hadoop", params.version)
+ stack_select.select("hadoop-client", params.version) # includes pig-client
+
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigClientWindows(PigClient):
+
+ def install(self, env):
+ import params
+ if params.pig_home is None:
+ self.install_packages(env)
+ self.configure(env)
+
+if __name__ == "__main__":
+ PigClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..ba4082e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/package/scripts/service_check.py
@@ -0,0 +1,142 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import os
+
+from resource_management.core.resources.system import Execute, File
+from resource_management.core.source import InlineTemplate, StaticFile
+from resource_management.libraries.functions.copy_tarball import copy_to_hdfs
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.resources.execute_hadoop import ExecuteHadoop
+from resource_management.libraries.resources.hdfs_resource import HdfsResource
+from resource_management.libraries.script.script import Script
+from ambari_commons import OSConst
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+
+class PigServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class PigServiceCheckLinux(PigServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ input_file = format('/user/{smokeuser}/passwd')
+ output_dir = format('/user/{smokeuser}/pigsmoke.out')
+
+ params.HdfsResource(format("/user/{smokeuser}"),
+ type="directory",
+ action="create_on_execute",
+ owner=params.smokeuser,
+ mode=params.smoke_hdfs_user_mode,
+ )
+
+ params.HdfsResource(output_dir,
+ type="directory",
+ action="delete_on_execute",
+ owner=params.smokeuser,
+ )
+ params.HdfsResource(input_file,
+ type="file",
+ source="/etc/passwd",
+ action="create_on_execute",
+ owner=params.smokeuser,
+ )
+ params.HdfsResource(None, action="execute")
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal};")
+ Execute(kinit_cmd,
+ user=params.smokeuser
+ )
+
+ File( format("{tmp_dir}/pigSmoke.sh"),
+ content = StaticFile("pigSmoke.sh"),
+ mode = 0755
+ )
+
+ # check for Pig-on-M/R
+ Execute( format("pig {tmp_dir}/pigSmoke.sh"),
+ tries = 3,
+ try_sleep = 5,
+ path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+ user = params.smokeuser,
+ logoutput = True
+ )
+
+ test_cmd = format("fs -test -e {output_dir}")
+ ExecuteHadoop( test_cmd,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+ if params.stack_version_formatted and check_stack_feature(StackFeature.PIG_ON_TEZ, params.stack_version_formatted):
+ # cleanup results from previous test
+ params.HdfsResource(output_dir,
+ type="directory",
+ action="delete_on_execute",
+ owner=params.smokeuser,
+ )
+ params.HdfsResource(input_file,
+ type="file",
+ source="/etc/passwd",
+ action="create_on_execute",
+ owner=params.smokeuser,
+ )
+
+ # Check for Pig-on-Tez
+ resource_created = copy_to_hdfs(
+ "tez", params.user_group,
+ params.hdfs_user,
+ skip=params.sysprep_skip_copy_tarballs_hdfs)
+ if resource_created:
+ params.HdfsResource(None, action="execute")
+
+ Execute(format("pig -x tez {tmp_dir}/pigSmoke.sh"),
+ tries = 3,
+ try_sleep = 5,
+ path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+ user = params.smokeuser,
+ logoutput = True
+ )
+
+ ExecuteHadoop(test_cmd,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class PigServiceCheckWindows(PigServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
+ service = "PIG"
+ Execute(format("cmd /C {smoke_cmd} {service}", smoke_cmd=smoke_cmd, service=service), logoutput=True, user=params.pig_user, timeout=300)
+
+if __name__ == "__main__":
+ PigServiceCheck().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
new file mode 100644
index 0000000..8f89c5d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.16.1.3.0/service_advisor.py
@@ -0,0 +1,171 @@
+#!/usr/bin/env ambari-python-wrap
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+# Python imports
+import imp
+import os
+import traceback
+import re
+import socket
+import fnmatch
+
+
+from resource_management.core.logger import Logger
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+STACKS_DIR = os.path.join(SCRIPT_DIR, '../../../stacks/')
+PARENT_FILE = os.path.join(STACKS_DIR, 'service_advisor.py')
+
+try:
+ with open(PARENT_FILE, 'rb') as fp:
+ service_advisor = imp.load_module('service_advisor', fp, PARENT_FILE, ('.py', 'rb', imp.PY_SOURCE))
+except Exception as e:
+ traceback.print_exc()
+ print "Failed to load parent"
+
+class PigServiceAdvisor(service_advisor.ServiceAdvisor):
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(PigServiceAdvisor, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ # Always call these methods
+ self.modifyMastersWithMultipleInstances()
+ self.modifyCardinalitiesDict()
+ self.modifyHeapSizeProperties()
+ self.modifyNotValuableComponents()
+ self.modifyComponentsNotPreferableOnServer()
+ self.modifyComponentLayoutSchemes()
+
+ def modifyMastersWithMultipleInstances(self):
+ """
+ Modify the set of masters with multiple instances.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyCardinalitiesDict(self):
+ """
+ Modify the dictionary of cardinalities.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyHeapSizeProperties(self):
+ """
+ Modify the dictionary of heap size properties.
+ Must be overriden in child class.
+ """
+ pass
+
+ def modifyNotValuableComponents(self):
+ """
+ Modify the set of components whose host assignment is based on other services.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentsNotPreferableOnServer(self):
+ """
+ Modify the set of components that are not preferable on the server.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def modifyComponentLayoutSchemes(self):
+ """
+ Modify layout scheme dictionaries for components.
+ The scheme dictionary basically maps the number of hosts to
+ host index where component should exist.
+ Must be overriden in child class.
+ """
+ # Nothing to do
+ pass
+
+ def getServiceComponentLayoutValidations(self, services, hosts):
+ """
+ Get a list of errors.
+ Must be overriden in child class.
+ """
+
+ return []
+
+ def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts):
+ """
+ Entry point.
+ Must be overriden in child class.
+ """
+ #Logger.info("Class: %s, Method: %s. Recommending Service Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ recommender = PigRecommender()
+
+
+ def getServiceConfigurationsValidationItems(self, configurations, recommendedDefaults, services, hosts):
+ """
+ Entry point.
+ Validate configurations for the service. Return a list of errors.
+ The code for this function should be the same for each Service Advisor.
+ """
+ #Logger.info("Class: %s, Method: %s. Validating Configurations." %
+ # (self.__class__.__name__, inspect.stack()[0][3]))
+
+ validator = PigValidator()
+ # Calls the methods of the validator using arguments,
+ # method(siteProperties, siteRecommendations, configurations, services, hosts)
+ return validator.validateListOfConfigUsingMethod(configurations, recommendedDefaults, services, hosts, validator.validators)
+
+
+
+class PigRecommender(service_advisor.ServiceAdvisor):
+ """
+ Pig Recommender suggests properties when adding the service for the first time or modifying configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(PigRecommender, self)
+ self.as_super.__init__(*args, **kwargs)
+
+
+
+
+
+
+
+class PigValidator(service_advisor.ServiceAdvisor):
+ """
+ Pig Validator checks the correctness of properties whenever the service is first added or the user attempts to
+ change configs via the UI.
+ """
+
+ def __init__(self, *args, **kwargs):
+ self.as_super = super(PigValidator, self)
+ self.as_super.__init__(*args, **kwargs)
+
+ self.validators = []
+
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/27da4c8f/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
new file mode 100644
index 0000000..fd664d0
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/PIG/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PIG</name>
+ <version>0.16.1.3.0</version>
+ <extends>common-services/PIG/0.16.1.3.0</extends>
+ </service>
+ </services>
+</metainfo>