You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2014/12/17 21:06:09 UTC
[37/37] ambari git commit: AMBARI-8747: Common Services: Refactor
HDP-2.0.6 PIG, SQOOP services (Jayush Luniya)
AMBARI-8747: Common Services: Refactor HDP-2.0.6 PIG, SQOOP services (Jayush Luniya)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/191232e2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/191232e2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/191232e2
Branch: refs/heads/trunk
Commit: 191232e290f0a3e9b0f3ac7e737cd374858d0941
Parents: 46e9bcb
Author: Jayush Luniya <jl...@hortonworks.com>
Authored: Wed Dec 17 12:00:52 2014 -0800
Committer: Jayush Luniya <jl...@hortonworks.com>
Committed: Wed Dec 17 12:00:52 2014 -0800
----------------------------------------------------------------------
.../PIG/0.12.0.2.0/configuration/pig-env.xml | 38 +++++++
.../PIG/0.12.0.2.0/configuration/pig-log4j.xml | 62 +++++++++++
.../0.12.0.2.0/configuration/pig-properties.xml | 89 +++++++++++++++
.../common-services/PIG/0.12.0.2.0/metainfo.xml | 85 +++++++++++++++
.../PIG/0.12.0.2.0/package/files/pigSmoke.sh | 18 ++++
.../PIG/0.12.0.2.0/package/scripts/params.py | 72 +++++++++++++
.../PIG/0.12.0.2.0/package/scripts/pig.py | 60 +++++++++++
.../0.12.0.2.0/package/scripts/pig_client.py | 41 +++++++
.../0.12.0.2.0/package/scripts/service_check.py | 107 +++++++++++++++++++
.../SQOOP/1.4.4.2.0/configuration/sqoop-env.xml | 54 ++++++++++
.../SQOOP/1.4.4.2.0/metainfo.xml | 92 ++++++++++++++++
.../SQOOP/1.4.4.2.0/package/scripts/__init__.py | 19 ++++
.../SQOOP/1.4.4.2.0/package/scripts/params.py | 53 +++++++++
.../1.4.4.2.0/package/scripts/service_check.py | 40 +++++++
.../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 51 +++++++++
.../1.4.4.2.0/package/scripts/sqoop_client.py | 41 +++++++
.../services/PIG/configuration/pig-env.xml | 38 -------
.../services/PIG/configuration/pig-log4j.xml | 62 -----------
.../PIG/configuration/pig-properties.xml | 89 ---------------
.../stacks/HDP/2.0.6/services/PIG/metainfo.xml | 61 +----------
.../services/PIG/package/files/pigSmoke.sh | 18 ----
.../services/PIG/package/scripts/params.py | 72 -------------
.../2.0.6/services/PIG/package/scripts/pig.py | 60 -----------
.../services/PIG/package/scripts/pig_client.py | 41 -------
.../PIG/package/scripts/service_check.py | 107 -------------------
.../services/SQOOP/configuration/sqoop-env.xml | 54 ----------
.../HDP/2.0.6/services/SQOOP/metainfo.xml | 68 +-----------
.../services/SQOOP/package/scripts/__init__.py | 19 ----
.../services/SQOOP/package/scripts/params.py | 53 ---------
.../SQOOP/package/scripts/service_check.py | 40 -------
.../services/SQOOP/package/scripts/sqoop.py | 51 ---------
.../SQOOP/package/scripts/sqoop_client.py | 41 -------
.../python/stacks/2.0.6/PIG/test_pig_client.py | 20 ++--
.../stacks/2.0.6/PIG/test_pig_service_check.py | 16 ++-
.../stacks/2.0.6/SQOOP/test_service_check.py | 14 ++-
.../python/stacks/2.0.6/SQOOP/test_sqoop.py | 8 +-
36 files changed, 965 insertions(+), 889 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-env.xml b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-env.xml
new file mode 100644
index 0000000..aded45f
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-env.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <!-- pig-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for pig-env.sh file</description>
+ <value>
+JAVA_HOME={{java64_home}}
+HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+if [ -d "/usr/lib/tez" ]; then
+ PIG_OPTS="$PIG_OPTS -Dmapreduce.framework.name=yarn"
+fi
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-log4j.xml b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-log4j.xml
new file mode 100644
index 0000000..4fe323c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-log4j.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+
+ <property>
+ <name>content</name>
+ <description>Custom log4j.properties</description>
+ <value>
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+#
+
+# ***** Set root logger level to DEBUG and its only appender to A.
+log4j.logger.org.apache.pig=info, A
+
+# ***** A is set to be a ConsoleAppender.
+log4j.appender.A=org.apache.log4j.ConsoleAppender
+# ***** A uses PatternLayout.
+log4j.appender.A.layout=org.apache.log4j.PatternLayout
+log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-properties.xml b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-properties.xml
new file mode 100644
index 0000000..59cc0c4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/configuration/pig-properties.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration supports_final="false">
+
+ <property>
+ <name>content</name>
+ <description>Describe all the Pig agent configurations</description>
+ <value>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
+# see bin/pig -help
+
+# brief logging (no timestamps)
+brief=false
+
+# debug level, INFO is default
+debug=INFO
+
+# verbose print all log messages to screen (default to print only INFO and above to screen)
+verbose=false
+
+# exectype local|mapreduce, mapreduce is default
+exectype=mapreduce
+
+# Enable insertion of information about script into hadoop job conf
+pig.script.info.enabled=true
+
+# Do not spill temp files smaller than this size (bytes)
+pig.spill.size.threshold=5000000
+
+# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
+# This should help reduce the number of files being spilled.
+pig.spill.gc.activation.size=40000000
+
+# the following two parameters are to help estimate the reducer number
+pig.exec.reducers.bytes.per.reducer=1000000000
+pig.exec.reducers.max=999
+
+# Temporary location to store the intermediate data.
+pig.temp.dir=/tmp/
+
+# Threshold for merging FRJoin fragment files
+pig.files.concatenation.threshold=100
+pig.optimistic.files.concatenation=false;
+
+pig.disable.counter=false
+
+hcat.bin=/usr/bin/hcat
+
+ </value>
+ </property>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/metainfo.xml
new file mode 100644
index 0000000..27bf492
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/metainfo.xml
@@ -0,0 +1,85 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>PIG</name>
+ <displayName>Pig</displayName>
+ <comment>Scripting platform for analyzing large datasets</comment>
+ <version>0.12.0.2.0</version>
+ <components>
+ <component>
+ <name>PIG</name>
+ <displayName>Pig</displayName>
+ <category>CLIENT</category>
+ <cardinality>0+</cardinality>
+ <commandScript>
+ <script>scripts/pig_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>600</timeout>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>env</type>
+ <fileName>pig-env.sh</fileName>
+ <dictionaryName>pig-env</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>log4j.properties</fileName>
+ <dictionaryName>pig-log4j</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>pig.properties</fileName>
+ <dictionaryName>pig-properties</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>pig</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>YARN</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>pig-env</config-type>
+ <config-type>pig-log4j</config-type>
+ <config-type>pig-properties</config-type>
+ </configuration-dependencies>
+
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/files/pigSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/files/pigSmoke.sh b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/files/pigSmoke.sh
new file mode 100644
index 0000000..a22456e
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/files/pigSmoke.sh
@@ -0,0 +1,18 @@
+/*Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License */
+
+A = load 'passwd' using PigStorage(':');
+B = foreach A generate \$0 as id;
+store B into 'pigsmoke.out';
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
new file mode 100644
index 0000000..57beac1
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/params.py
@@ -0,0 +1,72 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management import *
+
+# server configurations
+config = Script.get_config()
+tmp_dir = Script.get_tmp_dir()
+
+hdp_stack_version = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+
+#hadoop params
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+ hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
+ hadoop_home = '/usr/hdp/current/hadoop-client'
+ pig_bin_dir = '/usr/hdp/current/pig-client/bin'
+else:
+ hadoop_bin_dir = "/usr/bin"
+ hadoop_home = '/usr'
+ pig_bin_dir = ""
+
+hadoop_conf_dir = "/etc/hadoop/conf"
+pig_conf_dir = "/etc/pig/conf"
+hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
+hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
+hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+user_group = config['configurations']['cluster-env']['user_group']
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
+pig_env_sh_template = config['configurations']['pig-env']['content']
+
+# not supporting 32 bit jdk.
+java64_home = config['hostLevelParams']['java_home']
+
+pig_properties = config['configurations']['pig-properties']['content']
+
+log4j_props = config['configurations']['pig-log4j']['content']
+
+import functools
+#create partial functions with common arguments for every HdfsDirectory call
+#to create hdfs directory we need to call params.HdfsDirectory in code
+HdfsDirectory = functools.partial(
+ HdfsDirectory,
+ conf_dir=hadoop_conf_dir,
+ hdfs_user=hdfs_principal_name if security_enabled else hdfs_user,
+ security_enabled = security_enabled,
+ keytab = hdfs_user_keytab,
+ kinit_path_local = kinit_path_local,
+ bin_dir = hadoop_bin_dir
+)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
new file mode 100644
index 0000000..036b930
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig.py
@@ -0,0 +1,60 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+import os
+
+from resource_management import *
+
+def pig():
+ import params
+
+ Directory( params.pig_conf_dir,
+ recursive = True,
+ owner = params.hdfs_user,
+ group = params.user_group
+ )
+
+ File(format("{pig_conf_dir}/pig-env.sh"),
+ owner=params.hdfs_user,
+ mode=0755,
+ content=InlineTemplate(params.pig_env_sh_template)
+ )
+
+ # pig_properties is always set to a default even if it's not in the payload
+ File(format("{params.pig_conf_dir}/pig.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.pig_properties
+ )
+
+ if (params.log4j_props != None):
+ File(format("{params.pig_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user,
+ content=params.log4j_props
+ )
+ elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
+ File(format("{params.pig_conf_dir}/log4j.properties"),
+ mode=0644,
+ group=params.user_group,
+ owner=params.hdfs_user
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
new file mode 100644
index 0000000..931dceb
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/pig_client.py
@@ -0,0 +1,41 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+from pig import pig
+
+
+class PigClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ pig()
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ PigClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
new file mode 100644
index 0000000..a0e04ab
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/PIG/0.12.0.2.0/package/scripts/service_check.py
@@ -0,0 +1,107 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
+
+class PigServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+
+ input_file = 'passwd'
+ output_file = "pigsmoke.out"
+
+ cleanup_cmd = format("dfs -rmr {output_file} {input_file}")
+ #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
+ create_file_cmd = format("{cleanup_cmd}; hadoop --config {hadoop_conf_dir} dfs -put /etc/passwd {input_file} ") #TODO: inconsistent that second command needs hadoop
+ test_cmd = format("fs -test -e {output_file}")
+
+ ExecuteHadoop( create_file_cmd,
+ tries = 3,
+ try_sleep = 5,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ # for kinit run
+ keytab = params.smoke_user_keytab,
+ security_enabled = params.security_enabled,
+ kinit_path_local = params.kinit_path_local,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+ File( format("{tmp_dir}/pigSmoke.sh"),
+ content = StaticFile("pigSmoke.sh"),
+ mode = 0755
+ )
+
+ # check for Pig-on-M/R
+ Execute( format("pig {tmp_dir}/pigSmoke.sh"),
+ tries = 3,
+ try_sleep = 5,
+ path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+ user = params.smokeuser
+ )
+
+ ExecuteHadoop( test_cmd,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+ if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
+ # cleanup results from previous test
+ ExecuteHadoop( create_file_cmd,
+ tries = 3,
+ try_sleep = 5,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ # for kinit run
+ keytab = params.smoke_user_keytab,
+ security_enabled = params.security_enabled,
+ kinit_path_local = params.kinit_path_local,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+ # Check for Pig-on-Tez
+ copy_tarballs_to_hdfs('tez', params.smokeuser, params.hdfs_user, params.user_group)
+
+ if params.security_enabled:
+ kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
+ Execute(kinit_cmd,
+ user=params.smokeuser
+ )
+
+ Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"),
+ tries = 3,
+ try_sleep = 5,
+ path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
+ user = params.smokeuser
+ )
+
+ ExecuteHadoop( test_cmd,
+ user = params.smokeuser,
+ conf_dir = params.hadoop_conf_dir,
+ bin_dir = params.hadoop_bin_dir
+ )
+
+if __name__ == "__main__":
+ PigServiceCheck().execute()
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
new file mode 100644
index 0000000..9a740be
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-env.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+
+<configuration>
+ <!-- sqoop-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for sqoop-env.sh file</description>
+ <value>
+# Set Hadoop-specific environment variables here.
+
+#Set path to where bin/hadoop is available
+#Set path to where bin/hadoop is available
+export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+#set the path to where bin/hbase is available
+export HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}
+
+#Set the path to where bin/hive is available
+export HIVE_HOME=${HIVE_HOME:-{{hive_home}}}
+
+#Set the path for where zookeper config dir is
+export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
+
+# add libthrift in hive to sqoop class path first so hive imports work
+export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"
+ </value>
+ </property>
+ <property>
+ <name>sqoop_user</name>
+ <description>User to run Sqoop as</description>
+ <property-type>USER</property-type>
+ <value>sqoop</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
new file mode 100644
index 0000000..1f4a90b
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SQOOP</name>
+ <displayName>Sqoop</displayName>
+ <comment>Tool for transferring bulk data between Apache Hadoop and
+ structured data stores such as relational databases
+ </comment>
+ <version>1.4.4.2.0</version>
+
+ <components>
+ <component>
+ <name>SQOOP</name>
+ <displayName>Sqoop</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/sqoop_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>env</type>
+ <fileName>sqoop-env.sh</fileName>
+ <dictionaryName>sqoop-env</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>sqoop</name>
+ </package>
+ <package>
+ <name>mysql-connector-java</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>sqoop-env</config-type>
+ </configuration-dependencies>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
new file mode 100644
index 0000000..e8a3ac7
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params.py
@@ -0,0 +1,53 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
+from resource_management import *
+
+config = Script.get_config()
+
+hdp_stack_version = str(config['hostLevelParams']['stack_version'])
+hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
+
+#hadoop params
+if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
+ sqoop_conf_dir = '/etc/sqoop/conf'
+ sqoop_lib = '/usr/hdp/current/sqoop-client/lib'
+ hadoop_home = '/usr/hdp/current/hbase-client'
+ hbase_home = '/usr/hdp/current/hbase-client'
+ hive_home = '/usr/hdp/current/hive-client'
+ sqoop_bin_dir = '/usr/hdp/current/sqoop-client/bin/'
+else:
+ sqoop_conf_dir = "/usr/lib/sqoop/conf"
+ sqoop_lib = "/usr/lib/sqoop/lib"
+ hadoop_home = '/usr/lib/hadoop'
+ hbase_home = "/usr/lib/hbase"
+ hive_home = "/usr/lib/hive"
+ sqoop_bin_dir = "/usr/bin"
+
+zoo_conf_dir = "/etc/zookeeper"
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+user_group = config['configurations']['cluster-env']['user_group']
+sqoop_env_sh_template = config['configurations']['sqoop-env']['content']
+
+sqoop_user = config['configurations']['sqoop-env']['sqoop_user']
+
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py
new file mode 100644
index 0000000..bcd236c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/service_check.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+from resource_management import *
+
+
+class SqoopServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}"),
+ user = params.smokeuser,
+ )
+ Execute("sqoop version",
+ user = params.smokeuser,
+ path = params.sqoop_bin_dir,
+ logoutput = True
+ )
+
+if __name__ == "__main__":
+ SqoopServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
new file mode 100644
index 0000000..99ad575
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -0,0 +1,51 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management import *
+import os
+
+def sqoop(type=None):
+ import params
+ Link(params.sqoop_lib + "/mysql-connector-java.jar",
+ to = '/usr/share/java/mysql-connector-java.jar'
+ )
+ Directory(params.sqoop_conf_dir,
+ owner = params.sqoop_user,
+ group = params.user_group,
+ recursive = True
+ )
+ File(format("{sqoop_conf_dir}/sqoop-env.sh"),
+ owner=params.sqoop_user,
+ group = params.user_group,
+ content=InlineTemplate(params.sqoop_env_sh_template)
+ )
+ update_config_permissions(["sqoop-env-template.sh",
+ "sqoop-site-template.xml",
+ "sqoop-site.xml"])
+ pass
+
+def update_config_permissions(names):
+ import params
+ for filename in names:
+ full_filename = os.path.join(params.sqoop_conf_dir, filename)
+ File(full_filename,
+ owner = params.sqoop_user,
+ group = params.user_group,
+ only_if = format("test -e {full_filename}")
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py
new file mode 100644
index 0000000..6829557
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop_client.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import sys
+from resource_management import *
+
+from sqoop import sqoop
+
+
+class SqoopClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ sqoop(type='client')
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+if __name__ == "__main__":
+ SqoopClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-env.xml
deleted file mode 100644
index aded45f..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-env.xml
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <!-- pig-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for pig-env.sh file</description>
- <value>
-JAVA_HOME={{java64_home}}
-HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
-
-if [ -d "/usr/lib/tez" ]; then
- PIG_OPTS="$PIG_OPTS -Dmapreduce.framework.name=yarn"
-fi
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-log4j.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-log4j.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-log4j.xml
deleted file mode 100644
index 4fe323c..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-log4j.xml
+++ /dev/null
@@ -1,62 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Custom log4j.properties</description>
- <value>
-#
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-#
-#
-#
-
-# ***** Set root logger level to DEBUG and its only appender to A.
-log4j.logger.org.apache.pig=info, A
-
-# ***** A is set to be a ConsoleAppender.
-log4j.appender.A=org.apache.log4j.ConsoleAppender
-# ***** A uses PatternLayout.
-log4j.appender.A.layout=org.apache.log4j.PatternLayout
-log4j.appender.A.layout.ConversionPattern=%-4r [%t] %-5p %c %x - %m%n
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
deleted file mode 100644
index 59cc0c4..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/configuration/pig-properties.xml
+++ /dev/null
@@ -1,89 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration supports_final="false">
-
- <property>
- <name>content</name>
- <description>Describe all the Pig agent configurations</description>
- <value>
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# Pig default configuration file. All values can be overwritten by pig.properties and command line arguments.
-# see bin/pig -help
-
-# brief logging (no timestamps)
-brief=false
-
-# debug level, INFO is default
-debug=INFO
-
-# verbose print all log messages to screen (default to print only INFO and above to screen)
-verbose=false
-
-# exectype local|mapreduce, mapreduce is default
-exectype=mapreduce
-
-# Enable insertion of information about script into hadoop job conf
-pig.script.info.enabled=true
-
-# Do not spill temp files smaller than this size (bytes)
-pig.spill.size.threshold=5000000
-
-# EXPERIMENT: Activate garbage collection when spilling a file bigger than this size (bytes)
-# This should help reduce the number of files being spilled.
-pig.spill.gc.activation.size=40000000
-
-# the following two parameters are to help estimate the reducer number
-pig.exec.reducers.bytes.per.reducer=1000000000
-pig.exec.reducers.max=999
-
-# Temporary location to store the intermediate data.
-pig.temp.dir=/tmp/
-
-# Threshold for merging FRJoin fragment files
-pig.files.concatenation.threshold=100
-pig.optimistic.files.concatenation=false;
-
-pig.disable.counter=false
-
-hcat.bin=/usr/bin/hcat
-
- </value>
- </property>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
index 27bf492..c0e5aa9 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/metainfo.xml
@@ -20,66 +20,7 @@
<services>
<service>
<name>PIG</name>
- <displayName>Pig</displayName>
- <comment>Scripting platform for analyzing large datasets</comment>
- <version>0.12.0.2.0</version>
- <components>
- <component>
- <name>PIG</name>
- <displayName>Pig</displayName>
- <category>CLIENT</category>
- <cardinality>0+</cardinality>
- <commandScript>
- <script>scripts/pig_client.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>600</timeout>
- </commandScript>
- <configFiles>
- <configFile>
- <type>env</type>
- <fileName>pig-env.sh</fileName>
- <dictionaryName>pig-env</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>log4j.properties</fileName>
- <dictionaryName>pig-log4j</dictionaryName>
- </configFile>
- <configFile>
- <type>env</type>
- <fileName>pig.properties</fileName>
- <dictionaryName>pig-properties</dictionaryName>
- </configFile>
- </configFiles>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>pig</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
-
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>YARN</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>pig-env</config-type>
- <config-type>pig-log4j</config-type>
- <config-type>pig-properties</config-type>
- </configuration-dependencies>
-
+ <extends>common-services/PIG/0.12.0.2.0</extends>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/files/pigSmoke.sh
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/files/pigSmoke.sh b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/files/pigSmoke.sh
deleted file mode 100644
index a22456e..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/files/pigSmoke.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-/*Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License */
-
-A = load 'passwd' using PigStorage(':');
-B = foreach A generate \$0 as id;
-store B into 'pigsmoke.out';
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
deleted file mode 100644
index 57beac1..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/params.py
+++ /dev/null
@@ -1,72 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
-from resource_management import *
-
-# server configurations
-config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
-
-hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
-
-#hadoop params
-if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- hadoop_bin_dir = "/usr/hdp/current/hadoop-client/bin"
- hadoop_home = '/usr/hdp/current/hadoop-client'
- pig_bin_dir = '/usr/hdp/current/pig-client/bin'
-else:
- hadoop_bin_dir = "/usr/bin"
- hadoop_home = '/usr'
- pig_bin_dir = ""
-
-hadoop_conf_dir = "/etc/hadoop/conf"
-pig_conf_dir = "/etc/pig/conf"
-hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
-hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name']
-hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-user_group = config['configurations']['cluster-env']['user_group']
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
-pig_env_sh_template = config['configurations']['pig-env']['content']
-
-# not supporting 32 bit jdk.
-java64_home = config['hostLevelParams']['java_home']
-
-pig_properties = config['configurations']['pig-properties']['content']
-
-log4j_props = config['configurations']['pig-log4j']['content']
-
-import functools
-#create partial functions with common arguments for every HdfsDirectory call
-#to create hdfs directory we need to call params.HdfsDirectory in code
-HdfsDirectory = functools.partial(
- HdfsDirectory,
- conf_dir=hadoop_conf_dir,
- hdfs_user=hdfs_principal_name if security_enabled else hdfs_user,
- security_enabled = security_enabled,
- keytab = hdfs_user_keytab,
- kinit_path_local = kinit_path_local,
- bin_dir = hadoop_bin_dir
-)
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
deleted file mode 100644
index 036b930..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-import os
-
-from resource_management import *
-
-def pig():
- import params
-
- Directory( params.pig_conf_dir,
- recursive = True,
- owner = params.hdfs_user,
- group = params.user_group
- )
-
- File(format("{pig_conf_dir}/pig-env.sh"),
- owner=params.hdfs_user,
- mode=0755,
- content=InlineTemplate(params.pig_env_sh_template)
- )
-
- # pig_properties is always set to a default even if it's not in the payload
- File(format("{params.pig_conf_dir}/pig.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hdfs_user,
- content=params.pig_properties
- )
-
- if (params.log4j_props != None):
- File(format("{params.pig_conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hdfs_user,
- content=params.log4j_props
- )
- elif (os.path.exists(format("{params.pig_conf_dir}/log4j.properties"))):
- File(format("{params.pig_conf_dir}/log4j.properties"),
- mode=0644,
- group=params.user_group,
- owner=params.hdfs_user
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig_client.py
deleted file mode 100644
index 931dceb..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/pig_client.py
+++ /dev/null
@@ -1,41 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-import sys
-from resource_management import *
-from pig import pig
-
-
-class PigClient(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- pig()
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- PigClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
deleted file mode 100644
index a0e04ab..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/PIG/package/scripts/service_check.py
+++ /dev/null
@@ -1,107 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-Ambari Agent
-
-"""
-
-from resource_management import *
-from resource_management.libraries.functions.dynamic_variable_interpretation import copy_tarballs_to_hdfs
-
-class PigServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
-
- input_file = 'passwd'
- output_file = "pigsmoke.out"
-
- cleanup_cmd = format("dfs -rmr {output_file} {input_file}")
- #cleanup put below to handle retries; if retrying there wil be a stale file that needs cleanup; exit code is fn of second command
- create_file_cmd = format("{cleanup_cmd}; hadoop --config {hadoop_conf_dir} dfs -put /etc/passwd {input_file} ") #TODO: inconsistent that second command needs hadoop
- test_cmd = format("fs -test -e {output_file}")
-
- ExecuteHadoop( create_file_cmd,
- tries = 3,
- try_sleep = 5,
- user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir,
- # for kinit run
- keytab = params.smoke_user_keytab,
- security_enabled = params.security_enabled,
- kinit_path_local = params.kinit_path_local,
- bin_dir = params.hadoop_bin_dir
- )
-
- File( format("{tmp_dir}/pigSmoke.sh"),
- content = StaticFile("pigSmoke.sh"),
- mode = 0755
- )
-
- # check for Pig-on-M/R
- Execute( format("pig {tmp_dir}/pigSmoke.sh"),
- tries = 3,
- try_sleep = 5,
- path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
- user = params.smokeuser
- )
-
- ExecuteHadoop( test_cmd,
- user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir,
- bin_dir = params.hadoop_bin_dir
- )
-
- if params.hdp_stack_version != "" and compare_versions(params.hdp_stack_version, '2.2') >= 0:
- # cleanup results from previous test
- ExecuteHadoop( create_file_cmd,
- tries = 3,
- try_sleep = 5,
- user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir,
- # for kinit run
- keytab = params.smoke_user_keytab,
- security_enabled = params.security_enabled,
- kinit_path_local = params.kinit_path_local,
- bin_dir = params.hadoop_bin_dir
- )
-
- # Check for Pig-on-Tez
- copy_tarballs_to_hdfs('tez', params.smokeuser, params.hdfs_user, params.user_group)
-
- if params.security_enabled:
- kinit_cmd = format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser};")
- Execute(kinit_cmd,
- user=params.smokeuser
- )
-
- Execute( format("pig -x tez {tmp_dir}/pigSmoke.sh"),
- tries = 3,
- try_sleep = 5,
- path = format('{pig_bin_dir}:/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin'),
- user = params.smokeuser
- )
-
- ExecuteHadoop( test_cmd,
- user = params.smokeuser,
- conf_dir = params.hadoop_conf_dir,
- bin_dir = params.hadoop_bin_dir
- )
-
-if __name__ == "__main__":
- PigServiceCheck().execute()
-
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/configuration/sqoop-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/configuration/sqoop-env.xml
deleted file mode 100644
index 9a740be..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/configuration/sqoop-env.xml
+++ /dev/null
@@ -1,54 +0,0 @@
-<?xml version="1.0"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
--->
-
-<configuration>
- <!-- sqoop-env.sh -->
- <property>
- <name>content</name>
- <description>This is the jinja template for sqoop-env.sh file</description>
- <value>
-# Set Hadoop-specific environment variables here.
-
-#Set path to where bin/hadoop is available
-#Set path to where bin/hadoop is available
-export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
-
-#set the path to where bin/hbase is available
-export HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}
-
-#Set the path to where bin/hive is available
-export HIVE_HOME=${HIVE_HOME:-{{hive_home}}}
-
-#Set the path for where zookeper config dir is
-export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
-
-# add libthrift in hive to sqoop class path first so hive imports work
-export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"
- </value>
- </property>
- <property>
- <name>sqoop_user</name>
- <description>User to run Sqoop as</description>
- <property-type>USER</property-type>
- <value>sqoop</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/metainfo.xml
index 1f4a90b..1a9989e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/metainfo.xml
@@ -20,73 +20,7 @@
<services>
<service>
<name>SQOOP</name>
- <displayName>Sqoop</displayName>
- <comment>Tool for transferring bulk data between Apache Hadoop and
- structured data stores such as relational databases
- </comment>
- <version>1.4.4.2.0</version>
-
- <components>
- <component>
- <name>SQOOP</name>
- <displayName>Sqoop</displayName>
- <category>CLIENT</category>
- <cardinality>1+</cardinality>
- <dependencies>
- <dependency>
- <name>HDFS/HDFS_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- <dependency>
- <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
- <scope>host</scope>
- <auto-deploy>
- <enabled>true</enabled>
- </auto-deploy>
- </dependency>
- </dependencies>
- <commandScript>
- <script>scripts/sqoop_client.py</script>
- <scriptType>PYTHON</scriptType>
- </commandScript>
- <configFiles>
- <configFile>
- <type>env</type>
- <fileName>sqoop-env.sh</fileName>
- <dictionaryName>sqoop-env</dictionaryName>
- </configFile>
- </configFiles>
- </component>
- </components>
- <osSpecifics>
- <osSpecific>
- <osFamily>any</osFamily>
- <packages>
- <package>
- <name>sqoop</name>
- </package>
- <package>
- <name>mysql-connector-java</name>
- </package>
- </packages>
- </osSpecific>
- </osSpecifics>
- <commandScript>
- <script>scripts/service_check.py</script>
- <scriptType>PYTHON</scriptType>
- <timeout>300</timeout>
- </commandScript>
-
- <requiredServices>
- <service>HDFS</service>
- </requiredServices>
-
- <configuration-dependencies>
- <config-type>sqoop-env</config-type>
- </configuration-dependencies>
+ <extends>common-services/SQOOP/1.4.4.2.0</extends>
</service>
</services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/__init__.py
deleted file mode 100644
index 5561e10..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/__init__.py
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py
deleted file mode 100644
index e8a3ac7..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/params.py
+++ /dev/null
@@ -1,53 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management.libraries.functions.version import format_hdp_stack_version, compare_versions
-from resource_management import *
-
-config = Script.get_config()
-
-hdp_stack_version = str(config['hostLevelParams']['stack_version'])
-hdp_stack_version = format_hdp_stack_version(hdp_stack_version)
-
-#hadoop params
-if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0:
- sqoop_conf_dir = '/etc/sqoop/conf'
- sqoop_lib = '/usr/hdp/current/sqoop-client/lib'
- hadoop_home = '/usr/hdp/current/hbase-client'
- hbase_home = '/usr/hdp/current/hbase-client'
- hive_home = '/usr/hdp/current/hive-client'
- sqoop_bin_dir = '/usr/hdp/current/sqoop-client/bin/'
-else:
- sqoop_conf_dir = "/usr/lib/sqoop/conf"
- sqoop_lib = "/usr/lib/sqoop/lib"
- hadoop_home = '/usr/lib/hadoop'
- hbase_home = "/usr/lib/hbase"
- hive_home = "/usr/lib/hive"
- sqoop_bin_dir = "/usr/bin"
-
-zoo_conf_dir = "/etc/zookeeper"
-security_enabled = config['configurations']['cluster-env']['security_enabled']
-smokeuser = config['configurations']['cluster-env']['smokeuser']
-user_group = config['configurations']['cluster-env']['user_group']
-sqoop_env_sh_template = config['configurations']['sqoop-env']['content']
-
-sqoop_user = config['configurations']['sqoop-env']['sqoop_user']
-
-smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
-kinit_path_local = functions.get_kinit_path(["/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/service_check.py
deleted file mode 100644
index bcd236c..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/service_check.py
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-
-from resource_management import *
-
-
-class SqoopServiceCheck(Script):
- def service_check(self, env):
- import params
- env.set_params(params)
- if params.security_enabled:
- Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser}"),
- user = params.smokeuser,
- )
- Execute("sqoop version",
- user = params.smokeuser,
- path = params.sqoop_bin_dir,
- logoutput = True
- )
-
-if __name__ == "__main__":
- SqoopServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop.py
deleted file mode 100644
index 99ad575..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-from resource_management import *
-import os
-
-def sqoop(type=None):
- import params
- Link(params.sqoop_lib + "/mysql-connector-java.jar",
- to = '/usr/share/java/mysql-connector-java.jar'
- )
- Directory(params.sqoop_conf_dir,
- owner = params.sqoop_user,
- group = params.user_group,
- recursive = True
- )
- File(format("{sqoop_conf_dir}/sqoop-env.sh"),
- owner=params.sqoop_user,
- group = params.user_group,
- content=InlineTemplate(params.sqoop_env_sh_template)
- )
- update_config_permissions(["sqoop-env-template.sh",
- "sqoop-site-template.xml",
- "sqoop-site.xml"])
- pass
-
-def update_config_permissions(names):
- import params
- for filename in names:
- full_filename = os.path.join(params.sqoop_conf_dir, filename)
- File(full_filename,
- owner = params.sqoop_user,
- group = params.user_group,
- only_if = format("test -e {full_filename}")
- )
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop_client.py
deleted file mode 100644
index 6829557..0000000
--- a/ambari-server/src/main/resources/stacks/HDP/2.0.6/services/SQOOP/package/scripts/sqoop_client.py
+++ /dev/null
@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-"""
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-"""
-
-import sys
-from resource_management import *
-
-from sqoop import sqoop
-
-
-class SqoopClient(Script):
- def install(self, env):
- self.install_packages(env)
- self.configure(env)
-
- def configure(self, env):
- import params
- env.set_params(params)
- sqoop(type='client')
-
- def status(self, env):
- raise ClientComponentHasNoStatus()
-
-if __name__ == "__main__":
- SqoopClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/191232e2/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
index 0ed0364..2227708 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/PIG/test_pig_client.py
@@ -22,12 +22,16 @@ from stacks.utils.RMFTestCase import *
import json
class TestPigClient(RMFTestCase):
+ COMMON_SERVICES_PACKAGE_DIR = "PIG/0.12.0.2.0/package"
+ STACK_VERSION = "2.0.6"
def test_configure_default(self):
- self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
classname = "PigClient",
command = "configure",
- config_file="default.json"
+ config_file="default.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/etc/pig/conf',
@@ -57,10 +61,12 @@ class TestPigClient(RMFTestCase):
def test_configure_secured(self):
- self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
classname = "PigClient",
command = "configure",
- config_file="secured.json"
+ config_file="secured.json",
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/etc/pig/conf',
@@ -95,10 +101,12 @@ class TestPigClient(RMFTestCase):
default_json['hostLevelParams']['stack_version'] = '2.2'
- self.executeScript("2.0.6/services/PIG/package/scripts/pig_client.py",
+ self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/pig_client.py",
classname = "PigClient",
command = "configure",
- config_dict=default_json
+ config_dict=default_json,
+ hdp_stack_version = self.STACK_VERSION,
+ target = RMFTestCase.TARGET_COMMON_SERVICES
)
self.assertResourceCalled('Directory', '/etc/pig/conf',