You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by jl...@apache.org on 2017/05/17 20:51:39 UTC
[05/25] ambari git commit: AMBARI-21023. HDP 3.0 TP - create service
definition for Sqoop with configs, kerberos, widgets, etc.(vbrodetsky)
AMBARI-21023. HDP 3.0 TP - create service definition for Sqoop with configs, kerberos, widgets, etc.(vbrodetsky)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b3339123
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b3339123
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b3339123
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: b33391236b385e2e4620031b9773261bce8efdf6
Parents: 47b845f
Author: Vitaly Brodetskyi <vb...@hortonworks.com>
Authored: Tue May 16 22:16:59 2017 +0300
Committer: Vitaly Brodetskyi <vb...@hortonworks.com>
Committed: Tue May 16 22:16:59 2017 +0300
----------------------------------------------------------------------
.../sqoop-atlas-application.properties.xml | 47 +++++++
.../SQOOP/1.4.4.3.0/configuration/sqoop-env.xml | 87 ++++++++++++
.../1.4.4.3.0/configuration/sqoop-site.xml | 38 ++++++
.../SQOOP/1.4.4.3.0/kerberos.json | 20 +++
.../SQOOP/1.4.4.3.0/metainfo.xml | 115 ++++++++++++++++
.../SQOOP/1.4.4.3.0/package/scripts/__init__.py | 19 +++
.../SQOOP/1.4.4.3.0/package/scripts/params.py | 27 ++++
.../1.4.4.3.0/package/scripts/params_linux.py | 135 +++++++++++++++++++
.../1.4.4.3.0/package/scripts/params_windows.py | 30 +++++
.../1.4.4.3.0/package/scripts/service_check.py | 62 +++++++++
.../SQOOP/1.4.4.3.0/package/scripts/sqoop.py | 124 +++++++++++++++++
.../1.4.4.3.0/package/scripts/sqoop_client.py | 66 +++++++++
.../SQOOP/1.4.4.3.0/role_command_order.json | 6 +
.../stacks/HDP/3.0/services/SQOOP/metainfo.xml | 27 ++++
14 files changed, 803 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
new file mode 100644
index 0000000..1364776
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-atlas-application.properties.xml
@@ -0,0 +1,47 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!--
+ These two security properties will only be read if the cluster is Kerberized, but ok to add them even without Kerberos.
+ It's important to have at least one property in this config file so it gets added while merging configs during EU/RU
+ from an earlier stack to HDP 2.5+.
+ Also, it allows a fresh install with Sqoop to expose this config type in the UI.
+ -->
+ <property>
+ <name>atlas.jaas.KafkaClient.option.useTicketCache</name>
+ <value>true</value>
+ <description>
+ Set this to "true" if you want the TGT to be obtained from the ticket cache.
+ Set this option to "false" if you do not want this module to use the ticket cache.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>atlas.jaas.KafkaClient.option.renewTicket</name>
+ <value>true</value>
+ <description>
+ Set this to "true" if you want the TGT to renew the ticket when it expires.
+ Set this option to "false" if you do not want this module to renew tickets.
+ </description>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
new file mode 100644
index 0000000..6e16ab33
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-env.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_adding_forbidden="true">
+ <!-- sqoop-env.sh -->
+ <property>
+ <name>sqoop.atlas.hook</name>
+ <value>false</value>
+ <display-name>Enable Atlas Hook</display-name>
+ <description>Enable Atlas Hook</description>
+ <value-attributes>
+ <type>boolean</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
+ <name>content</name>
+ <display-name>sqoop-env template</display-name>
+ <description>This is the jinja template for sqoop-env.sh file</description>
+ <value>
+# Set Hadoop-specific environment variables here.
+
+#Set path to where bin/hadoop is available
+#Set path to where bin/hadoop is available
+export HADOOP_HOME=${HADOOP_HOME:-{{hadoop_home}}}
+
+#set the path to where bin/hbase is available
+export HBASE_HOME=${HBASE_HOME:-{{hbase_home}}}
+
+#Set the path to where bin/hive is available
+export HIVE_HOME=${HIVE_HOME:-{{hive_home}}}
+
+#Set the path for where zookeper config dir is
+export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
+
+# add libthrift in hive to sqoop class path first so hive imports work
+export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>sqoop_user</name>
+ <display-name>Sqoop User</display-name>
+ <description>User to run Sqoop as</description>
+ <property-type>USER</property-type>
+ <value>sqoop</value>
+ <value-attributes>
+ <type>user</type>
+ <overridable>false</overridable>
+ </value-attributes>
+ <on-ambari-upgrade add="false"/>
+ </property>
+ <property>
+ <name>jdbc_drivers</name>
+ <description>Comma separated list of additional JDBC drivers class names</description>
+ <value> </value>
+ <on-ambari-upgrade add="false"/>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
new file mode 100644
index 0000000..389550d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/configuration/sqoop-site.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <!-- This property was valid in HDP 2.3 and 2.4, but removed in HDP 2.5 -->
+ <property>
+ <name>sqoop.job.data.publish.class</name>
+ <on-ambari-upgrade add="false"/>
+ <depends-on>
+ <property>
+ <type>sqoop-env</type>
+ <name>sqoop.atlas.hook</name>
+ </property>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.rest.address</name>
+ </property>
+ </depends-on>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
new file mode 100644
index 0000000..de12e7c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/kerberos.json
@@ -0,0 +1,20 @@
+{
+ "services": [
+ {
+ "name": "SQOOP",
+ "configurations": [
+ {
+ "sqoop-atlas-application.properties": {
+ "atlas.jaas.KafkaClient.option.useTicketCache": "true",
+ "atlas.jaas.KafkaClient.option.renewTicket": "true"
+ }
+ }
+ ],
+ "components": [
+ {
+ "name": "SQOOP"
+ }
+ ]
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
new file mode 100644
index 0000000..999d93a
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/metainfo.xml
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SQOOP</name>
+ <displayName>Sqoop</displayName>
+ <comment>Tool for transferring bulk data between Apache Hadoop and
+ structured data stores such as relational databases
+ </comment>
+ <version>1.4.4.3.0</version>
+
+ <components>
+ <component>
+ <name>SQOOP</name>
+ <displayName>Sqoop Client</displayName>
+ <category>CLIENT</category>
+ <cardinality>1+</cardinality>
+ <versionAdvertised>true</versionAdvertised>
+ <dependencies>
+ <dependency>
+ <name>HDFS/HDFS_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ <dependency>
+ <name>MAPREDUCE2/MAPREDUCE2_CLIENT</name>
+ <scope>host</scope>
+ <auto-deploy>
+ <enabled>true</enabled>
+ </auto-deploy>
+ </dependency>
+ </dependencies>
+ <commandScript>
+ <script>scripts/sqoop_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
+ <configFiles>
+ <configFile>
+ <type>xml</type>
+ <fileName>sqoop-site.xml</fileName>
+ <dictionaryName>sqoop-site</dictionaryName>
+ </configFile>
+ <configFile>
+ <type>env</type>
+ <fileName>sqoop-env.sh</fileName>
+ <dictionaryName>sqoop-env</dictionaryName>
+ </configFile>
+ </configFiles>
+ </component>
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osFamily>any</osFamily>
+ <packages>
+ <package>
+ <name>mysql-connector-java</name>
+ <skipUpgrade>true</skipUpgrade>
+ <condition>should_install_mysql_connector</condition>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>redhat7,amazon2015,redhat6,suse11,suse12</osFamily>
+ <packages>
+ <package>
+ <name>sqoop_${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ <osSpecific>
+ <osFamily>debian7,ubuntu12,ubuntu14,ubuntu16</osFamily>
+ <packages>
+ <package>
+ <name>sqoop-${stack_version}</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+
+ <requiredServices>
+ <service>HDFS</service>
+ </requiredServices>
+
+ <configuration-dependencies>
+ <config-type>sqoop-env</config-type>
+ <config-type>sqoop-site</config-type>
+ <config-type>application.properties</config-type>
+ </configuration-dependencies>
+ </service>
+ </services>
+</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
new file mode 100644
index 0000000..5561e10
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/__init__.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
new file mode 100644
index 0000000..61573ee
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params.py
@@ -0,0 +1,27 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+from ambari_commons import OSCheck
+from resource_management.libraries.functions.default import default
+
+if OSCheck.is_windows_family():
+ from params_windows import *
+else:
+ from params_linux import *
+
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
new file mode 100644
index 0000000..c1138b3
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_linux.py
@@ -0,0 +1,135 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+import os
+
+from resource_management.libraries.functions.version import format_stack_version
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions.get_kinit_path import get_kinit_path
+from resource_management.libraries.script import Script
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from resource_management.libraries.functions.expect import expect
+from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster
+
+
+# a map of the Ambari role to the component name
+# for use with <stack-root>/current/<component>
+SERVER_ROLE_DIRECTORY_MAP = {
+ 'SQOOP' : 'sqoop-client'
+}
+
+component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP")
+
+config = Script.get_config()
+stack_root = Script.get_stack_root()
+
+# Needed since this is an Atlas Hook service.
+cluster_name = config['clusterName']
+
+ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
+
+stack_name = default("/hostLevelParams/stack_name", None)
+
+stack_version_unformatted = config['hostLevelParams']['stack_version']
+stack_version_formatted = format_stack_version(stack_version_unformatted)
+
+agent_stack_retry_on_unavailability = config['hostLevelParams']['agent_stack_retry_on_unavailability']
+agent_stack_retry_count = expect("/hostLevelParams/agent_stack_retry_count", int)
+
+# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
+version = default("/commandParams/version", None)
+
+# default hadoop params
+sqoop_conf_dir = "/usr/lib/sqoop/conf"
+sqoop_lib = "/usr/lib/sqoop/lib"
+hadoop_home = '/usr/lib/hadoop'
+hbase_home = "/usr/lib/hbase"
+hive_home = "/usr/lib/hive"
+sqoop_bin_dir = "/usr/bin"
+zoo_conf_dir = "/etc/zookeeper"
+
+# For stack versions supporting rolling upgrade
+if stack_version_formatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_formatted):
+ sqoop_conf_dir = format("{stack_root}/current/sqoop-client/conf")
+ sqoop_lib = format("{stack_root}/current/sqoop-client/lib")
+ hadoop_home = format("{stack_root}/current/hadoop-client")
+ hbase_home = format("{stack_root}/current/hbase-client")
+ hive_home = format("{stack_root}/current/hive-client")
+ sqoop_bin_dir = format("{stack_root}/current/sqoop-client/bin/")
+ zoo_conf_dir = format("{stack_root}/current/zookeeper-client/conf")
+
+security_enabled = config['configurations']['cluster-env']['security_enabled']
+smokeuser = config['configurations']['cluster-env']['smokeuser']
+smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
+user_group = config['configurations']['cluster-env']['user_group']
+sqoop_env_sh_template = config['configurations']['sqoop-env']['content']
+
+sqoop_user = config['configurations']['sqoop-env']['sqoop_user']
+
+smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
+kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
+#JDBC driver jar name
+sqoop_jdbc_drivers_dict = []
+sqoop_jdbc_drivers_name_dict = {}
+sqoop_jdbc_drivers_to_remove = {}
+if "jdbc_drivers" in config['configurations']['sqoop-env']:
+ sqoop_jdbc_drivers = config['configurations']['sqoop-env']['jdbc_drivers'].split(',')
+
+ for driver_name in sqoop_jdbc_drivers:
+ previous_jdbc_jar_name = None
+ driver_name = driver_name.strip()
+ if driver_name and not driver_name == '':
+ if driver_name == "com.microsoft.sqlserver.jdbc.SQLServerDriver":
+ jdbc_name = default("/hostLevelParams/custom_mssql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mssql_jdbc_name", None)
+ jdbc_driver_name = "mssql"
+ elif driver_name == "com.mysql.jdbc.Driver":
+ jdbc_name = default("/hostLevelParams/custom_mysql_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_mysql_jdbc_name", None)
+ jdbc_driver_name = "mysql"
+ elif driver_name == "org.postgresql.Driver":
+ jdbc_name = default("/hostLevelParams/custom_postgres_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_postgres_jdbc_name", None)
+ jdbc_driver_name = "postgres"
+ elif driver_name == "oracle.jdbc.driver.OracleDriver":
+ jdbc_name = default("/hostLevelParams/custom_oracle_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_oracle_jdbc_name", None)
+ jdbc_driver_name = "oracle"
+ elif driver_name == "org.hsqldb.jdbc.JDBCDriver":
+ jdbc_name = default("/hostLevelParams/custom_hsqldb_jdbc_name", None)
+ previous_jdbc_jar_name = default("/hostLevelParams/previous_custom_hsqldb_jdbc_name", None)
+ jdbc_driver_name = "hsqldb"
+ else:
+ continue
+ sqoop_jdbc_drivers_dict.append(jdbc_name)
+ sqoop_jdbc_drivers_to_remove[jdbc_name] = previous_jdbc_jar_name
+ sqoop_jdbc_drivers_name_dict[jdbc_name] = jdbc_driver_name
+jdk_location = config['hostLevelParams']['jdk_location']
+
+
+########################################################
+############# Atlas related params #####################
+########################################################
+#region Atlas Hooks
+sqoop_atlas_application_properties = default('/configurations/sqoop-atlas-application.properties', {})
+enable_atlas_hook = default('/configurations/sqoop-env/sqoop.atlas.hook', False)
+atlas_hook_filename = default('/configurations/atlas-env/metadata_conf_file', 'atlas-application.properties')
+#endregion
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
new file mode 100644
index 0000000..f930765
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/params_windows.py
@@ -0,0 +1,30 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.libraries.script import Script
+import os
+
+config = Script.get_config()
+
+sqoop_user = "sqoop"
+
+stack_root = os.path.abspath(os.path.join(os.environ["HADOOP_HOME"], ".."))
+sqoop_env_cmd_template = config['configurations']['sqoop-env']['content']
+sqoop_home_dir = os.environ["SQOOP_HOME"]
+sqoop_conf_dir = os.path.join(sqoop_home_dir, "conf")
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
new file mode 100644
index 0000000..bb503f5
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/service_check.py
@@ -0,0 +1,62 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+
+from resource_management.libraries.script.script import Script
+from resource_management.core.resources import Execute
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.functions import format
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+import os
+
+class SqoopServiceCheck(Script):
+ pass
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class SqoopServiceCheckDefault(SqoopServiceCheck):
+
+ def get_component_name(self):
+ return "sqoop-server"
+
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smokeuser_principal}"),
+ user = params.smokeuser,
+ )
+ Execute("sqoop version",
+ user = params.smokeuser,
+ path = params.sqoop_bin_dir,
+ logoutput = True
+ )
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class SqoopServiceCheckWindows(SqoopServiceCheck):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ smoke_cmd = os.path.join(params.stack_root,"Run-SmokeTests.cmd")
+ service = "SQOOP"
+ Execute(format("cmd /C {smoke_cmd} {service}"), logoutput=True)
+
+if __name__ == "__main__":
+ SqoopServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
new file mode 100644
index 0000000..436402c
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop.py
@@ -0,0 +1,124 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+# Python Imports
+import os
+
+# Local Imports
+from resource_management.core.source import InlineTemplate, DownloadSource
+from resource_management.libraries.functions import format
+from resource_management.libraries.functions.get_config import get_config
+from resource_management.libraries.resources.xml_config import XmlConfig
+from resource_management.core.resources.system import File, Link, Directory
+from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
+from ambari_commons import OSConst
+from resource_management.libraries.functions.setup_atlas_hook import has_atlas_in_cluster, setup_atlas_hook, setup_atlas_jar_symlinks
+from ambari_commons.constants import SERVICE
+
+
+@OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
+def sqoop(type=None):
+ import params
+ File(os.path.join(params.sqoop_conf_dir, "sqoop-env.cmd"),
+ content=InlineTemplate(params.sqoop_env_cmd_template)
+ )
+
+@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
+def sqoop(type=None):
+ import params
+ Link(params.sqoop_lib + "/mysql-connector-java.jar",
+ to = '/usr/share/java/mysql-connector-java.jar'
+ )
+
+ jdbc_connector()
+
+ Directory(params.sqoop_conf_dir,
+ owner = params.sqoop_user,
+ group = params.user_group,
+ create_parents = True
+ )
+
+ configs = {}
+ sqoop_site_config = get_config('sqoop-site')
+ if sqoop_site_config:
+ configs.update(sqoop_site_config)
+
+ XmlConfig("sqoop-site.xml",
+ conf_dir = params.sqoop_conf_dir,
+ configurations = configs,
+ configuration_attributes=params.config['configuration_attributes']['sqoop-site'],
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+
+ # Generate atlas-application.properties.xml file and symlink the hook jars
+ if params.enable_atlas_hook:
+ atlas_hook_filepath = os.path.join(params.sqoop_conf_dir, params.atlas_hook_filename)
+ setup_atlas_hook(SERVICE.SQOOP, params.sqoop_atlas_application_properties, atlas_hook_filepath, params.sqoop_user, params.user_group)
+ setup_atlas_jar_symlinks("sqoop", params.sqoop_lib)
+
+
+ File(format("{sqoop_conf_dir}/sqoop-env.sh"),
+ owner=params.sqoop_user,
+ group = params.user_group,
+ content=InlineTemplate(params.sqoop_env_sh_template)
+ )
+ update_config_permissions(["sqoop-env-template.sh",
+ "sqoop-site-template.xml",
+ "sqoop-site.xml"])
+ pass
+
+def update_config_permissions(names):
+ import params
+ for filename in names:
+ full_filename = os.path.join(params.sqoop_conf_dir, filename)
+ File(full_filename,
+ owner = params.sqoop_user,
+ group = params.user_group,
+ only_if = format("test -e {full_filename}")
+ )
+
+def jdbc_connector():
+ import params
+ from urllib2 import HTTPError
+ from resource_management import Fail
+ for jar_name in params.sqoop_jdbc_drivers_dict:
+ if not jar_name or 'mysql' in jar_name:
+ continue
+ downloaded_custom_connector = format("{sqoop_lib}/{jar_name}")
+ custom_connector_to_remove = format("{sqoop_lib}/" + str(params.sqoop_jdbc_drivers_to_remove[jar_name]))
+ jdbc_driver_label = params.sqoop_jdbc_drivers_name_dict[jar_name]
+ driver_curl_source = format("{jdk_location}/{jar_name}")
+ environment = {
+ "no_proxy": format("{ambari_server_hostname}")
+ }
+ try:
+ if custom_connector_to_remove and os.path.isfile(custom_connector_to_remove):
+ File(custom_connector_to_remove, action='delete')
+
+ File(downloaded_custom_connector,
+ content = DownloadSource(driver_curl_source),
+ mode = 0644,
+ )
+ except HTTPError:
+ error_string = format("Could not download {driver_curl_source}\n\
+ Please upload jdbc driver to server by run command:\n\
+ ambari-server setup --jdbc-db={jdbc_driver_label} --jdbc-driver=<PATH TO DRIVER>\n\
+ at {ambari_server_hostname}")
+ raise Fail(error_string)
+
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
new file mode 100644
index 0000000..d420fab
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/package/scripts/sqoop_client.py
@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+"""
+
+from resource_management.core.exceptions import ClientComponentHasNoStatus
+from resource_management.core.resources.system import Execute
+from resource_management.libraries.functions.default import default
+from resource_management.libraries.script.script import Script
+from resource_management.libraries.functions import conf_select
+from resource_management.libraries.functions import stack_select
+from resource_management.libraries.functions.format import format
+from resource_management.libraries.functions import StackFeature
+from resource_management.libraries.functions.stack_features import check_stack_feature
+from sqoop import sqoop
+from ambari_commons.os_family_impl import OsFamilyImpl
+from ambari_commons import OSConst
+
+class SqoopClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ sqoop(type='client')
+
+ def status(self, env):
+ raise ClientComponentHasNoStatus()
+
+@OsFamilyImpl(os_family=OsFamilyImpl.DEFAULT)
+class SqoopClientDefault(SqoopClient):
+ def get_component_name(self):
+ return "sqoop-client"
+
+ def pre_upgrade_restart(self, env, upgrade_type=None):
+ import params
+ env.set_params(params)
+
+ if params.version and check_stack_feature(StackFeature.ROLLING_UPGRADE, params.version):
+ conf_select.select(params.stack_name, "sqoop", params.version)
+ stack_select.select("sqoop-client", params.version)
+
+
+@OsFamilyImpl(os_family=OSConst.WINSRV_FAMILY)
+class SqoopClientWindows(SqoopClient):
+ pass
+
+if __name__ == "__main__":
+ SqoopClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
new file mode 100644
index 0000000..f9a6cb4
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.3.0/role_command_order.json
@@ -0,0 +1,6 @@
+{
+ "general_deps" : {
+ "_comment" : "dependencies for SQOOP",
+ "SQOOP_SERVICE_CHECK-SERVICE_CHECK": ["NODEMANAGER-START", "RESOURCEMANAGER-START"]
+ }
+}
http://git-wip-us.apache.org/repos/asf/ambari/blob/b3339123/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
new file mode 100644
index 0000000..757bce5
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/3.0/services/SQOOP/metainfo.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<metainfo>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SQOOP</name>
+ <version>1.4.4.3.0</version>
+ <extends>common-services/SQOOP/1.4.4.3.0</extends>
+ </service>
+ </services>
+</metainfo>