You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2013/11/29 17:34:16 UTC
git commit: AMBARI-3931. Sqoop on HDP2. Using resource management lib
(Eugene Chekanskiy via dlysnichenko)
Updated Branches:
refs/heads/trunk 95ba3b452 -> d2ebacc09
AMBARI-3931. Sqoop on HDP2. Using resource management lib (Eugene Chekanskiy via dlysnichenko)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/d2ebacc0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/d2ebacc0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/d2ebacc0
Branch: refs/heads/trunk
Commit: d2ebacc09e91ebdbf9b2442864214b28af5ffff6
Parents: 95ba3b4
Author: Lisnichenko Dmitro <dl...@hortonworks.com>
Authored: Fri Nov 29 18:33:27 2013 +0200
Committer: Lisnichenko Dmitro <dl...@hortonworks.com>
Committed: Fri Nov 29 18:33:27 2013 +0200
----------------------------------------------------------------------
.../HDP/2.0._/services/SQOOP/metainfo.xml | 46 ++++++++++++++---
.../services/SQOOP/package/scripts/__init__.py | 21 ++++++++
.../services/SQOOP/package/scripts/params.py | 40 +++++++++++++++
.../SQOOP/package/scripts/service_check.py | 39 ++++++++++++++
.../services/SQOOP/package/scripts/sqoop.py | 54 ++++++++++++++++++++
.../SQOOP/package/scripts/sqoop_client.py | 40 +++++++++++++++
.../SQOOP/package/templates/sqoop-env.sh.j2 | 36 +++++++++++++
7 files changed, 268 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml
index 9a50700..0070c34 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/metainfo.xml
@@ -16,15 +16,45 @@
limitations under the License.
-->
<metainfo>
- <user>root</user>
- <comment>Tool for transferring bulk data between Apache Hadoop and structured data stores such as relational databases</comment>
- <version>1.4.4.2.0.6.0</version>
+ <schemaVersion>2.0</schemaVersion>
+ <services>
+ <service>
+ <name>SQOOP</name>
+ <comment>Tool for transferring bulk data between Apache Hadoop and
+ structured data stores such as relational databases
+ </comment>
+ <version>1.4.4.2.0.6.0</version>
- <components>
+ <components>
<component>
- <name>SQOOP</name>
- <category>CLIENT</category>
+ <name>SQOOP</name>
+ <category>CLIENT</category>
+ <commandScript>
+ <script>scripts/sqoop_client.py</script>
+ <scriptType>PYTHON</scriptType>
+ </commandScript>
</component>
- </components>
-
+ </components>
+ <osSpecifics>
+ <osSpecific>
+ <osType>any</osType>
+ <packages>
+ <package>
+ <type>rpm</type>
+ <name>sqoop</name>
+ </package>
+ <package>
+ <type>rpm</type>
+ <name>mysql-connector-java</name>
+ </package>
+ </packages>
+ </osSpecific>
+ </osSpecifics>
+ <commandScript>
+ <script>scripts/service_check.py</script>
+ <scriptType>PYTHON</scriptType>
+ <timeout>300</timeout>
+ </commandScript>
+ </service>
+ </services>
</metainfo>
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py
new file mode 100644
index 0000000..b80fda6
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/__init__.py
@@ -0,0 +1,21 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py
new file mode 100644
index 0000000..559ee81
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/params.py
@@ -0,0 +1,40 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+
+config = Script.get_config()
+
+security_enabled = config['configurations']['global']['security_enabled']
+smokeuser = config['configurations']['global']['smokeuser']
+user_group = config['configurations']['global']['user_group']
+
+sqoop_conf_dir = "/usr/lib/sqoop/conf"
+hbase_home = "/usr"
+hive_home = "/usr"
+zoo_conf_dir = "/etc/zookeeper"
+sqoop_lib = "/usr/lib/sqoop/lib"
+sqoop_user = "sqoop"
+
+keytab_path = default("keytab_path","/etc/security/keytabs")
+
+smoke_user_keytab = "/etc/security/keytabs/smokeuser.headless.keytab" # smoke_user_keytab
+kinit_path_local = functions.get_kinit_path([default('kinit_path_local'),"/usr/bin", "/usr/kerberos/bin", "/usr/sbin"])
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
new file mode 100644
index 0000000..88ac32c
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/service_check.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+
+from resource_management import *
+
+
+class SqoopServiceCheck(Script):
+ def service_check(self, env):
+ import params
+ env.set_params(params)
+ if params.security_enabled:
+ Execute(format("{kinit_path_local} -kt {smoke_user_keytab} {smoke_test_user}"))
+ Execute("sqoop version",
+ user = params.smokeuser,
+ logoutput = True
+ )
+
+if __name__ == "__main__":
+ SqoopServiceCheck().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py
new file mode 100644
index 0000000..b4f24b5
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop.py
@@ -0,0 +1,54 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+from resource_management import *
+import sys
+
+def sqoop(type=None):
+ import params
+ Link(params.sqoop_lib + "/mysql-connector-java.jar",
+ to = '/usr/share/java/mysql-connector-java.jar'
+ )
+ Directory(params.sqoop_conf_dir,
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+ sqoop_TemplateConfig("sqoop-env.sh")
+ File (params.sqoop_conf_dir + "/sqoop-env-template.sh",
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+ File (params.sqoop_conf_dir + "/sqoop-site-template.xml",
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+ File (params.sqoop_conf_dir + "/sqoop-site.xml",
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+ pass
+
+def sqoop_TemplateConfig(name, tag=None):
+ import params
+ TemplateConfig( format("{sqoop_conf_dir}/{name}"),
+ owner = params.sqoop_user,
+ template_tag = tag
+ )
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py
new file mode 100644
index 0000000..90dfecf
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/scripts/sqoop_client.py
@@ -0,0 +1,40 @@
+#!/usr/bin/env python2.6
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+Ambari Agent
+
+"""
+
+import sys
+from resource_management import *
+
+from sqoop import sqoop
+
+
+class SqoopClient(Script):
+ def install(self, env):
+ self.install_packages(env)
+ self.configure(env)
+
+ def configure(self, env):
+ import params
+ env.set_params(params)
+ sqoop(type='client')
+
+if __name__ == "__main__":
+ SqoopClient().execute()
http://git-wip-us.apache.org/repos/asf/ambari/blob/d2ebacc0/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2 b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2
new file mode 100644
index 0000000..90cbc75
--- /dev/null
+++ b/ambari-server/src/main/resources/stacks/HDP/2.0._/services/SQOOP/package/templates/sqoop-env.sh.j2
@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# included in all the hadoop scripts with source command
+# should not be executable directly
+# also should not be passed any arguments, since we need original $*
+
+# Set Hadoop-specific environment variables here.
+
+#Set path to where bin/hadoop is available
+#Set path to where bin/hadoop is available
+export HADOOP_HOME=${HADOOP_HOME:-/usr/lib/hadoop}
+
+#set the path to where bin/hbase is available
+export HBASE_HOME=${HBASE_HOME:-/usr/lib/hbase}
+
+#Set the path to where bin/hive is available
+export HIVE_HOME=${HIVE_HOME:-/usr/lib/hive}
+
+#Set the path for where zookeper config dir is
+export ZOOCFGDIR=${ZOOCFGDIR:-/etc/zookeeper/conf}
+
+# add libthrift in hive to sqoop class path first so hive imports work
+export SQOOP_USER_CLASSPATH="`ls ${HIVE_HOME}/lib/libthrift-*.jar 2> /dev/null`:${SQOOP_USER_CLASSPATH}"