You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by dm...@apache.org on 2016/02/19 14:26:23 UTC
[06/21] ambari git commit: AMBARI-14921 - Atlas Integration : Support
Atlas / Sqoop integration
AMBARI-14921 - Atlas Integration : Support Atlas / Sqoop integration
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/260419bb
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/260419bb
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/260419bb
Branch: refs/heads/branch-dev-patch-upgrade
Commit: 260419bb848ce962bbc238379221096eb2a31a58
Parents: 0a9101e
Author: tbeerbower <tb...@hortonworks.com>
Authored: Thu Feb 18 11:22:03 2016 -0500
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Thu Feb 18 12:15:55 2016 -0500
----------------------------------------------------------------------
.../1.4.4.2.0/configuration/sqoop-site.xml | 43 ++++++++++++++++++++
.../SQOOP/1.4.4.2.0/metainfo.xml | 6 +++
.../1.4.4.2.0/package/scripts/params_linux.py | 20 ++++++++-
.../SQOOP/1.4.4.2.0/package/scripts/sqoop.py | 26 ++++++++++++
.../python/stacks/2.0.6/SQOOP/test_sqoop.py | 12 ++++++
.../python/stacks/2.0.6/configs/default.json | 5 +++
.../python/stacks/2.0.6/configs/secured.json | 5 +++
.../test/python/stacks/2.3/configs/default.json | 5 +++
8 files changed, 121 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
new file mode 100644
index 0000000..dd299fe
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/configuration/sqoop-site.xml
@@ -0,0 +1,43 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>atlas.cluster.name</name>
+ <value>{{cluster_name}}</value>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.cluster.name</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
+ <name>sqoop.job.data.publish.class</name>
+ <value>{{job_data_publish_class}}</value>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.cluster.name</name>
+ </property>
+ </depends-on>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
index b5db91b..2c9d881 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/metainfo.xml
@@ -55,6 +55,11 @@
</commandScript>
<configFiles>
<configFile>
+ <type>xml</type>
+ <fileName>squoop-site.xml</fileName>
+ <dictionaryName>squoop-site</dictionaryName>
+ </configFile>
+ <configFile>
<type>env</type>
<fileName>sqoop-env.sh</fileName>
<dictionaryName>sqoop-env</dictionaryName>
@@ -89,6 +94,7 @@
<configuration-dependencies>
<config-type>sqoop-env</config-type>
+ <config-type>sqoop-site</config-type>
</configuration-dependencies>
</service>
</services>
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
index 3a3f93e..0006078 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/params_linux.py
@@ -21,6 +21,7 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions.get_kinit_path import get_kinit_path
from resource_management.libraries.script import Script
+import os
# a map of the Ambari role to the component name
# for use with /usr/hdp/current/<component>
@@ -31,6 +32,9 @@ SERVER_ROLE_DIRECTORY_MAP = {
component_directory = Script.get_component_from_role(SERVER_ROLE_DIRECTORY_MAP, "SQOOP")
config = Script.get_config()
+
+cluster_name = config['clusterName']
+
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
stack_name = default("/hostLevelParams/stack_name", None)
@@ -103,4 +107,18 @@ if "jdbc_drivers" in config['configurations']['sqoop-env']:
continue
sqoop_jdbc_drivers_dict[jdbc_jar_name] = jdbc_symlink_name
sqoop_jdbc_drivers_name_dict[jdbc_jar_name] = jdbc_driver_name
-jdk_location = config['hostLevelParams']['jdk_location']
\ No newline at end of file
+jdk_location = config['hostLevelParams']['jdk_location']
+
+job_data_publish_class = ''
+
+########################################################
+############# Atlas related params #####################
+########################################################
+
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+has_atlas = len(atlas_hosts) > 0
+
+if has_atlas:
+ atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
+ atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+ job_data_publish_class = 'org.apache.atlas.sqoop.hook.SqoopHook'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
index 66a5ee4..392bff7 100644
--- a/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
+++ b/ambari-server/src/main/resources/common-services/SQOOP/1.4.4.2.0/package/scripts/sqoop.py
@@ -19,6 +19,7 @@ limitations under the License.
from resource_management.core.source import InlineTemplate, DownloadSource
from resource_management.libraries.functions import format
+from resource_management.libraries.resources.xml_config import XmlConfig
from resource_management.core.resources.system import File, Link, Directory
from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl
from ambari_commons import OSConst
@@ -46,6 +47,31 @@ def sqoop(type=None):
group = params.user_group,
create_parents = True
)
+
+ configs = {}
+ configs.update(params.config['configurations']['sqoop-site'])
+
+ XmlConfig("sqoop-site.xml",
+ conf_dir = params.sqoop_conf_dir,
+ configurations = configs,
+ configuration_attributes=params.config['configuration_attributes']['sqoop-site'],
+ owner = params.sqoop_user,
+ group = params.user_group
+ )
+
+ if params.has_atlas:
+ Link(params.sqoop_conf_dir + "/application.properties",
+ to = params.atlas_conf_dir + "/application.properties"
+ )
+
+ atlas_sqoop_hook_dir = params.atlas_home_dir + "/hook/sqoop"
+ src_files = os.listdir(atlas_sqoop_hook_dir)
+ for file_name in src_files:
+ atlas_sqoop_hook_file_name = os.path.join(atlas_sqoop_hook_dir, file_name)
+ sqoop_lib_file_name = os.path.join(params.sqoop_lib, file_name)
+ if (os.path.isfile(atlas_sqoop_hook_file_name)):
+ Link(sqoop_lib_file_name, to = atlas_sqoop_hook_file_name)
+
File(format("{sqoop_conf_dir}/sqoop-env.sh"),
owner=params.sqoop_user,
group = params.user_group,
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
index 5a10305..e12db4d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/SQOOP/test_sqoop.py
@@ -39,6 +39,12 @@ class TestSqoop(RMFTestCase):
create_parents = True,
owner = 'sqoop',
group = 'hadoop',)
+ self.assertResourceCalled('XmlConfig', 'sqoop-site.xml',
+ owner = 'sqoop',
+ group = 'hadoop',
+ conf_dir = '/usr/lib/sqoop/conf',
+ configurations = self.getConfig()['configurations']['sqoop-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site'])
self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh',
owner = 'sqoop',
group = 'hadoop',
@@ -89,6 +95,12 @@ class TestSqoop(RMFTestCase):
group = 'hadoop',
create_parents = True,
)
+ self.assertResourceCalled('XmlConfig', 'sqoop-site.xml',
+ owner = 'sqoop',
+ group = 'hadoop',
+ conf_dir = '/usr/lib/sqoop/conf',
+ configurations = self.getConfig()['configurations']['sqoop-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['sqoop-site'])
self.assertResourceCalled('File', '/usr/lib/sqoop/conf/sqoop-env.sh',
content = InlineTemplate(self.getConfig()['configurations']['sqoop-env']['content']),
owner = 'sqoop',
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index 3d3987b..adf12ad 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -40,6 +40,10 @@
"taskId": 152,
"public_hostname": "c6401.ambari.apache.org",
"configurations": {
+ "sqoop-site": {
+ "atlas.cluster.name": "c1",
+ "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+ },
"mapred-site": {
"mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020",
"mapreduce.cluster.administrators": " hadoop",
@@ -877,6 +881,7 @@
"ams-hbase-log4j": {},
"ams-site": {},
"ams-ssl-server": {},
+ "sqoop-site": {},
"yarn-site": {
"final": {
"yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
index d673973..e17559d 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/secured.json
@@ -37,6 +37,10 @@
"taskId": 186,
"public_hostname": "c6401.ambari.apache.org",
"configurations": {
+ "sqoop-site": {
+ "atlas.cluster.name": "c1",
+ "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+ },
"mapred-site": {
"mapreduce.jobhistory.address": "c6402.ambari.apache.org:10020",
"mapreduce.jobhistory.webapp.spnego-keytab-file": "/etc/security/keytabs/spnego.service.keytab",
@@ -765,6 +769,7 @@
}
},
"configuration_attributes": {
+ "sqoop-site": {},
"yarn-site": {
"final": {
"yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",
http://git-wip-us.apache.org/repos/asf/ambari/blob/260419bb/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index a36455c..4a72f08 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -33,6 +33,10 @@
"slider-client": {
"slider.yarn.queue": "default"
},
+ "sqoop-site": {
+ "atlas.cluster.name": "c1",
+ "sqoop.job.data.publish.class": "org.apache.atlas.sqoop.hook.SqoopHook"
+ },
"mahout-env": {
"mahout_user": "mahout"
},
@@ -214,6 +218,7 @@
}
},
"configuration_attributes": {
+ "sqoop-site": {},
"yarn-site": {
"final": {
"yarn.nodemanager.disk-health-checker.min-healthy-disks": "true",