You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2016/02/25 15:50:48 UTC
ambari git commit: AMBARI-14922 - Atlas Integration: Support Atlas /
Falcon integration
Repository: ambari
Updated Branches:
refs/heads/trunk 46b4a797e -> efb4b8778
AMBARI-14922 - Atlas Integration: Support Atlas / Falcon integration
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/efb4b877
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/efb4b877
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/efb4b877
Branch: refs/heads/trunk
Commit: efb4b877850ca5d56153afe278d48a459b0f07b3
Parents: 46b4a79
Author: tbeerbower <tb...@hortonworks.com>
Authored: Thu Feb 25 09:50:37 2016 -0500
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Thu Feb 25 09:50:37 2016 -0500
----------------------------------------------------------------------
.../configuration/application-properties.xml | 35 ++++++++++++++++++++
.../FALCON/0.5.0.2.1/package/scripts/falcon.py | 15 +++++++++
.../0.5.0.2.1/package/scripts/params_linux.py | 29 +++++++++++++++-
.../configuration/falcon-startup.properties.xml | 10 +-----
4 files changed, 79 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
index ed9f1bc..a6deda1 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
@@ -176,4 +176,39 @@
<value>21443</value>
<description></description>
</property>
+ <property>
+ <name>atlas.http.authentication.type</name>
+ <value>simple</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.synchronous</name>
+ <value>false</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.numRetries</name>
+ <value>3</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.minThreads</name>
+ <value>5</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.maxThreads</name>
+ <value>5</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.keepAliveTime</name>
+ <value>10</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.hook.falcon.queueSize</name>
+ <value>1000</value>
+ <description></description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
index 81ea39e..ca967b0 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/falcon.py
@@ -27,6 +27,7 @@ from resource_management.core.resources.service import Service
from resource_management.core.resources.service import ServiceConfig
from resource_management.core.resources.system import Directory
from resource_management.core.resources.system import File
+from resource_management.core.resources.system import Link
from resource_management.libraries.script import Script
from resource_management.libraries.resources import PropertiesFile
from resource_management.libraries.functions import format
@@ -106,6 +107,11 @@ def falcon(type, action = None, upgrade_type=None):
create_parents = True,
cd_access = "a")
+ if params.has_atlas:
+ Link(params.falcon_conf_dir + "/application.properties",
+ to = params.atlas_conf_dir + "/application.properties"
+ )
+
if type == 'server':
if action == 'config':
if params.store_uri[0:4] == "hdfs":
@@ -176,6 +182,15 @@ def falcon(type, action = None, upgrade_type=None):
path = params.hadoop_bin_dir,
environment=environment_dictionary)
+ if params.has_atlas:
+ atlas_falcon_hook_dir = params.atlas_home_dir + "/hook/falcon"
+ src_files = os.listdir(atlas_falcon_hook_dir)
+ for file_name in src_files:
+ atlas_falcon_hook_file_name = os.path.join(atlas_falcon_hook_dir, file_name)
+ falcon_lib_file_name = os.path.join(params.falcon_webinf_lib, file_name)
+ if (os.path.isfile(atlas_falcon_hook_file_name)):
+ Link(falcon_lib_file_name, to = atlas_falcon_hook_file_name)
+
if action == 'stop':
Execute(format('{falcon_home}/bin/falcon-stop'),
user = params.falcon_user,
http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
index 6b37451..74ac010 100644
--- a/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/FALCON/0.5.0.2.1/package/scripts/params_linux.py
@@ -25,6 +25,7 @@ from resource_management.libraries.functions.version import format_hdp_stack_ver
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
+import os
config = Script.get_config()
@@ -59,6 +60,8 @@ else:
falcon_webapp_dir = '/var/lib/falcon/webapp'
falcon_home = '/usr/lib/falcon'
+falcon_webinf_lib = falcon_home + "/server/webapp/falcon/WEB-INF/lib"
+
hadoop_conf_dir = status_params.hadoop_conf_dir
falcon_conf_dir = status_params.falcon_conf_dir
oozie_user = config['configurations']['oozie-env']['oozie_user']
@@ -106,7 +109,31 @@ supports_hive_dr = config['configurations']['falcon-env']['supports_hive_dr']
local_data_mirroring_dir = "/usr/hdp/current/falcon-server/data-mirroring"
dfs_data_mirroring_dir = "/apps/data-mirroring"
-
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+has_atlas = len(atlas_hosts) > 0
+
+if has_atlas:
+ atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+ atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
+
+ application_services = "org.apache.falcon.security.AuthenticationInitializationService,\
+ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
+ org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.entity.store.ConfigurationStore,\
+ org.apache.falcon.rerun.service.RetryService,\
+ org.apache.falcon.rerun.service.LateRunService,\
+ org.apache.falcon.service.LogCleanupService,\
+ org.apache.falcon.metadata.MetadataMappingService,\
+ org.apache.falcon.atlas.service.AtlasService"
+else:
+ application_services = "org.apache.falcon.security.AuthenticationInitializationService,\
+ org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
+ org.apache.falcon.service.ProcessSubscriberService,\
+ org.apache.falcon.entity.store.ConfigurationStore,\
+ org.apache.falcon.rerun.service.RetryService,\
+ org.apache.falcon.rerun.service.LateRunService,\
+ org.apache.falcon.service.LogCleanupService,\
+ org.apache.falcon.metadata.MetadataMappingService"
hdfs_site = config['configurations']['hdfs-site']
default_fs = config['configurations']['core-site']['fs.defaultFS']
http://git-wip-us.apache.org/repos/asf/ambari/blob/efb4b877/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
index 4e4ec67..5d4fa1e 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.2/services/FALCON/configuration/falcon-startup.properties.xml
@@ -27,15 +27,7 @@
</property>
<property>
<name>*.application.services</name>
- <value>org.apache.falcon.security.AuthenticationInitializationService,\
- org.apache.falcon.workflow.WorkflowJobEndNotificationService, \
- org.apache.falcon.service.ProcessSubscriberService,\
- org.apache.falcon.entity.store.ConfigurationStore,\
- org.apache.falcon.rerun.service.RetryService,\
- org.apache.falcon.rerun.service.LateRunService,\
- org.apache.falcon.service.LogCleanupService,\
- org.apache.falcon.metadata.MetadataMappingService
- </value>
+ <value>{{application_services}}</value>
<description>Falcon Services</description>
</property>
<property>