You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2016/04/10 23:36:52 UTC
ambari git commit: AMBARI-14920 - Support Atlas / Storm integration
Repository: ambari
Updated Branches:
refs/heads/trunk 7b64e964d -> 55e5a785f
AMBARI-14920 - Support Atlas / Storm integration
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/55e5a785
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/55e5a785
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/55e5a785
Branch: refs/heads/trunk
Commit: 55e5a785fea5cff8dceb5dc1d43371b68726b848
Parents: 7b64e96
Author: tbeerbower <tb...@hortonworks.com>
Authored: Sun Apr 10 17:36:22 2016 -0400
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Sun Apr 10 17:36:38 2016 -0400
----------------------------------------------------------------------
.../0.9.1.2.1/configuration/storm-site.xml | 20 +++++++++++++
.../0.9.1.2.1/package/scripts/params_linux.py | 14 +++++++++
.../STORM/0.9.1.2.1/package/scripts/storm.py | 15 +++++++++-
.../services/STORM/configuration/storm-env.xml | 31 ++++++++++++++++----
4 files changed, 74 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
index aada363..cf702c0 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/configuration/storm-site.xml
@@ -727,4 +727,24 @@
<empty-value-valid>true</empty-value-valid>
</value-attributes>
</property>
+ <property>
+ <name>atlas.cluster.name</name>
+ <value>{{cluster_name}}</value>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.cluster.name</name>
+ </property>
+ </depends-on>
+ </property>
+ <property>
+ <name>storm.topology.submission.notifier.plugin.class</name>
+ <value>org.apache.atlas.storm.hook.StormAtlasHook</value>
+ <depends-on>
+ <property>
+ <type>application-properties</type>
+ <name>atlas.cluster.name</name>
+ </property>
+ </depends-on>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
index e9e8ce5..101ca46 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/params_linux.py
@@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
+import os
import re
import ambari_simplejson as json # simplejson is much faster comparing to Python 2.6 json module and has the same functions set.
@@ -43,6 +44,8 @@ tmp_dir = Script.get_tmp_dir()
stack_root = status_params.stack_root
sudo = AMBARI_SUDO_BINARY
+cluster_name = config['clusterName']
+
stack_name = default("/hostLevelParams/stack_name", None)
upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE)
version = default("/commandParams/version", None)
@@ -182,6 +185,17 @@ metrics_report_interval = default("/configurations/ams-site/timeline.metrics.sin
metrics_collection_period = default("/configurations/ams-site/timeline.metrics.sink.collection.period", 10)
metric_collector_sink_jar = "/usr/lib/storm/lib/ambari-metrics-storm-sink*.jar"
+jar_jvm_opts = ''
+
+# Atlas related params
+atlas_hosts = default('/clusterHostInfo/atlas_server_hosts', [])
+has_atlas = len(atlas_hosts) > 0
+
+if has_atlas:
+ atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else stack_root + '/current/atlas-server'
+ atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
+ jar_jvm_opts = '-Datlas.conf=' + atlas_conf_dir
+
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
index d2fafa6..74050d8 100644
--- a/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
+++ b/ambari-server/src/main/resources/common-services/STORM/0.9.1.2.1/package/scripts/storm.py
@@ -52,6 +52,7 @@ def storm(name=None):
@OsFamilyFuncImpl(os_family=OsFamilyImpl.DEFAULT)
def storm(name=None):
import params
+ import os
Directory(params.log_dir,
owner=params.storm_user,
@@ -93,6 +94,18 @@ def storm(name=None):
content=InlineTemplate(params.storm_env_sh_template)
)
+ if params.has_atlas:
+ atlas_storm_hook_dir = os.path.join(params.atlas_home_dir, "hook", "storm")
+ if os.path.exists(atlas_storm_hook_dir):
+ storm_extlib_dir = os.path.join(params.storm_component_home_dir, "extlib")
+ if os.path.exists(storm_extlib_dir):
+ src_files = os.listdir(atlas_storm_hook_dir)
+ for file_name in src_files:
+ atlas_storm_hook_file_name = os.path.join(atlas_storm_hook_dir, file_name)
+ storm_lib_file_name = os.path.join(storm_extlib_dir, file_name)
+ if (os.path.isfile(atlas_storm_hook_file_name)):
+ Link(storm_lib_file_name, to = atlas_storm_hook_file_name)
+
if params.has_metric_collector:
File(format("{conf_dir}/storm-metrics2.properties"),
owner=params.storm_user,
@@ -127,7 +140,7 @@ def storm(name=None):
owner=params.storm_user,
content=InlineTemplate(params.storm_worker_log4j_content)
)
-
+
if params.security_enabled:
TemplateConfig(format("{conf_dir}/storm_jaas.conf"),
owner=params.storm_user
http://git-wip-us.apache.org/repos/asf/ambari/blob/55e5a785/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml
index 2bb0862..8d2da69 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/STORM/configuration/storm-env.xml
@@ -26,9 +26,30 @@
<value>true</value>
<description></description>
</property>
- <property>
- <name>storm_logs_supported</name>
- <value>true</value>
- <description></description>
- </property>
+ <property>
+ <name>storm_logs_supported</name>
+ <value>true</value>
+ <description></description>
+ </property>
+ <!-- storm-env.sh -->
+ <property>
+ <name>content</name>
+ <description>This is the jinja template for storm-env.sh file</description>
+ <value>
+ #!/bin/bash
+
+ # Set Storm specific environment variables here.
+
+ # The java implementation to use.
+ export JAVA_HOME={{java64_home}}
+
+ export STORM_CONF_DIR={{conf_dir}}
+ export STORM_HOME={{storm_component_home_dir}}
+
+ export STORM_JAR_JVM_OPTS={{jar_jvm_opts}}
+ </value>
+ <value-attributes>
+ <type>content</type>
+ </value-attributes>
+ </property>
</configuration>