You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2016/02/03 15:57:44 UTC
ambari git commit: AMBARI-14853 - Atlas Integration: Support
deploying latest Atlas(which depends on kafka) using Ambari (tbeerbower)
Repository: ambari
Updated Branches:
refs/heads/trunk 18463c86a -> 1d9f1bb06
AMBARI-14853 - Atlas Integration: Support deploying latest Atlas(which depends on kafka) using Ambari (tbeerbower)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/1d9f1bb0
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/1d9f1bb0
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/1d9f1bb0
Branch: refs/heads/trunk
Commit: 1d9f1bb069895d6dbc344a5cf0d81e56ff0b7371
Parents: 18463c8
Author: tbeerbower <tb...@hortonworks.com>
Authored: Wed Feb 3 09:20:46 2016 -0500
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Wed Feb 3 09:57:36 2016 -0500
----------------------------------------------------------------------
.../configuration/application-properties.xml | 36 +++++++++++++++++++-
.../ATLAS/0.1.0.2.3/metainfo.xml | 4 +++
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 11 ++++++
.../HIVE/0.12.0.2.0/configuration/hive-env.xml | 2 ++
.../0.12.0.2.0/package/scripts/params_linux.py | 2 ++
.../services/HIVE/configuration/hive-env.xml | 2 ++
.../test/python/stacks/2.3/configs/default.json | 9 ++++-
7 files changed, 64 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
index 82dacb6..74b1537 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
@@ -131,6 +131,41 @@
<description></description>
</property>
<property>
+ <name>atlas.notification.embedded</name>
+ <value>false</value>
+ <description>Indicates whether or not the notification service should be embedded.</description>
+ </property>
+ <property>
+ <name>atlas.kafka.data</name>
+ <value>{{metadata_home}}/data/kafka</value>
+ <description>The Kafka data directory.</description>
+ </property>
+ <property>
+ <name>atlas.kafka.bootstrap.servers</name>
+ <value>{{kafka_bootstrap_servers}}</value>
+ <description>Comma separated list of Kafka broker endpoints in host:port form</description>
+ </property>
+ <property>
+ <name>atlas.kafka.zookeeper.connect</name>
+ <value>{{kafka_zookeeper_connect}}</value>
+ <description>Comma separated list of servers forming Zookeeper quorum used by Kafka.</description>
+ </property>
+ <property>
+ <name>atlas.kafka.hook.group.id</name>
+ <value>atlas</value>
+ <description>Kafka group id for the hook topic.</description>
+ </property>
+ <property>
+ <name>atlas.kafka.entities.group.id</name>
+ <value>entities</value>
+ <description>Kafka group id for the entity topic.</description>
+ </property>
+ <property>
+ <name>atlas.cluster.name</name>
+ <value>{{cluster_name}}</value>
+ <description>The cluster name.</description>
+ </property>
+ <property>
<name>atlas.server.http.port</name>
<value>21000</value>
<description></description>
@@ -140,5 +175,4 @@
<value>21443</value>
<description></description>
</property>
-
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
index 2600fc4..719f92c 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
@@ -92,6 +92,10 @@
<timeout>300</timeout>
</commandScript>
+ <requiredServices>
+ <service>KAFKA</service>
+ </requiredServices>
+
<configuration-dependencies>
<config-type>application-properties</config-type>
<config-type>atlas-env</config-type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 1a0c67b..eb2d816 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -28,6 +28,8 @@ import status_params
# server configurations
config = Script.get_config()
+cluster_name = config['clusterName']
+
# security enabled
security_enabled = status_params.security_enabled
@@ -112,3 +114,12 @@ if security_enabled:
smoke_cmd = format('curl --negotiate -u : -b ~/cookiejar.txt -c ~/cookiejar.txt -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')
else:
smoke_cmd = format('curl -s -o /dev/null -w "%{{http_code}}" http://{metadata_host}:{metadata_port}/')
+
+# kafka
+kafka_bootstrap_servers = ""
+kafka_broker_hosts = config['clusterHostInfo']['kafka_broker_hosts']
+if not len(kafka_broker_hosts) == 0:
+ kafka_broker_port = default("/configurations/kafka-broker/port", 6667)
+ kafka_bootstrap_servers = kafka_broker_hosts[0] + ":" + str(kafka_broker_port)
+
+kafka_zookeeper_connect = default("/configurations/kafka-broker/zookeeper.connect", None)
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
index 281bb41..7aa4a35 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/configuration/hive-env.xml
@@ -226,6 +226,8 @@ else
fi
export METASTORE_PORT={{hive_metastore_port}}
+export HADOOP_CLASSPATH={{atlas_conf_dir}}:{{atlas_home_dir}}/hook/hive:${HADOOP_CLASSPATH}
+
</value>
<value-attributes>
<type>content</type>
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
index cd3741d..dc17dba 100644
--- a/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/HIVE/0.12.0.2.0/package/scripts/params_linux.py
@@ -398,6 +398,8 @@ atlas_plugin_package = "atlas-metadata*-hive-plugin"
atlas_ubuntu_plugin_package = "atlas-metadata.*-hive-plugin"
if has_atlas:
+ atlas_home_dir = os.environ['METADATA_HOME_DIR'] if 'METADATA_HOME_DIR' in os.environ else '/usr/hdp/current/atlas-server'
+ atlas_conf_dir = os.environ['METADATA_CONF'] if 'METADATA_CONF' in os.environ else '/etc/atlas/conf'
# client.properties
atlas_client_props = {}
auth_enabled = config['configurations']['application-properties'].get(
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
index 92c0c03..1f2ca96 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.3/services/HIVE/configuration/hive-env.xml
@@ -69,6 +69,8 @@ fi
export METASTORE_PORT={{hive_metastore_port}}
+export HADOOP_CLASSPATH={{atlas_conf_dir}}:{{atlas_home_dir}}/hook/hive:${HADOOP_CLASSPATH}
+
{% if sqla_db_used or lib_dir_available %}
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:{{jdbc_libs_dir}}"
export JAVA_LIBRARY_PATH="$JAVA_LIBRARY_PATH:{{jdbc_libs_dir}}"
http://git-wip-us.apache.org/repos/asf/ambari/blob/1d9f1bb0/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index 21bff13..a36455c 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -164,6 +164,7 @@
"log.retention.hours": "168"
},
"application-properties": {
+ "atlas.cluster.name" : "c2",
"atlas.graph.storage.backend": "berkeleyje",
"atlas.graph.storage.directory": "data/berkley",
"atlas.graph.index.search.backend": "elasticsearch",
@@ -185,7 +186,13 @@
"atlas.http.authentication.kerberos.principal": "HTTP/_HOST@EXAMPLE.COM",
"atlas.http.authentication.kerberos.keytab": "/etc/security/keytabs/spnego.service.keytab",
"atlas.http.authentication.kerberos.name.rules": "DEFAULT",
- "atlas.server.http.port" : "21000"
+ "atlas.server.http.port" : "21000",
+ "atlas.notification.embedded" : false,
+ "atlas.kafka.bootstrap.servers" : "c6401.ambari.apache.org:6667",
+ "atlas.kafka.data" : "/usr/hdp/current/atlas-server/data/kafka",
+ "atlas.kafka.entities.group.id" : "entities",
+ "atlas.kafka.hook.group.id" : "atlas",
+ "atlas.kafka.zookeeper.connect" : "c6401.ambari.apache.org:2181"
},
"atlas-env": {
"content": "# The java implementation to use. If JAVA_HOME is not found we expect java and jar to be in path\nexport JAVA_HOME={{java64_home}}\n# any additional java opts you want to set. This will apply to both client and server operations\nexport METADATA_OPTS={{metadata_opts}}\n# metadata configuration directory\nexport METADATA_CONF={{conf_dir}}\n# Where log files are stored. Defatult is logs directory under the base install location\nexport METADATA_LOG_DIR={{log_dir}}\n# additional classpath entries\nexport METADATACPPATH={{metadata_classpath}}\n# data dir\nexport METADATA_DATA_DIR={{data_dir}}\n# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.\nexport METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}",