You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by tb...@apache.org on 2016/04/22 13:22:31 UTC
ambari git commit: AMBARI-15981 - Atlas Integration : Make default
Atlas configuration for embedded Hbase
Repository: ambari
Updated Branches:
refs/heads/trunk 1734b5c7b -> 5db6f1d21
AMBARI-15981 - Atlas Integration : Make default Atlas configuration for embedded Hbase
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5db6f1d2
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5db6f1d2
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5db6f1d2
Branch: refs/heads/trunk
Commit: 5db6f1d21f05b61d9da1240da693350021984001
Parents: 1734b5c
Author: tbeerbower <tb...@hortonworks.com>
Authored: Fri Apr 22 07:20:06 2016 -0400
Committer: tbeerbower <tb...@hortonworks.com>
Committed: Fri Apr 22 07:20:20 2016 -0400
----------------------------------------------------------------------
.../configuration/application-properties.xml | 21 ++++++-
.../ATLAS/0.1.0.2.3/configuration/atlas-env.xml | 2 +-
.../configuration/atlas-hbase-site.xml | 58 ++++++++++++++++++++
.../ATLAS/0.1.0.2.3/metainfo.xml | 15 ++++-
.../ATLAS/0.1.0.2.3/package/scripts/metadata.py | 27 ++++++++-
.../ATLAS/0.1.0.2.3/package/scripts/params.py | 5 ++
.../services/ATLAS/configuration/atlas-env.xml | 2 +-
.../stacks/HDP/2.5/services/ATLAS/metainfo.xml | 4 +-
.../stacks/2.3/ATLAS/test_metadata_server.py | 21 +++++++
.../test/python/stacks/2.3/configs/default.json | 4 ++
.../test/python/stacks/2.3/configs/secure.json | 4 ++
.../stacks/2.5/ATLAS/test_atlas_server.py | 21 +++++++
.../test/python/stacks/2.5/configs/default.json | 4 ++
13 files changed, 178 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
index 95141ca..f3ea36f 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/application-properties.xml
@@ -21,12 +21,12 @@
<configuration supports_final="false">
<property>
<name>atlas.graph.storage.backend</name>
- <value>berkeleyje</value>
+ <value>hbase</value>
<description></description>
</property>
<property>
- <name>atlas.graph.storage.directory</name>
- <value>/var/lib/atlas/data/berkeley</value>
+ <name>atlas.graph.storage.hostname</name>
+ <value>localhost:{{atlas_hbase_zk_port}}</value>
<description></description>
</property>
<property>
@@ -210,4 +210,19 @@
<value>1000</value>
<description></description>
</property>
+ <property>
+ <name>atlas.audit.hbase.tablename</name>
+ <value>ATLAS_ENTITY_AUDIT_EVENTS</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.audit.zookeeper.session.timeout.ms</name>
+ <value>1000</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.audit.hbase.zookeeper.quorum</name>
+ <value>localhost:{{atlas_hbase_zk_port}}</value>
+ <description></description>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
index d64d48c..90eabef 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-env.xml
@@ -102,7 +102,7 @@ export METADATA_DATA_DIR={{data_dir}}
export METADATA_PID_DIR={{pid_dir}}
# hbase conf dir
-export HBASE_CONF_DIR=/etc/hbase/conf
+export HBASE_CONF_DIR={{atlas_hbase_conf_dir}}
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
export METADATA_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
new file mode 100644
index 0000000..3c4826d
--- /dev/null
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/configuration/atlas-hbase-site.xml
@@ -0,0 +1,58 @@
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration supports_final="true">
+ <property>
+ <name>hbase.rootdir</name>
+ <value>file://{{metadata_home}}/data/root</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.dataDir</name>
+ <value>{{metadata_home}}/data/zookeeper-data</value>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>61510</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>61530</value>
+ </property>
+ <property>
+ <name>hbase.master.port</name>
+ <value>61500</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.port</name>
+ <value>61520</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>61581</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.peerport</name>
+ <value>61588</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.leaderport</name>
+ <value>61589</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
index 133b264..5d7766b 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/metainfo.xml
@@ -46,7 +46,7 @@
</logs>
<configFile>
<type>properties</type>
- <fileName>application.properties</fileName>
+ <fileName>application.properties.xml</fileName>
<dictionaryName>application-properties</dictionaryName>
</configFile>
<configFile>
@@ -59,6 +59,11 @@
<fileName>atlas-log4j.xml</fileName>
<dictionaryName>atlas-log4j</dictionaryName>
</configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>atlas-hbase-site.xml</fileName>
+ <dictionaryName>atlas-hbase-site</dictionaryName>
+ </configFile>
</component>
<component>
@@ -75,7 +80,7 @@
</commandScript>
<configFile>
<type>properties</type>
- <fileName>application.properties</fileName>
+ <fileName>application.properties.xml</fileName>
<dictionaryName>application-properties</dictionaryName>
</configFile>
<configFile>
@@ -88,6 +93,11 @@
<fileName>atlas-log4j.xml</fileName>
<dictionaryName>atlas-log4j</dictionaryName>
</configFile>
+ <configFile>
+ <type>xml</type>
+ <fileName>atlas-hbase-site.xml</fileName>
+ <dictionaryName>atlas-hbase-site</dictionaryName>
+ </configFile>
</component>
</components>
@@ -116,6 +126,7 @@
<config-type>application-properties</config-type>
<config-type>atlas-env</config-type>
<config-type>atlas-log4j</config-type>
+ <config-type>atlas-hbase-site</config-type>
</configuration-dependencies>
</service>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
index 041e04e..43c1ea6 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/metadata.py
@@ -18,7 +18,7 @@ limitations under the License.
"""
from resource_management import Directory, Fail, Logger, File, \
- InlineTemplate, PropertiesFile, StaticFile
+ InlineTemplate, PropertiesFile, StaticFile, XmlConfig
from resource_management.libraries.functions import format
from resource_management.libraries.resources.template_config import TemplateConfig
@@ -50,6 +50,22 @@ def metadata():
create_parents = True
)
+ Directory(params.atlas_hbase_log_dir,
+ mode=0755,
+ cd_access='a',
+ owner=params.metadata_user,
+ group=params.user_group,
+ create_parents = True
+ )
+
+ Directory(params.atlas_hbase_data_dir,
+ mode=0755,
+ cd_access='a',
+ owner=params.metadata_user,
+ group=params.user_group,
+ create_parents = True
+ )
+
Directory(params.data_dir,
mode=0644,
cd_access='a',
@@ -91,6 +107,15 @@ def metadata():
content=InlineTemplate(params.metadata_log4j_content)
)
+ # hbase-site for embedded hbase used by Atlas
+ XmlConfig( "hbase-site.xml",
+ conf_dir = params.atlas_hbase_conf_dir,
+ configurations = params.config['configurations']['atlas-hbase-site'],
+ configuration_attributes=params.config['configuration_attributes']['atlas-hbase-site'],
+ owner = params.metadata_user,
+ group = params.user_group
+ )
+
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
index 04f6bf2..4191212 100644
--- a/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/ATLAS/0.1.0.2.3/package/scripts/params.py
@@ -61,6 +61,11 @@ log_dir = config['configurations']['atlas-env']['metadata_log_dir']
conf_dir = status_params.conf_dir # "/etc/metadata/conf"
conf_file = status_params.conf_file
+atlas_hbase_conf_dir = os.path.join(metadata_home, "hbase", "conf")
+atlas_hbase_log_dir = os.path.join(metadata_home, "hbase", "logs")
+atlas_hbase_data_dir = os.path.join(metadata_home, "data")
+atlas_hbase_zk_port = default("/configurations/atlas-hbase-site/hbase.zookeeper.property.clientPort", None)
+
# service locations
hadoop_conf_dir = os.path.join(os.environ["HADOOP_HOME"], "conf") if 'HADOOP_HOME' in os.environ else '/etc/hadoop/conf'
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
index 42503b5..a2ea7e5 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/configuration/atlas-env.xml
@@ -102,7 +102,7 @@
export ATLAS_PID_DIR={{pid_dir}}
# hbase conf dir
- export HBASE_CONF_DIR=/etc/hbase/conf
+ export HBASE_CONF_DIR={{atlas_hbase_conf_dir}}
# Where do you want to expand the war file. By Default it is in /server/webapp dir under the base install dir.
export ATLAS_EXPANDED_WEBAPP_DIR={{expanded_war_dir}}
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
index c787b40..92f198a 100644
--- a/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
+++ b/ambari-server/src/main/resources/stacks/HDP/2.5/services/ATLAS/metainfo.xml
@@ -27,7 +27,7 @@
<cardinality>1+</cardinality>
<configFile>
<type>properties</type>
- <fileName>atlas-application.properties</fileName>
+ <fileName>application.properties.xml</fileName>
<dictionaryName>application-properties</dictionaryName>
</configFile>
<logs>
@@ -42,7 +42,7 @@
<name>ATLAS_CLIENT</name>
<configFile>
<type>properties</type>
- <fileName>atlas-application.properties</fileName>
+ <fileName>application.properties.xml</fileName>
<dictionaryName>application-properties</dictionaryName>
</configFile>
</component>
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
index b1db109..cf18f92 100644
--- a/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
+++ b/ambari-server/src/test/python/stacks/2.3/ATLAS/test_metadata_server.py
@@ -52,6 +52,20 @@ class TestMetadataServer(RMFTestCase):
cd_access='a',
mode=0755
)
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
self.assertResourceCalled('Directory', '/var/lib/atlas/data',
owner='atlas',
group='hadoop',
@@ -96,6 +110,13 @@ class TestMetadataServer(RMFTestCase):
group='hadoop',
mode=0644,
)
+ self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
+ owner = 'atlas',
+ group = 'hadoop',
+ conf_dir = '/usr/hdp/current/atlas-server/hbase/conf',
+ configurations = self.getConfig()['configurations']['atlas-hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site']
+ )
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/default.json b/ambari-server/src/test/python/stacks/2.3/configs/default.json
index f87a4b8..d423ce5 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/default.json
@@ -217,6 +217,9 @@
"atlas_log_level": "debug",
"audit_log_level": "OFF"
},
+ "atlas-hbase-site": {
+ "hbase.master.info.port": "61510"
+ },
"ranger-hbase-plugin-properties": {
"ranger-hbase-plugin-enabled":"yes"
},
@@ -261,6 +264,7 @@
"kafka-env": {},
"kafka-log4j": {},
"kafka-broker": {},
+ "atlas-hbase-site": {},
"metadata-env": {}
},
"configurationTags": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.3/configs/secure.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/secure.json b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
index 524484f..89ab6f0 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/secure.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/secure.json
@@ -209,6 +209,9 @@
"atlas_log_level": "debug",
"audit_log_level": "OFF"
},
+ "atlas-hbase-site": {
+ "hbase.master.info.port": "61510"
+ },
"ranger-hbase-plugin-properties": {
"ranger-hbase-plugin-enabled":"yes"
},
@@ -252,6 +255,7 @@
"kafka-env": {},
"kafka-log4j": {},
"kafka-broker": {},
+ "atlas-hbase-site": {},
"metadata-env": {}
},
"configurationTags": {
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
index a882189..83d4c5b 100644
--- a/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
+++ b/ambari-server/src/test/python/stacks/2.5/ATLAS/test_atlas_server.py
@@ -49,6 +49,20 @@ class TestAtlasServer(RMFTestCase):
cd_access='a',
mode=0755
)
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/hbase/logs',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
+ self.assertResourceCalled('Directory', '/usr/hdp/current/atlas-server/data',
+ owner='atlas',
+ group='hadoop',
+ create_parents = True,
+ cd_access='a',
+ mode=0755
+ )
self.assertResourceCalled('Directory', '/var/lib/atlas/data',
owner='atlas',
group='hadoop',
@@ -94,6 +108,13 @@ class TestAtlasServer(RMFTestCase):
group='hadoop',
mode=0644,
)
+ self.assertResourceCalled('XmlConfig', 'hbase-site.xml',
+ owner = 'atlas',
+ group = 'hadoop',
+ conf_dir = '/usr/hdp/current/atlas-server/hbase/conf',
+ configurations = self.getConfig()['configurations']['atlas-hbase-site'],
+ configuration_attributes = self.getConfig()['configuration_attributes']['atlas-hbase-site']
+ )
def test_configure_default(self):
self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/metadata_server.py",
http://git-wip-us.apache.org/repos/asf/ambari/blob/5db6f1d2/ambari-server/src/test/python/stacks/2.5/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.5/configs/default.json b/ambari-server/src/test/python/stacks/2.5/configs/default.json
index ddf4c9a..5188730 100644
--- a/ambari-server/src/test/python/stacks/2.5/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.5/configs/default.json
@@ -217,6 +217,9 @@
"atlas_log_level": "debug",
"audit_log_level": "OFF"
},
+ "atlas-hbase-site": {
+ "hbase.master.info.port": "61510"
+ },
"ranger-hbase-plugin-properties": {
"ranger-hbase-plugin-enabled":"yes"
},
@@ -288,6 +291,7 @@
"kafka-log4j": {},
"kafka-broker": {},
"metadata-env": {},
+ "atlas-hbase-site": {},
"tagsync-application-properties": {},
"ranger-tagsync-site": {}
},