You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/10/28 17:00:18 UTC
[42/50] [abbrv] incubator-ranger git commit: RANGER-660: TagSync
updated to support mapping of Atlas instance-name to Ranger service-name via
property
RANGER-660: TagSync updated to support mapping of Atlas instance-name to Ranger service-name via property
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/892f6bf3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/892f6bf3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/892f6bf3
Branch: refs/heads/master
Commit: 892f6bf3002ee18225bfcf758b9662513d3d1171
Parents: 2f64a68
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Oct 6 14:29:53 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Oct 14 09:52:31 2015 -0700
----------------------------------------------------------------------
.../conf/templates/installprop2xml.properties | 2 ++
.../conf/templates/ranger-tagsync-template.xml | 4 +++
tagsync/scripts/install.properties | 7 ++++-
tagsync/scripts/setup.py | 27 ++++++++++++++++++--
.../source/atlas/AtlasNotificationMapper.java | 2 ++
5 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/conf/templates/installprop2xml.properties
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties
index 9fe1cac..5d445ce 100644
--- a/tagsync/conf/templates/installprop2xml.properties
+++ b/tagsync/conf/templates/installprop2xml.properties
@@ -34,3 +34,5 @@ TAGSYNC_FILESOURCE_FILENAME = ranger.tagsync.filesource.filename
TAGSYNC_ATLAS_KAFKA_ENDPOINTS = atlas.notification.kafka.bootstrap.servers
TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = atlas.notification.kafka.zookeeper.connect
TAGSYNC_ATLAS_CONSUMER_GROUP = atlas.notification.kafka.group.id
+
+TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.service.mapping
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/conf/templates/ranger-tagsync-template.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml
index 5f9f3b5..f9bc31b 100644
--- a/tagsync/conf/templates/ranger-tagsync-template.xml
+++ b/tagsync/conf/templates/ranger-tagsync-template.xml
@@ -51,4 +51,8 @@
<name>ranger.tagsync.source.impl.class</name>
<value></value>
</property>
+ <property>
+ <name>ranger.tagsync.atlas.to.service.mapping</name>
+ <value></value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/scripts/install.properties
----------------------------------------------------------------------
diff --git a/tagsync/scripts/install.properties b/tagsync/scripts/install.properties
index 82477d3..fb11ede 100644
--- a/tagsync/scripts/install.properties
+++ b/tagsync/scripts/install.properties
@@ -55,4 +55,9 @@ SYNC_INTERVAL = 60000
TAGSYNC_ATLAS_KAFKA_ENDPOINTS = localhost:6667
TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = localhost:2181
-TAGSYNC_ATLAS_CONSUMER_GROUP = entityConsumer
\ No newline at end of file
+TAGSYNC_ATLAS_CONSUMER_GROUP = entityConsumer
+
+# Mapping from Atlas hive instance-name to Ranger service-name
+# this needs to be in format clusterName,componentType,serviceName;clusterName2,componentType2,serviceName2
+# Note that there are no blanks anywhere in the value-string
+TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = c0,hive,cl1_hive;c0,hbase,other_hbase
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/scripts/setup.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py
index 383964a..faa0af4 100755
--- a/tagsync/scripts/setup.py
+++ b/tagsync/scripts/setup.py
@@ -70,7 +70,10 @@ SYNC_INTERVAL_NEW_KEY = 'ranger.tagsync.sleeptimeinmillisbetweensynccycle'
TAGSYNC_ATLAS_KAFKA_ENDPOINTS_KEY = 'TAGSYNC_ATLAS_KAFKA_ENDPOINTS'
TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT_KEY = 'TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT'
TAGSYNC_ATLAS_CONSUMER_GROUP_KEY = 'TAGSYNC_ATLAS_CONSUMER_GROUP'
-
+TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = 'ranger.tagsync.atlas.to.service.mapping'
+TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING = 'ranger.tagsync.atlas.'
+TAGSYNC_ATLAS_CLUSTER_IDENTIFIER = '.instance.'
+TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING = '.ranger.service'
TAG_SOURCE_ATLAS = 'atlas'
TAG_SOURCE_FILE = 'file'
TAG_SOURCE_LIST = [ TAG_SOURCE_ATLAS, TAG_SOURCE_FILE ]
@@ -138,7 +141,27 @@ def writeXMLUsingProperties(xmlTemplateFileName,prop,xmlOutputFileName):
config.find('value').text = "_"
continue
if (name in prop.keys()):
- config.find('value').text = str(prop[name])
+ if (name == TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING):
+ # Expected value is 'clusterName,componentName,serviceName;clusterName,componentName,serviceName' ...
+ # Blanks are not supported anywhere in the value.
+ valueString = str(prop[name])
+ multiValues = valueString.split(';')
+ listLen = len(multiValues)
+ index = 0
+ while index < listLen:
+ parts = multiValues[index].split(',')
+ if len(parts) == 3:
+ newConfig = ET.SubElement(root, 'property')
+ newName = ET.SubElement(newConfig, 'name')
+ newValue = ET.SubElement(newConfig, 'value')
+ newName.text = TAGSYNC_INSTALL_PROP_PREFIX_FOR_ATLAS_RANGER_MAPPING + str(parts[1]) + TAGSYNC_ATLAS_CLUSTER_IDENTIFIER + str(parts[0]) + TAGSYNC_INSTALL_PROP_SUFFIX_FOR_ATLAS_RANGER_MAPPING
+ newValue.text = str(parts[2])
+ else:
+ print "ERROR: incorrect syntax for %s, value=%s" % (TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING, multiValues[index])
+ index += 1
+ root.remove(config)
+ else:
+ config.find('value').text = str(prop[name])
#else:
# print "ERROR: key not found: %s" % (name)
if isfile(xmlOutputFileName):
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/892f6bf3/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index d5108a1..8046b68 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -340,6 +340,8 @@ class AtlasNotificationMapper {
static private String getServiceName(String instanceName, String entityTypeName) {
// Parse entityTypeName to get the Apache-component Name
+ // Assumption: entityTypeName is <componentName>_<component_specific_type_name>
+ // such as hive_table, hadoop_path, hbase_queue, etc.
String apacheComponents[] = entityTypeName.split("_");
String apacheComponent = null;
if (apacheComponents.length > 0) {