You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/12/12 18:34:41 UTC
incubator-ranger git commit: RANGER-780: update Atlas notification
processing for recent changes to qualifiedName attribute
Repository: incubator-ranger
Updated Branches:
refs/heads/master 269617d5d -> 6194ea0c4
RANGER-780: update Atlas notification processing for recent changes to qualifiedName attribute
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/6194ea0c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/6194ea0c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/6194ea0c
Branch: refs/heads/master
Commit: 6194ea0c4d4059669f4f679741991fcfba3e495c
Parents: 269617d
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Dec 9 17:01:22 2015 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Dec 12 09:30:25 2015 -0800
----------------------------------------------------------------------
.../source/atlas/AtlasNotificationMapper.java | 117 ++++++++++++-------
.../tagsync/process/TestTagSynchronizer.java | 32 +++++
2 files changed, 106 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6194ea0c/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index 7a2422b..666c2c8 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -36,7 +36,7 @@ import org.apache.ranger.tagsync.process.TagSyncConfig;
import java.util.*;
-class AtlasNotificationMapper {
+public class AtlasNotificationMapper {
private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class);
public static final String ENTITY_TYPE_HIVE_DB = "hive_db";
@@ -50,8 +50,7 @@ class AtlasNotificationMapper {
public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME = "qualifiedName";
public static final String ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE = "name";
public static final String QUALIFIED_NAME_FORMAT_DELIMITER_STRING = "\\.";
-
- private static final int MAX_HIERARCHY_LEVELS = 5;
+ public static final String QUALIFIED_NAME_FORMAT_CLUSTER_DELIMITER_STRING = "@";
private static Properties properties = null;
@@ -177,43 +176,45 @@ class AtlasNotificationMapper {
elements = new HashMap<String, RangerPolicy.RangerPolicyResource>();
- String[] components = getQualifiedNameComponents(entity);
- // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order
+ List<String> components = getQualifiedNameComponents(entity);
+ // components should contain qualifiedName, clusterName, dbName, tableName, columnName in that order
String entityTypeName = entity.getTypeName();
- String instanceName, dbName, tableName, columnName;
+ String qualifiedName = components.get(0);
+
+ String clusterName, dbName, tableName, columnName;
- if (components.length > 1) {
- instanceName = components[1];
- serviceName = getServiceName(instanceName, entityTypeName);
+ if (components.size() > 1) {
+ clusterName = components.get(1);
+ serviceName = getServiceName(clusterName, entityTypeName);
}
if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) {
- if (components.length > 2) {
- dbName = components[2];
+ if (components.size() > 2) {
+ dbName = components.get(2);
RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
} else {
- LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]);
+ LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + qualifiedName);
}
} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) {
- if (components.length > 3) {
- dbName = components[2];
- tableName = components[3];
+ if (components.size() > 3) {
+ dbName = components.get(2);
+ tableName = components.get(3);
RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
} else {
- LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]);
+ LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + qualifiedName);
}
} else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) {
- if (components.length > 4) {
- dbName = components[2];
- tableName = components[3];
- columnName = components[4];
+ if (components.size() > 4) {
+ dbName = components.get(2);
+ tableName = components.get(3);
+ columnName = components.get(4);
RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
@@ -221,7 +222,7 @@ class AtlasNotificationMapper {
RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName);
elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource);
} else {
- LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]);
+ LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + qualifiedName);
}
}
@@ -298,46 +299,76 @@ class AtlasNotificationMapper {
return ret;
}
- static private String[] getQualifiedNameComponents(IReferenceableInstance entity) throws Exception {
- String ret[] = new String[MAX_HIERARCHY_LEVELS];
-
- String qualifiedNameAttributeName = StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_TABLE) ?
+ static private String getQualifiedNameAttributeName(String entityTypeName) {
+ String ret = StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE) ?
ENTITY_ATTRIBUTE_QUALIFIED_NAME_FOR_HIVE_TABLE : ENTITY_ATTRIBUTE_QUALIFIED_NAME;
- String qualifiedName = getEntityAttribute(entity, qualifiedNameAttributeName, String.class);
+ return ret;
+ }
- if (StringUtils.isBlank(qualifiedName)) {
- throw new Exception("Could not get a valid value for " + qualifiedNameAttributeName + " attribute from entity notification.");
- }
+ static private List<String> getQualifiedNameComponents(IReferenceableInstance entity) throws Exception {
- String nameHierarchy[] = qualifiedName.split(QUALIFIED_NAME_FORMAT_DELIMITER_STRING);
+ List<String> ret = null;
- int hierarchyLevels = nameHierarchy.length;
+ String qualifiedNameAttributeName = getQualifiedNameAttributeName(entity.getTypeName());
+
+ String qualifiedName = getEntityAttribute(entity, qualifiedNameAttributeName, String.class);
+
+ ret = getQualifiedNameComponents(entity.getTypeName(), qualifiedName);
if (LOG.isDebugEnabled()) {
LOG.debug("----- Entity-Id:" + entity.getId()._getId());
LOG.debug("----- Entity-Type-Name:" + entity.getTypeName());
- LOG.debug("----- Entity-Qualified-Name:" + qualifiedName);
- LOG.debug("----- Entity-Qualified-Name-Components -----");
- for (int i = 0; i < hierarchyLevels; i++) {
- LOG.debug("----- Index:" + i + " Value:" + nameHierarchy[i]);
+ LOG.debug("----- Entity-Components -----");
+ int i = 0;
+ for (String value : ret) {
+ LOG.debug("----- Index:" + i++ + " Value:" + value);
}
}
+ return ret;
+ }
+
+ static public List<String> getQualifiedNameComponents(String entityTypeName, String qualifiedName) throws Exception {
+
+ List<String> ret = null;
+
+ String qualifiedNameAttributeName = getQualifiedNameAttributeName(entityTypeName);
+
+ if (StringUtils.isBlank(qualifiedName)) {
+ throw new Exception("Could not get a valid value for " + qualifiedNameAttributeName + " attribute from entity notification.");
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Received .... " + qualifiedNameAttributeName + "=" + qualifiedName + " for entity type " + entityTypeName);
+ }
+
+ String components[] = qualifiedName.split(QUALIFIED_NAME_FORMAT_CLUSTER_DELIMITER_STRING);
- int i;
- for (i = 0; i < ret.length; i++) {
- ret[i] = null;
+ if (components == null || components.length != 2) {
+ throw new Exception("Qualified Name does not contain cluster-name, qualifiedName=" + qualifiedName);
}
- ret[0] = qualifiedName;
- for (i = 0; i < hierarchyLevels; i++) {
- ret[i + 1] = nameHierarchy[i];
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("name-hierarchy=" + components[0] + ", cluster-name=" + components[1]);
+ }
+
+ String nameHierarchy[] = components[0].split(QUALIFIED_NAME_FORMAT_DELIMITER_STRING);
+
+ int hierarchyLevels = nameHierarchy.length;
+
+ ret = new ArrayList<String>();
+
+ ret.add(qualifiedName);
+ ret.add(components[1]);
+
+ for (int i = 0; i < hierarchyLevels; i++) {
+ ret.add(nameHierarchy[i]);
}
return ret;
}
- static private String getServiceName(String instanceName, String entityTypeName) {
+ static private String getServiceName(String clusterName, String entityTypeName) {
// Parse entityTypeName to get the Apache-component Name
// Assumption: entityTypeName is <componentName>_<component_specific_type_name>
// such as hive_table, hadoop_path, hbase_queue, etc.
@@ -347,7 +378,7 @@ class AtlasNotificationMapper {
apacheComponent = apacheComponents[0].toLowerCase();
}
- return TagSyncConfig.getServiceName(apacheComponent, instanceName, properties);
+ return TagSyncConfig.getServiceName(apacheComponent, clusterName, properties);
}
static private <T> T getEntityAttribute(IReferenceableInstance entity, String name, Class<T> type) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/6194ea0c/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
index 9d603d4..59d521c 100644
--- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
+++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestTagSynchronizer.java
@@ -20,15 +20,18 @@
package org.apache.ranger.tagsync.process;
+import org.apache.commons.lang.StringUtils;
import org.apache.ranger.tagsync.model.TagSource;
import org.apache.ranger.tagsync.process.TagSyncConfig;
import org.apache.ranger.tagsync.process.TagSynchronizer;
+import org.apache.ranger.tagsync.source.atlas.AtlasNotificationMapper;
import org.apache.ranger.tagsync.source.atlas.TagAtlasSource;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
import java.io.*;
+import java.util.List;
import java.util.Properties;
import static org.junit.Assert.*;
@@ -45,6 +48,7 @@ public class TestTagSynchronizer {
TagSyncConfig config = TagSyncConfig.getInstance();
TagSyncConfig.dumpConfiguration(config, new BufferedWriter(new OutputStreamWriter(System.out)));
+ System.out.println();
Properties props = config.getProperties();
@@ -99,4 +103,32 @@ public class TestTagSynchronizer {
System.out.println("Exiting testTagDownload()");
}
+
+ @Test
+ public void testQualifiedNames() {
+
+ List<String> components;
+ try {
+ components = AtlasNotificationMapper.getQualifiedNameComponents("hive_db", "database@cluster");
+ printComponents(components);
+
+ components = AtlasNotificationMapper.getQualifiedNameComponents("hive_table", "database.table@cluster");
+ printComponents(components);
+
+ components = AtlasNotificationMapper.getQualifiedNameComponents("hive_column", "database.table.column@cluster");
+ printComponents(components);
+
+ assert(true);
+ } catch (Exception e) {
+ System.out.println("Failed...");
+ assert(false);
+ }
+
+ }
+ private void printComponents(List<String> components) {
+ int i = 0;
+ for (String value : components) {
+ System.out.println("----- Index:" + i++ + " Value:" + value);
+ }
+ }
}