You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/10/28 17:00:11 UTC
[35/50] [abbrv] incubator-ranger git commit: RANGER-660: updated
Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED
notifications
RANGER-660: updated Atlas notification handler to handle TRAIT_DELETED and ENTITY_UPDATED notifications
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/5db14737
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/5db14737
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/5db14737
Branch: refs/heads/master
Commit: 5db147378f004176638cddc626f6a0e9baa5efc1
Parents: 54270d5
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Oct 1 15:08:21 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Oct 1 19:07:41 2015 -0700
----------------------------------------------------------------------
.../ranger/rest/ServiceTagsProcessor.java | 8 +-
.../source/atlas/AtlasNotificationMapper.java | 153 ++++++++++---------
.../tagsync/source/atlas/TagAtlasSource.java | 2 +-
3 files changed, 88 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
index da99b72..b34313f 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -111,10 +111,10 @@ public class ServiceTagsProcessor {
if(existing == null) {
tagDefInStore = tagStore.createTagDef(tagDef);
} else {
- tagDef.setId(existing.getId());
- tagDef.setGuid(existing.getGuid());
-
- tagDefInStore = tagStore.updateTagDef(tagDef);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("tagDef for name:" + tagDef.getName() + " exists, will not update it");
+ }
+ tagDefInStore = existing;
}
if(tagDefsInStore != null) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index dd5cd44..13d67b3 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -94,19 +94,20 @@ class AtlasNotificationMapper {
String opName = entityNotification.getOperationType().name();
switch (opType) {
case ENTITY_CREATED: {
- ret = getServiceTags(entity, opType);
+ LOG.info("ENTITY_CREATED notification is not handled, as Ranger will get necessary information from any subsequent TRAIT_ADDED notification");
break;
}
case ENTITY_UPDATED: {
- ret = handleEntityUpdate(entity);
- break;
- }
- case TRAIT_ADDED: {
- ret = getServiceTags(entity, opType);
+ ret = getServiceTags(entity);
+ if (MapUtils.isEmpty(ret.getTags())) {
+ LOG.info("No traits associated with this entity update notification. Ignoring it altogether");
+ ret = null;
+ }
break;
}
+ case TRAIT_ADDED:
case TRAIT_DELETED: {
- ret = handleTraitDelete(entity);
+ ret = getServiceTags(entity);
break;
}
default:
@@ -116,31 +117,36 @@ class AtlasNotificationMapper {
return ret;
}
- static private ServiceTags getServiceTags(Entity entity, EntityNotification.OperationType opType) throws Exception {
+ static private ServiceTags getServiceTags(Entity entity) throws Exception {
ServiceTags ret = null;
List<RangerServiceResource> serviceResources = new ArrayList<RangerServiceResource>();
- RangerServiceResource serviceResource = getServiceResource(entity, opType);
+ RangerServiceResource serviceResource = getServiceResource(entity);
serviceResources.add(serviceResource);
Map<Long, RangerTag> tags = getTags(entity);
- Map<Long, RangerTagDef> tagDefs = getTagDefs(tags, EntityNotification.OperationType.ENTITY_CREATED);
+ Map<Long, RangerTagDef> tagDefs = getTagDefs(tags);
Map<Long, List<Long>> resourceIdToTagIds = null;
+ resourceIdToTagIds = new HashMap<Long, List<Long>>();
+ List<Long> tagList = new ArrayList<Long>();
+
+
if (MapUtils.isNotEmpty(tags)) {
resourceIdToTagIds = new HashMap<Long, List<Long>>();
- List<Long> tagList = new ArrayList<Long>();
for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
tagList.add(entry.getKey());
}
- resourceIdToTagIds.put(1L, tagList);
}
+ resourceIdToTagIds.put(1L, tagList);
+
+
ret = new ServiceTags();
ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
@@ -154,68 +160,67 @@ class AtlasNotificationMapper {
}
- static private RangerServiceResource getServiceResource(Entity entity, EntityNotification.OperationType opType) throws Exception {
+ static private RangerServiceResource getServiceResource(Entity entity) throws Exception {
RangerServiceResource ret = null;
Map<String, RangerPolicy.RangerPolicyResource> elements = null;
String serviceName = null;
- if (opType == EntityNotification.OperationType.ENTITY_CREATED) {
- elements = new HashMap<String, RangerPolicy.RangerPolicyResource>();
+ elements = new HashMap<String, RangerPolicy.RangerPolicyResource>();
- String[] components = getQualifiedNameComponents(entity);
- // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order
+ String[] components = getQualifiedNameComponents(entity);
+ // components should contain qualifiedName, instanceName, dbName, tableName, columnName in that order
- String entityTypeName = entity.getTypeName();
+ String entityTypeName = entity.getTypeName();
- String instanceName, dbName, tableName, columnName;
+ String instanceName, dbName, tableName, columnName;
- if (components.length > 1) {
- instanceName = components[1];
- serviceName = getServiceName(instanceName, entityTypeName);
- }
-
- if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) {
- if (components.length > 2) {
- dbName = components[2];
- RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
- elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+ if (components.length > 1) {
+ instanceName = components[1];
+ serviceName = getServiceName(instanceName, entityTypeName);
+ }
- } else {
- LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]);
- }
- } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) {
- if (components.length > 3) {
- dbName = components[2];
- tableName = components[3];
- RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
- elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
- RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
- elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
- } else {
- LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]);
- }
- } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) {
- if (components.length > 4) {
- dbName = components[2];
- tableName = components[3];
- columnName = components[4];
- RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
- elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
- RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
- elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
- RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName);
- elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource);
- } else {
- LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]);
- }
+ if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_DB)) {
+ if (components.length > 2) {
+ dbName = components[2];
+ RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+ elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+ } else {
+ LOG.error("invalid qualifiedName for HIVE_DB, qualifiedName=" + components[0]);
+ }
+ } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_TABLE)) {
+ if (components.length > 3) {
+ dbName = components[2];
+ tableName = components[3];
+ RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+ elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+ RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
+ elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
+ } else {
+ LOG.error("invalid qualifiedName for HIVE_TABLE, qualifiedName=" + components[0]);
}
+ } else if (StringUtils.equals(entityTypeName, ENTITY_TYPE_HIVE_COLUMN)) {
+ if (components.length > 4) {
+ dbName = components[2];
+ tableName = components[3];
+ columnName = components[4];
+ RangerPolicy.RangerPolicyResource dbPolicyResource = new RangerPolicy.RangerPolicyResource(dbName);
+ elements.put(RANGER_TYPE_HIVE_DB, dbPolicyResource);
+ RangerPolicy.RangerPolicyResource tablePolicyResource = new RangerPolicy.RangerPolicyResource(tableName);
+ elements.put(RANGER_TYPE_HIVE_TABLE, tablePolicyResource);
+ RangerPolicy.RangerPolicyResource columnPolicyResource = new RangerPolicy.RangerPolicyResource(columnName);
+ elements.put(RANGER_TYPE_HIVE_COLUMN, columnPolicyResource);
+ } else {
+ LOG.error("invalid qualifiedName for HIVE_COLUMN, qualifiedName=" + components[0]);
+ }
+
}
+
ret = new RangerServiceResource();
ret.setGuid(entity.getId().getGuid());
ret.setId(1L);
@@ -255,7 +260,6 @@ class AtlasNotificationMapper {
RangerTag tag = new RangerTag();
- tag.setGuid(entity.getId().getGuid() + "-" + traitName);
tag.setType(traitName);
tag.setAttributes(tagAttrValues);
@@ -266,19 +270,17 @@ class AtlasNotificationMapper {
return ret;
}
- static private Map<Long, RangerTagDef> getTagDefs(Map<Long, RangerTag> tags, EntityNotification.OperationType opType) {
+ static private Map<Long, RangerTagDef> getTagDefs(Map<Long, RangerTag> tags) {
Map<Long, RangerTagDef> ret = null;
- if (opType == EntityNotification.OperationType.ENTITY_CREATED || opType == EntityNotification.OperationType.TRAIT_ADDED) {
- if (MapUtils.isNotEmpty(tags)) {
- ret = new HashMap<Long, RangerTagDef>();
- for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
- RangerTagDef tagDef = new RangerTagDef();
- tagDef.setName(entry.getValue().getType());
- tagDef.setId(entry.getKey());
- ret.put(entry.getKey(), tagDef);
- }
+ if (MapUtils.isNotEmpty(tags)) {
+ ret = new HashMap<Long, RangerTagDef>();
+ for (Map.Entry<Long, RangerTag> entry : tags.entrySet()) {
+ RangerTagDef tagDef = new RangerTagDef();
+ tagDef.setName(entry.getValue().getType());
+ tagDef.setId(entry.getKey());
+ ret.put(entry.getKey(), tagDef);
}
}
@@ -289,10 +291,21 @@ class AtlasNotificationMapper {
String ret[] = new String[5];
if (StringUtils.equals(entity.getTypeName(), ENTITY_TYPE_HIVE_DB)) {
- ret[1] = getAttribute(entity.getValues(), "clusterName", String.class);
- ret[2] = getAttribute(entity.getValues(), "name", String.class);
+
+ String clusterName = getAttribute(entity.getValues(), "clusterName", String.class);
+ String name = getAttribute(entity.getValues(), "name", String.class);
+
+ ret[1] = clusterName;
+ ret[2] = name;
ret[3] = null;
ret[0] = ret[1] + "." + ret[2];
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("----- Entity-Id:" + entity.getId().getGuid());
+ LOG.debug("----- Entity-Type-Name:" + entity.getTypeName());
+ LOG.debug("----- Entity-Cluster-Name:" + clusterName);
+ LOG.debug("----- Entity-Name:" + name);
+ }
} else {
String qualifiedName = getAttribute(entity.getValues(), ENTITY_ATTRIBUTE_QUALIFIED_NAME, String.class);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/5db14737/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
index 243aee5..0c0dee9 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/TagAtlasSource.java
@@ -151,7 +151,7 @@ public class TagAtlasSource implements TagSource {
printNotification(notification);
ServiceTags serviceTags = AtlasNotificationMapper.processEntityNotification(notification, properties);
if (serviceTags == null) {
- LOG.error("Failed to map Atlas notification to ServiceTags structure");
+ LOG.info("Did not create ServiceTags structure for notification type:" + notification.getOperationType().name());
} else {
if (LOG.isDebugEnabled()) {
String serviceTagsJSON = new Gson().toJson(serviceTags);