You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/04/28 03:40:45 UTC
incubator-ranger git commit: Added handling for ENTITY_DELETED
notification, rolled back 'None' tag-source, addressed review comments
Repository: incubator-ranger
Updated Branches:
refs/heads/master 6b22ecbac -> cba444bfd
Added handling for ENTITY_DELETED notification, rolled back 'None' tag-source, addressed review comments
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/cba444bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/cba444bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/cba444bf
Branch: refs/heads/master
Commit: cba444bfd621725339b14ee805967d73be849d14
Parents: 6b22ecb
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Apr 13 15:54:39 2016 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Wed Apr 27 18:31:52 2016 -0700
----------------------------------------------------------------------
pom.xml | 2 +-
.../ranger/rest/ServiceTagsProcessor.java | 42 ++++++++++++-----
.../conf/templates/installprop2xml.properties | 2 -
.../conf/templates/ranger-tagsync-template.xml | 4 --
tagsync/scripts/setup.py | 2 +-
.../apache/ranger/tagsync/model/TagSource.java | 2 -
.../ranger/tagsync/process/TagSynchronizer.java | 2 +-
.../source/atlas/AtlasNotificationMapper.java | 49 +++++++++++++++++---
.../tagsync/source/atlas/AtlasTagSource.java | 2 +-
.../src/main/resources/ranger-tagsync-site.xml | 4 --
10 files changed, 76 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aef42c6..6377ed3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -119,7 +119,7 @@
<asm.all.version>3.2</asm.all.version>
<asm.version>3.1</asm.version>
<aspectj.version>1.8.2</aspectj.version>
- <atlas.version>0.6-incubating-SNAPSHOT</atlas.version>
+ <atlas.version>0.7-incubating-SNAPSHOT</atlas.version>
<avatica.version>1.7.1</avatica.version>
<c3p0.version>0.9.1.2</c3p0.version>
<calcite.version>0.9.2-incubating</calcite.version>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
index baafb0e..9c19bb0 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -428,8 +428,24 @@ public class ServiceTagsProcessor {
if (CollectionUtils.isNotEmpty(serviceResources)) {
for (RangerServiceResource serviceResource : serviceResources) {
+
+ RangerServiceResource objToDelete = null;
+
try {
- RangerServiceResource objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid());
+
+ if (StringUtils.isNotBlank(serviceResource.getGuid())) {
+ objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid());
+ }
+
+ if (objToDelete == null) {
+ if (MapUtils.isNotEmpty(serviceResource.getResourceElements())) {
+ RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(serviceResource);
+
+ String serviceResourceSignature = serializer.getSignature();
+
+ objToDelete = tagStore.getServiceResourceByServiceAndResourceSignature(serviceResource.getServiceName(), serviceResourceSignature);
+ }
+ }
if (objToDelete != null) {
@@ -512,25 +528,27 @@ public class ServiceTagsProcessor {
List<String> serviceResourcesInDb = tagStore.getServiceResourceGuidsByService(serviceTags.getServiceName());
- for (String dbServiceResourceGuid : serviceResourcesInDb) {
+ if (CollectionUtils.isNotEmpty(serviceResourcesInDb)) {
+ for (String dbServiceResourceGuid : serviceResourcesInDb) {
- if (! serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) {
+ if (!serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations...");
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations...");
+ }
- List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid);
+ List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid);
- if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
- for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
- tagStore.deleteTagResourceMap(tagResourceMap.getId());
+ if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
+ for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
+ tagStore.deleteTagResourceMap(tagResourceMap.getId());
+ }
}
+
+ tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid);
}
- tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid);
}
-
}
// Add/update resources and other tag-model objects provided in service-tags
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/conf/templates/installprop2xml.properties
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/installprop2xml.properties b/tagsync/conf/templates/installprop2xml.properties
index 27d3203..aa0c568 100644
--- a/tagsync/conf/templates/installprop2xml.properties
+++ b/tagsync/conf/templates/installprop2xml.properties
@@ -36,8 +36,6 @@ TAG_SOURCE_FILE_ENABLED = ranger.tagsync.source.file
TAG_SOURCE_FILE_FILENAME = ranger.tagsync.source.file.filename
TAG_SOURCE_FILE_CHECK_INTERVAL_IN_MILLIS = ranger.tagsync.source.file.check.interval.millis
-TAG_SOURCE_NONE_ENABLED = ranger.tagsync.source.none
-
TAGSYNC_ATLAS_TO_RANGER_SERVICE_MAPPING = ranger.tagsync.atlas.to.ranger.service.mapping
TAGSYNC_ATLAS_CUSTOM_RESOURCE_MAPPERS = ranger.tagsync.atlas.custom.resource.mappers
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/conf/templates/ranger-tagsync-template.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf/templates/ranger-tagsync-template.xml b/tagsync/conf/templates/ranger-tagsync-template.xml
index 03bb3fa..3d656df 100644
--- a/tagsync/conf/templates/ranger-tagsync-template.xml
+++ b/tagsync/conf/templates/ranger-tagsync-template.xml
@@ -60,10 +60,6 @@
<value></value>
</property>
<property>
- <name>ranger.tagsync.source.none</name>
- <value></value>
- </property>
- <property>
<name>ranger.tagsync.atlas.to.ranger.service.mapping</name>
<value></value>
</property>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/scripts/setup.py
----------------------------------------------------------------------
diff --git a/tagsync/scripts/setup.py b/tagsync/scripts/setup.py
index 4404cdc..cd91c07 100755
--- a/tagsync/scripts/setup.py
+++ b/tagsync/scripts/setup.py
@@ -49,7 +49,7 @@ log4jFileName = 'log4j.properties'
install2xmlMapFileName = 'installprop2xml.properties'
templateFileName = 'ranger-tagsync-template.xml'
initdProgramName = 'ranger-tagsync'
-atlasApplicationPropFileName = 'application.properties'
+atlasApplicationPropFileName = 'atlas-application.properties'
installTemplateDirName = join(installPropDirName,'templates')
confDistDirName = join(installPropDirName, confDistBaseDirName)
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java
index 01bc365..5ef6c57 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/model/TagSource.java
@@ -24,8 +24,6 @@ import java.util.Properties;
public interface TagSource {
- String TAG_SOURCE_NONE = "None";
-
boolean initialize(Properties properties);
void setTagSink(TagSink sink);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
index 4436a77..ac4dcd0 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/process/TagSynchronizer.java
@@ -279,7 +279,7 @@ public class TagSynchronizer {
className = "org.apache.ranger.tagsync.source.atlas.AtlasTagSource";
} else if (tagSourceName.equals("atlasrest")) {
className = "org.apache.ranger.tagsync.source.atlasrest.AtlasRESTTagSource";
- } else if (!tagSourceName.equalsIgnoreCase(TagSource.TAG_SOURCE_NONE)) {
+ } else {
LOG.error("tagSource name doesn't have any class associated with it. tagSourceName="
+ tagSourceName + ", propertyPrefix=" + propPrefix);
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index fb974a5..510732b 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -25,9 +25,9 @@ import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.IStruct;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.StringUtils;
import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.RangerTagDef;
@@ -39,6 +39,7 @@ import java.util.*;
public class AtlasNotificationMapper {
private static final Log LOG = LogFactory.getLog(AtlasNotificationMapper.class);
+ @SuppressWarnings("unchecked")
public static ServiceTags processEntityNotification(EntityNotification entityNotification) {
ServiceTags ret = null;
@@ -47,13 +48,22 @@ public class AtlasNotificationMapper {
try {
IReferenceableInstance entity = entityNotification.getEntity();
- if (AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) {
- AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits());
- ret = buildServiceTags(entityWithTraits, null);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Ranger not interested in Entity Notification for entity-type " + entityNotification.getEntity().getTypeName());
+ if (entity != null) {
+ if (AtlasResourceMapperUtil.isEntityTypeHandled(entity.getTypeName())) {
+ AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entityNotification.getEntity(), entityNotification.getAllTraits());
+ if (entityNotification.getOperationType() == EntityNotification.OperationType.ENTITY_DELETE) {
+ // Special case for ENTITY_DELETE notifications
+ ret = buildServiceTagsForEntityDeleteNotification(entityWithTraits);
+ } else {
+ ret = buildServiceTags(entityWithTraits, null);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ranger not interested in Entity Notification for entity-type " + entityNotification.getEntity().getTypeName());
+ }
}
+ } else {
+ LOG.error("EntityNotification contains NULL entity");
}
} catch (Exception exception) {
LOG.error("createServiceTags() failed!! ", exception);
@@ -86,6 +96,7 @@ public class AtlasNotificationMapper {
break;
}
case ENTITY_UPDATE:
+ case ENTITY_DELETE:
case TRAIT_ADD:
case TRAIT_DELETE: {
ret = true;
@@ -99,6 +110,30 @@ public class AtlasNotificationMapper {
return ret;
}
+ static private ServiceTags buildServiceTagsForEntityDeleteNotification(AtlasEntityWithTraits entityWithTraits) throws Exception {
+ final ServiceTags ret;
+
+ IReferenceableInstance entity = entityWithTraits.getEntity();
+
+ String guid = entity.getId()._getId();
+ if (StringUtils.isNotBlank(guid)) {
+ ret = new ServiceTags();
+ RangerServiceResource serviceResource = new RangerServiceResource();
+ serviceResource.setGuid(guid);
+ ret.getServiceResources().add(serviceResource);
+ } else {
+ ret = buildServiceTags(entityWithTraits, null);
+ // tag-definitions should NOT be deleted as part of service-resource delete
+ ret.setTagDefinitions(MapUtils.EMPTY_MAP);
+ // Ranger deletes tags associated with deleted service-resource
+ ret.setTags(MapUtils.EMPTY_MAP);
+ }
+
+ ret.setOp(ServiceTags.OP_DELETE);
+
+ return ret;
+ }
+
static private Map<String, ServiceTags> buildServiceTags(List<AtlasEntityWithTraits> entitiesWithTraits) throws Exception {
Map<String, ServiceTags> ret = new HashMap<String, ServiceTags>();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java
index 49d6f61..af19823 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasTagSource.java
@@ -43,7 +43,7 @@ import java.util.*;
public class AtlasTagSource extends AbstractTagSource {
private static final Log LOG = LogFactory.getLog(AtlasTagSource.class);
- public static final String TAGSYNC_ATLAS_PROPERTIES_FILE_NAME = "application.properties";
+ public static final String TAGSYNC_ATLAS_PROPERTIES_FILE_NAME = "atlas-application.properties";
public static final String TAGSYNC_ATLAS_KAFKA_ENDPOINTS = "atlas.kafka.bootstrap.servers";
public static final String TAGSYNC_ATLAS_ZOOKEEPER_ENDPOINT = "atlas.kafka.zookeeper.connect";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/cba444bf/tagsync/src/main/resources/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/src/main/resources/ranger-tagsync-site.xml b/tagsync/src/main/resources/ranger-tagsync-site.xml
index fe300b4..aa2357a 100644
--- a/tagsync/src/main/resources/ranger-tagsync-site.xml
+++ b/tagsync/src/main/resources/ranger-tagsync-site.xml
@@ -62,10 +62,6 @@
<value>60000</value>
</property>
<property>
- <name>ranger.tagsync.source.none</name>
- <value>false</value>
- </property>
- <property>
<name>ranger.tagsync.atlas.to.ranger.service.mapping</name>
<value />
</property>