You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ve...@apache.org on 2017/02/10 23:41:22 UTC
ranger git commit: RANGER-1357: tag objects are not removed when
attempting a full sync with Atlas
Repository: ranger
Updated Branches:
refs/heads/master fbc7f19db -> e4122919c
RANGER-1357: tag objects are not removed when attempting a full sync with Atlas
Signed-off-by: Velmurugan Periasamy <ve...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/ranger/commit/e4122919
Tree: http://git-wip-us.apache.org/repos/asf/ranger/tree/e4122919
Diff: http://git-wip-us.apache.org/repos/asf/ranger/diff/e4122919
Branch: refs/heads/master
Commit: e4122919c46bd9fec38a76b27c7e0f5fb98a5f5b
Parents: fbc7f19
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Wed Feb 1 17:18:20 2017 -0800
Committer: Velmurugan Periasamy <ve...@apache.org>
Committed: Fri Feb 10 18:41:09 2017 -0500
----------------------------------------------------------------------
.../source/atlas/AtlasNotificationMapper.java | 44 ++++++++++++--------
.../source/atlasrest/AtlasRESTTagSource.java | 8 ++--
.../tagsync/source/atlasrest/AtlasRESTUtil.java | 28 ++++++-------
3 files changed, 44 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index ec8508e..922317e 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -108,7 +108,7 @@ public class AtlasNotificationMapper {
return ret;
}
- public static Map<String, ServiceTags> processEntitiesWithTraits(List<AtlasEntityWithTraits> atlasEntities) {
+ public static Map<String, ServiceTags> processAtlasEntities(List<AtlasEntityWithTraits> atlasEntities) {
Map<String, ServiceTags> ret = null;
try {
@@ -245,32 +245,40 @@ public class AtlasNotificationMapper {
RangerServiceResource serviceResource = AtlasResourceMapperUtil.getRangerServiceResource(entity);
if (serviceResource != null) {
- List<RangerTag> tags = getTags(entityWithTraits);
- List<RangerTagDef> tagDefs = getTagDefs(entityWithTraits);
- String serviceName = serviceResource.getServiceName();
+ List<RangerTag> tags = getTags(entityWithTraits);
+ List<RangerTagDef> tagDefs = getTagDefs(entityWithTraits);
+ String serviceName = serviceResource.getServiceName();
ret = createOrGetServiceTags(serviceTagsMap, serviceName);
- serviceResource.setId((long)ret.getServiceResources().size());
- ret.getServiceResources().add(serviceResource);
+ if (serviceTagsMap == null || CollectionUtils.isNotEmpty(tags)) {
+
+ serviceResource.setId((long) ret.getServiceResources().size());
+ ret.getServiceResources().add(serviceResource);
- List<Long> tagIds = new ArrayList<Long>();
+ List<Long> tagIds = new ArrayList<Long>();
- if(CollectionUtils.isNotEmpty(tags)) {
- for(RangerTag tag : tags) {
- tag.setId((long)ret.getTags().size());
- ret.getTags().put(tag.getId(), tag);
+ if (CollectionUtils.isNotEmpty(tags)) {
+ for (RangerTag tag : tags) {
+ tag.setId((long) ret.getTags().size());
+ ret.getTags().put(tag.getId(), tag);
- tagIds.add(tag.getId());
+ tagIds.add(tag.getId());
+ }
}
- }
- ret.getResourceToTagIds().put(serviceResource.getId(), tagIds);
+ ret.getResourceToTagIds().put(serviceResource.getId(), tagIds);
- if(CollectionUtils.isNotEmpty(tagDefs)) {
- for(RangerTagDef tagDef : tagDefs) {
- tagDef.setId((long)ret.getTagDefinitions().size());
- ret.getTagDefinitions().put(tagDef.getId(), tagDef);
+ if (CollectionUtils.isNotEmpty(tagDefs)) {
+ for (RangerTagDef tagDef : tagDefs) {
+ tagDef.setId((long) ret.getTagDefinitions().size());
+ ret.getTagDefinitions().put(tagDef.getId(), tagDef);
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Entity " + entityWithTraits + " does not have any tags associated with it when full-sync is being done.");
+ LOG.debug("Will not add this entity to serviceTags, so that this entity, if exists, will be removed from ranger");
}
}
} else {
http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
index e0976cd..7da4ed4 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTTagSource.java
@@ -194,16 +194,16 @@ public class AtlasRESTTagSource extends AbstractTagSource implements Runnable {
public void synchUp() {
- List<AtlasEntityWithTraits> atlasEntitiesWithTraits = atlasRESTUtil.getEntitiesWithTraits();
+ List<AtlasEntityWithTraits> atlasEntities = atlasRESTUtil.getAtlasEntities();
- if (CollectionUtils.isNotEmpty(atlasEntitiesWithTraits)) {
+ if (CollectionUtils.isNotEmpty(atlasEntities)) {
if (LOG.isDebugEnabled()) {
- for (AtlasEntityWithTraits element : atlasEntitiesWithTraits) {
+ for (AtlasEntityWithTraits element : atlasEntities) {
LOG.debug(element);
}
}
- Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processEntitiesWithTraits(atlasEntitiesWithTraits);
+ Map<String, ServiceTags> serviceTagsMap = AtlasNotificationMapper.processAtlasEntities(atlasEntities);
if (MapUtils.isNotEmpty(serviceTagsMap)) {
for (Map.Entry<String, ServiceTags> entry : serviceTagsMap.entrySet()) {
http://git-wip-us.apache.org/repos/asf/ranger/blob/e4122919/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
index bf09cfd..167fe68 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlasrest/AtlasRESTUtil.java
@@ -88,9 +88,9 @@ public class AtlasRESTUtil {
}
}
- public List<AtlasEntityWithTraits> getEntitiesWithTraits() {
+ public List<AtlasEntityWithTraits> getAtlasEntities() {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> getEntriesWithTraits()");
+ LOG.debug("==> getAtlasEntities()");
}
List<AtlasEntityWithTraits> ret = new ArrayList<AtlasEntityWithTraits>();
@@ -129,9 +129,9 @@ public class AtlasRESTUtil {
Map<String, Object> traitsAttribute = getAttribute(definition, TRAITS_ATTRIBUTE, Map.class);
- if (MapUtils.isNotEmpty(traitsAttribute)) {
+ List<IStruct> allTraits = new LinkedList<>();
- List<IStruct> allTraits = new LinkedList<>();
+ if (MapUtils.isNotEmpty(traitsAttribute)) {
for (Map.Entry<String, Object> entry : traitsAttribute.entrySet()) {
@@ -151,24 +151,24 @@ public class AtlasRESTUtil {
allTraits.add(trait1);
allTraits.addAll(superTypes);
}
+ }
- IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true);
+ IReferenceableInstance entity = InstanceSerialization.fromJsonReferenceable(gson.toJson(definition), true);
- if (entity != null) {
- AtlasEntityWithTraits entityWithTraits = new AtlasEntityWithTraits(entity, allTraits);
- ret.add(entityWithTraits);
- } else {
- if (LOG.isInfoEnabled()) {
- LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid);
- }
+ if (entity != null) {
+ AtlasEntityWithTraits atlasEntity = new AtlasEntityWithTraits(entity, allTraits);
+ ret.add(atlasEntity);
+ } else {
+ if (LOG.isInfoEnabled()) {
+ LOG.info("Could not create Atlas entity from its definition, type=" + type + ", guid=" + guid);
}
-
}
}
+
}
if (LOG.isDebugEnabled()) {
- LOG.debug("<== getEntriesWithTraits()");
+ LOG.debug("<== getAtlasEntities()");
}
}