You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2016/04/09 11:36:58 UTC
incubator-ranger git commit: RANGER-807: TagSync - fixed 'replace' to
delete service-resources that are not in full-sync serviceTags
Repository: incubator-ranger
Updated Branches:
refs/heads/master 3a363c530 -> 2867cc55e
RANGER-807: TagSync - fixed 'replace' to delete service-resources that are not in full-sync serviceTags
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/2867cc55
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/2867cc55
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/2867cc55
Branch: refs/heads/master
Commit: 2867cc55e7f4a4923bc9e73c5b3854d2cfe7305f
Parents: 3a363c5
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Tue Apr 5 17:34:58 2016 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sat Apr 9 01:25:07 2016 -0700
----------------------------------------------------------------------
.../apache/ranger/plugin/store/TagStore.java | 4 ++
.../ranger/plugin/store/file/TagFileStore.java | 47 ++++++++++++++++++
.../java/org/apache/ranger/biz/TagDBStore.java | 40 +++++++++++++++
.../apache/ranger/db/XXServiceResourceDao.java | 12 +++++
.../ranger/rest/ServiceTagsProcessor.java | 51 ++++++++++++++++++--
.../resources/META-INF/jpa_named_queries.xml | 4 ++
.../source/atlas/AtlasNotificationMapper.java | 6 +++
7 files changed, 161 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
index b135423..3c5a43b 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
@@ -85,12 +85,16 @@ public interface TagStore {
void deleteServiceResource(Long id) throws Exception;
+ void deleteServiceResourceByGuid(String guid) throws Exception;
+
RangerServiceResource getServiceResource(Long id) throws Exception;
RangerServiceResource getServiceResourceByGuid(String guid) throws Exception;
List<RangerServiceResource> getServiceResourcesByService(String serviceName) throws Exception;
+ List<String> getServiceResourceGuidsByService(String serviceName) throws Exception;
+
RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception;
List<RangerServiceResource> getServiceResources(SearchFilter filter) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
index 5f22f0d..cc983a6 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
@@ -665,6 +665,26 @@ public class TagFileStore extends AbstractTagStore {
}
@Override
+ public void deleteServiceResourceByGuid(String guid) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagFileStore.deleteServiceResourceByGuid(" + guid + ")");
+ }
+
+ try {
+ RangerServiceResource resource = getServiceResourceByGuid(guid);
+
+ deleteServiceResource(resource);
+ } catch (Exception excp) {
+ throw new Exception("failed to delete service-resource with GUID=" + guid, excp);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagFileStore.deleteServiceResourceByGuid(" + guid + ")");
+ }
+
+ }
+
+ @Override
public RangerServiceResource getServiceResource(Long id) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagFileStore.getServiceResource(" + id + ")");
@@ -732,6 +752,33 @@ public class TagFileStore extends AbstractTagStore {
}
@Override
+ public List<String> getServiceResourceGuidsByService(String serviceName) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagFileStore.getServiceResourceGuidsByService(" + serviceName + ")");
+ }
+
+ List<String> ret = null;
+
+ if (StringUtils.isNotBlank(serviceName)) {
+ List<RangerServiceResource> serviceResources = this.getServiceResourcesByService(serviceName);
+
+ if(CollectionUtils.isNotEmpty(serviceResources)) {
+ ret = new ArrayList<String>(serviceResources.size());
+
+ for(RangerServiceResource serviceResource : serviceResources) {
+ ret.add(serviceResource.getGuid());
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagFileStore.getServiceResourceGuidsByService(" + serviceName + "): count=" + (ret == null ? 0 : ret.size()));
+ }
+
+ return ret;
+ }
+
+ @Override
public RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagFileStore.getServiceResourceByServiceAndResourceSignature(" + serviceName + ", " + resourceSignature + ")");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 28d7bf6..0ec37f1 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -88,6 +88,7 @@ public class TagDBStore extends AbstractTagStore {
@Autowired
GUIDUtil guidUtil;
+
@Override
public RangerTagDef createTagDef(RangerTagDef tagDef) throws Exception {
if (LOG.isDebugEnabled()) {
@@ -531,6 +532,24 @@ public class TagDBStore extends AbstractTagStore {
}
@Override
+ public void deleteServiceResourceByGuid(String guid) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagDBStore.deleteServiceResourceByGuid(" + guid + ")");
+ }
+
+ RangerServiceResource resource = getServiceResourceByGuid(guid);
+
+ if(resource != null) {
+ deleteResourceForServiceResource(resource.getId());
+ rangerServiceResourceService.delete(resource);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagDBStore.deleteServiceResourceByGuid(" + guid + ")");
+ }
+ }
+
+ @Override
public RangerServiceResource getServiceResource(Long id) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagDBStore.getServiceResource(" + id + ")");
@@ -582,6 +601,27 @@ public class TagDBStore extends AbstractTagStore {
}
@Override
+ public List<String> getServiceResourceGuidsByService(String serviceName) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagDBStore.getServiceResourceGuidsByService(" + serviceName + ")");
+ }
+
+ List<String> ret = null;
+
+ XXService service = daoManager.getXXService().findByName(serviceName);
+
+ if (service != null) {
+ ret = daoManager.getXXServiceResource().findServiceResourceGuidsInServiceId(service.getId());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagDBStore.getServiceResourceGuidsByService(" + serviceName + "): count=" + (ret == null ? 0 : ret.size()));
+ }
+
+ return ret;
+ }
+
+ @Override
public RangerServiceResource getServiceResourceByServiceAndResourceSignature(String serviceName, String resourceSignature) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagDBStore.getServiceResourceByServiceAndResourceSignature(" + serviceName + ", " + resourceSignature + ")");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java
index 9257aaa..0907e2f 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXServiceResourceDao.java
@@ -83,4 +83,16 @@ public class XXServiceResourceDao extends BaseDao<XXServiceResource> {
return new ArrayList<XXServiceResource>();
}
}
+
+ public List<String> findServiceResourceGuidsInServiceId(Long serviceId) {
+ if (serviceId == null) {
+ return new ArrayList<String>();
+ }
+ try {
+ return getEntityManager().createNamedQuery("XXServiceResource.findServiceResourceGuidsInServiceId", String.class)
+ .setParameter("serviceId", serviceId).getResultList();
+ } catch (NoResultException e) {
+ return new ArrayList<String>();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
index 7e6900e..cf07deb 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -165,9 +165,10 @@ public class ServiceTagsProcessor {
RangerServiceResource resourceInStore = null;
if (existing == null) {
- resource.setResourceSignature(resourceSignature);
+ resource.setResourceSignature(resourceSignature);
resourceInStore = tagStore.createServiceResource(resource);
+
} else if (StringUtils.isEmpty(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) {
resourceInStore = existing;
} else {
@@ -499,16 +500,60 @@ public class ServiceTagsProcessor {
}
}
- // Delete all tagdef, tag, serviceResource and tagResourceMaps and then add all objects in provided ServiceTagsids
private void replace(ServiceTags serviceTags) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ServiceTagsProcessor.replace()");
}
- tagStore.deleteAllTagObjectsForService(serviceTags.getServiceName());
+ // Delete those service-resources which are in ranger database but not in provided service-tags
+
+ Map<String, RangerServiceResource> serviceResourcesInServiceTagsMap = new HashMap<String, RangerServiceResource>();
+
+ List<RangerServiceResource> serviceResourcesInServiceTags = serviceTags.getServiceResources();
+
+ for (RangerServiceResource rangerServiceResource : serviceResourcesInServiceTags) {
+ String guid = rangerServiceResource.getGuid();
+
+ if(serviceResourcesInServiceTagsMap.containsKey(guid)) {
+ LOG.warn("duplicate service-resource found: guid=" + guid);
+ }
+
+ serviceResourcesInServiceTagsMap.put(guid, rangerServiceResource);
+ }
+
+ List<String> serviceResourcesInDb = tagStore.getServiceResourceGuidsByService(serviceTags.getServiceName());
+
+ for (String dbServiceResourceGuid : serviceResourcesInDb) {
+
+ if (! serviceResourcesInServiceTagsMap.containsKey(dbServiceResourceGuid)) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleting serviceResource(guid=" + dbServiceResourceGuid + ") and its tag-associations...");
+ }
+
+ List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(dbServiceResourceGuid);
+
+ if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
+ for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
+ tagStore.deleteTagResourceMap(tagResourceMap.getId());
+ }
+ }
+
+ tagStore.deleteServiceResourceByGuid(dbServiceResourceGuid);
+ }
+
+ }
+
+ // Add/update resources and other tag-model objects provided in service-tags
addOrUpdate(serviceTags);
+ // All private tags at this point are associated with some service-resource and shared
+ // tags cannot be deleted as they belong to some other service. In any case, any tags that
+ // are not associated with service-resource will not be downloaded to plugin.
+
+ // Tag-defs cannot be deleted as there may be a shared tag that it refers to it.
+
if (LOG.isDebugEnabled()) {
LOG.debug("<== ServiceTagsProcessor.replace()");
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index c70dcba..469a400 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -967,6 +967,10 @@
<query>select obj from XXServiceResource obj where obj.serviceId = :serviceId and obj.resourceSignature = :resourceSignature</query>
</named-query>
+ <named-query name="XXServiceResource.findServiceResourceGuidsInServiceId">
+ <query>select obj.guid from XXServiceResource obj where obj.serviceId = :serviceId</query>
+ </named-query>
+
<!-- End <== JPA Queries for Tag Based Policies -->
<named-query name="XXTrxLog.getMaxIdOfXXTrxLog">
<query>select max(obj.id) from XXTrxLog obj</query>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/2867cc55/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
index a9316b5..7dc487c 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasNotificationMapper.java
@@ -146,6 +146,12 @@ public class AtlasNotificationMapper {
}
}
+ if (MapUtils.isNotEmpty(ret)) {
+ for (Map.Entry<String, ServiceTags> entry : ret.entrySet()) {
+ ServiceTags serviceTags = entry.getValue();
+ serviceTags.setOp(ServiceTags.OP_REPLACE);
+ }
+ }
return ret;
}