You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/10/28 17:00:00 UTC
[24/50] [abbrv] incubator-ranger git commit: RANGER-660: implement
TagSync process to populate Ranger with tag details
RANGER-660: implement TagSync process to populate Ranger with tag details
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/eb874038
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/eb874038
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/eb874038
Branch: refs/heads/master
Commit: eb8740380fc53a353ddfc239455c5e9254476056
Parents: 9c42a8a
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Thu Sep 10 17:49:16 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Sep 28 09:57:15 2015 -0700
----------------------------------------------------------------------
.../plugin/model/RangerTagResourceMap.java | 4 +-
.../apache/ranger/plugin/store/TagStore.java | 2 +
.../ranger/plugin/store/TagValidator.java | 169 ++++--
.../ranger/plugin/store/file/TagFileStore.java | 26 +-
.../apache/ranger/plugin/util/ServiceTags.java | 18 +
.../atlas-client-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 34445 bytes
...las-notification-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 34596 bytes
...atlas-typesystem-0.6-incubating-SNAPSHOT.jar | Bin 0 -> 355378 bytes
pom.xml | 7 +
.../java/org/apache/ranger/biz/TagDBStore.java | 16 +
.../apache/ranger/db/XXTagResourceMapDao.java | 9 +
.../ranger/rest/ServiceTagsProcessor.java | 370 ++++++++++++
.../java/org/apache/ranger/rest/TagREST.java | 117 +++-
.../apache/ranger/rest/TagRESTConstants.java | 1 +
.../service/RangerTagResourceMapService.java | 6 +
.../resources/META-INF/jpa_named_queries.xml | 6 +
src/main/assembly/tagsync.xml | 48 +-
tagsync/conf.dist/log4j.xml | 4 +-
.../conf.dist/ranger-tagsync-default-site.xml | 36 --
tagsync/conf.dist/ranger-tagsync-site.xml | 67 +++
tagsync/pom.xml | 8 +-
tagsync/samples/tags.json | 55 ++
tagsync/scripts/ranger-tagsync-services.sh | 22 +-
.../java/org/apache/ranger/model/TagSink.java | 3 -
.../java/org/apache/ranger/model/TagSource.java | 13 +-
.../apache/ranger/process/TagSyncConfig.java | 132 ++---
.../apache/ranger/process/TagSynchronizer.java | 60 +-
.../ranger/sink/policymgr/TagRESTSink.java | 14 +-
.../source/atlas/AtlasNotificationMapper.java | 391 ++++++++++++
.../ranger/source/atlas/TagAtlasSource.java | 588 +++++++++++++++++++
.../ranger/source/file/TagFileSource.java | 183 +-----
.../main/resources/ranger-tagsync-default.xml | 91 +++
.../src/main/resources/ranger-tagsync-site.xml | 67 ---
.../ranger/process/TestTagSynchronizer.java | 17 +-
34 files changed, 2024 insertions(+), 526 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
index 3b69ee7..3ee0b3e 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/model/RangerTagResourceMap.java
@@ -71,9 +71,9 @@ public class RangerTagResourceMap extends RangerBaseModelObject {
sb.append("{ ");
- sb.append("tagId=").append(tagId).append("} ");
+ sb.append("resourceId=").append(resourceId).append(", ");
- sb.append("resourceId={").append(resourceId).append("} ");
+ sb.append("tagId=").append(tagId);
sb.append(" }");
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
index 1c48ea0..ed20c51 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagStore.java
@@ -66,6 +66,8 @@ public interface TagStore {
RangerTag getTagByGuid(String guid) throws Exception;
+ List<Long> getTagIdsForResourceId(Long resourceId) throws Exception;
+
List<RangerTag> getTagsByType(String name) throws Exception;
List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
index 4eda166..c334e39 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/TagValidator.java
@@ -35,10 +35,31 @@ public class TagValidator {
this.tagStore = tagStore;
}
- public void preCreateTag(final RangerTag tag) throws Exception {
- if (StringUtils.isBlank(tag.getType())) {
+ public RangerTagDef preCreateTagDef(final RangerTagDef tagDef, boolean updateIfExists) throws Exception {
+ String name = tagDef.getName();
+
+ if (StringUtils.isBlank(name)) {
+ throw new Exception("TagDef has no name");
+ }
+
+ RangerTagDef existing = tagStore.getTagDefByName(name);
+
+ return existing;
+ }
+
+ public RangerTag preCreateTag(final RangerTag tag) throws Exception {
+ if(StringUtils.isBlank(tag.getType()) ) {
throw new Exception("Tag has no type");
}
+
+ RangerTag ret = null;
+
+ String guid = tag.getGuid();
+ if (! StringUtils.isBlank(guid)) {
+ ret = tagStore.getTagByGuid(guid);
+ }
+
+ return ret;
}
public void preUpdateTag(final Long id, final RangerTag tag) throws Exception {
@@ -50,12 +71,14 @@ public class TagValidator {
throw new Exception("Invalid/null id");
}
- RangerTag exist = tagStore.getTag(id);
+ RangerTag existing = tagStore.getTag(id);
- if (exist == null) {
+ if (existing == null) {
throw new Exception("Attempt to update nonexistant tag, id=" + id);
}
- tag.setId(exist.getId());
+
+ tag.setId(existing.getId());
+ tag.setGuid(existing.getGuid());
}
public void preUpdateTagByGuid(String guid, final RangerTag tag) throws Exception {
@@ -72,39 +95,27 @@ public class TagValidator {
tag.setGuid(existing.getGuid());
}
- public void preUpdateTagByName(String name, final RangerTag tag) throws Exception {
- if (StringUtils.isNotBlank(tag.getType())) {
- throw new Exception("tag has no type");
- }
-
- List<RangerTag> exist = tagStore.getTagsByType(name);
- if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) {
- throw new Exception("Attempt to update nonexistent or multiple tags, name=" + name);
+ public RangerTag preDeleteTag(Long id) throws Exception {
+ if (id == null) {
+ throw new Exception("Invalid/null id");
}
- RangerTag onlyTag = exist.get(0);
+ RangerTag existing = tagStore.getTag(id);
- tag.setId(onlyTag.getId());
- tag.setType(name);
-
- }
-
- public RangerTag preDeleteTag(Long id) throws Exception {
- RangerTag exist;
- exist = tagStore.getTag(id);
- if (exist == null) {
+ if (existing == null) {
throw new Exception("Attempt to delete nonexistent tag, id=" + id);
}
- List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(exist.getId());
+ List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(existing.getId());
if (CollectionUtils.isNotEmpty(associations)) {
throw new Exception("Attempt to delete tag which is associated with a service-resource, id=" + id);
}
- return exist;
+ return existing;
}
public RangerTag preDeleteTagByGuid(String guid) throws Exception {
RangerTag exiting = tagStore.getTagByGuid(guid);
+
if (exiting == null) {
throw new Exception("Attempt to delete nonexistent tag, guid=" + guid);
}
@@ -116,37 +127,23 @@ public class TagValidator {
return exiting;
}
- public RangerTag preDeleteTagByName(String name) throws Exception {
- List<RangerTag> exist;
- exist = tagStore.getTagsByType(name);
- if (CollectionUtils.isEmpty(exist) || CollectionUtils.size(exist) != 1) {
- throw new Exception("Attempt to delete nonexistent or multiple tags, name=" + name);
- }
- RangerTag ret = exist.get(0);
- List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForTagId(ret.getId());
- if (CollectionUtils.isNotEmpty(associations)) {
- throw new Exception("Attempt to delete tag which is associated with a service-resource, name=" + name);
- }
- return ret;
-
- }
+ public RangerServiceResource preCreateServiceResource(RangerServiceResource resource) throws Exception {
+ RangerServiceResource ret = null;
- public void preCreateServiceResource(RangerServiceResource resource) throws Exception {
if (StringUtils.isBlank(resource.getServiceName()) || MapUtils.isEmpty(resource.getResourceElements())) {
throw new Exception("No serviceName or resource in RangerServiceResource");
}
- RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
-
- String resourceSignature = serializer.getSignature();
+ String guid = resource.getGuid();
+ if (! StringUtils.isBlank(guid)) {
+ ret = tagStore.getServiceResourceByGuid(guid);
+ }
- RangerServiceResource exist = tagStore.getServiceResourceByResourceSignature(resourceSignature);
+ RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
- if (exist != null) {
- throw new Exception("Attempt to create existing resource, serviceName=" + resource.getServiceName());
- }
+ resource.setResourceSignature(serializer.getSignature());
- resource.setResourceSignature(resourceSignature);
+ return ret;
}
public void preUpdateServiceResource(Long id, RangerServiceResource resource) throws Exception {
@@ -158,15 +155,16 @@ public class TagValidator {
throw new Exception("Invalid/null id");
}
- RangerServiceResource exist = tagStore.getServiceResource(id);
- if (exist == null) {
+ RangerServiceResource existing = tagStore.getServiceResource(id);
+ if (existing == null) {
throw new Exception("Attempt to update nonexistent resource, id=" + id);
}
- resource.setId(exist.getId());
RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
- resource.setResourceSignature(serializer.getSignature());
+ resource.setId(existing.getId());
+ resource.setGuid(existing.getGuid());
+ resource.setResourceSignature(serializer.getSignature());
}
public void preUpdateServiceResourceByGuid(String guid, RangerServiceResource resource) throws Exception {
@@ -179,35 +177,40 @@ public class TagValidator {
throw new Exception("Attempt to update nonexistent resource, guid=" + guid);
}
+ RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
resource.setId(existing.getId());
resource.setGuid(guid);
-
- RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
resource.setResourceSignature(serializer.getSignature());
}
public RangerServiceResource preDeleteServiceResource(Long id) throws Exception {
- RangerServiceResource exist;
- exist = tagStore.getServiceResource(id);
- if (exist == null) {
+ RangerServiceResource existing = tagStore.getServiceResource(id);
+
+ if (existing == null) {
throw new Exception("Attempt to delete nonexistent resource, id=" + id);
}
- List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(exist.getId());
+
+ List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId());
if (CollectionUtils.isNotEmpty(associations)) {
throw new Exception("Attempt to delete serviceResource which is associated with a tag, id=" + id);
}
- return exist;
+
+ return existing;
}
- public RangerServiceResource preDeleteServiceResourceByGuid(String guid) throws Exception {
+ public RangerServiceResource preDeleteServiceResourceByGuid(String guid, boolean deleteReferences) throws Exception {
RangerServiceResource existing = tagStore.getServiceResourceByGuid(guid);
+
if (existing == null) {
throw new Exception("Attempt to delete nonexistent resource, guid=" + guid);
}
+
List<RangerTagResourceMap> associations = tagStore.getTagResourceMapsForResourceId(existing.getId());
- if (CollectionUtils.isNotEmpty(associations)) {
+ if (CollectionUtils.isNotEmpty(associations) && !deleteReferences) {
throw new Exception("Attempt to delete serviceResource which is associated with a tag, guid=" + guid);
}
+
return existing;
}
@@ -216,8 +219,9 @@ public class TagValidator {
throw new Exception("Both resourceGuid and resourceId need to be non-empty");
}
- RangerTagResourceMap exist = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
- if (exist != null) {
+ RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+
+ if (existing != null) {
throw new Exception("Attempt to create existing association between resourceId=" + resourceGuid + " and tagId=" + tagGuid);
}
@@ -240,8 +244,35 @@ public class TagValidator {
return newTagResourceMap;
}
+ public RangerTagResourceMap preCreateTagResourceMapByIds(Long tagId, Long resourceId) throws Exception {
+ RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId);
+
+ if (existing != null) {
+ throw new Exception("Attempt to create existing association between resourceId=" + resourceId + " and tagId=" + tagId);
+ }
+
+ RangerServiceResource existingServiceResource = tagStore.getServiceResource(resourceId);
+
+ if(existingServiceResource == null) {
+ throw new Exception("No resource found for id=" + resourceId);
+ }
+
+ RangerTag existingTag = tagStore.getTag(tagId);
+
+ if(existingTag == null) {
+ throw new Exception("No tag found for id=" + tagId);
+ }
+
+ RangerTagResourceMap newTagResourceMap = new RangerTagResourceMap();
+ newTagResourceMap.setResourceId(resourceId);
+ newTagResourceMap.setTagId(tagId);
+
+ return newTagResourceMap;
+ }
+
public RangerTagResourceMap preDeleteTagResourceMap(Long id) throws Exception {
RangerTagResourceMap existing = tagStore.getTagResourceMap(id);
+
if (existing == null) {
throw new Exception("Attempt to delete nonexistent tagResourceMap(id=" + id + ")");
}
@@ -251,6 +282,7 @@ public class TagValidator {
public RangerTagResourceMap preDeleteTagResourceMapByGuid(String guid) throws Exception {
RangerTagResourceMap existing = tagStore.getTagResourceMapByGuid(guid);
+
if (existing == null) {
throw new Exception("Attempt to delete nonexistent tagResourceMap(guid=" + guid + ")");
}
@@ -260,10 +292,21 @@ public class TagValidator {
public RangerTagResourceMap preDeleteTagResourceMap(String tagGuid, String resourceGuid) throws Exception {
RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+
if (existing == null) {
throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceGuid + " and tagId=" + tagGuid);
}
return existing;
}
+
+ public RangerTagResourceMap preDeleteTagResourceMapByIds(Long tagId, Long resourceId) throws Exception {
+ RangerTagResourceMap existing = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceId);
+
+ if (existing == null) {
+ throw new Exception("Attempt to delete nonexistent association between resourceId=" + resourceId + " and tagId=" + tagId);
+ }
+
+ return existing;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
index 27b4779..8f94164 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
@@ -31,6 +31,7 @@ import org.apache.ranger.authorization.hadoop.config.RangerConfiguration;
import org.apache.ranger.plugin.model.*;
import org.apache.ranger.plugin.store.AbstractTagStore;
import org.apache.ranger.plugin.store.PList;
+import org.apache.ranger.plugin.store.RangerServiceResourceSignature;
import org.apache.ranger.plugin.store.TagPredicateUtil;
import org.apache.ranger.plugin.store.TagStore;
import org.apache.ranger.plugin.util.SearchFilter;
@@ -42,6 +43,7 @@ public class TagFileStore extends AbstractTagStore {
public static final String PROPERTY_TAG_FILE_STORE_DIR = "ranger.tag.store.file.dir";
+
protected static final String FILE_PREFIX_TAG_DEF = "ranger-tagdef-";
protected static final String FILE_PREFIX_TAG = "ranger-tag-";
protected static final String FILE_PREFIX_RESOURCE = "ranger-serviceresource-";
@@ -116,12 +118,6 @@ public class TagFileStore extends AbstractTagStore {
LOG.debug("==> TagFileStore.createTagDef(" + tagDef + ")");
}
- RangerTagDef existing = getTagDefByName(tagDef.getName());
-
- if (existing != null) {
- throw new Exception(tagDef.getName() + ": tag-def already exists (id=" + existing.getId() + ")");
- }
-
RangerTagDef ret = null;
try {
@@ -160,7 +156,7 @@ public class TagFileStore extends AbstractTagStore {
throw new Exception("tag-def does not exist: name=" + tagDef.getName());
}
} else {
- existing = this.getTagDef(tagDef.getId());
+ existing = getTagDef(tagDef.getId());
if (existing == null) {
throw new Exception("tag-def does not exist: id=" + tagDef.getId());
@@ -504,6 +500,21 @@ public class TagFileStore extends AbstractTagStore {
}
@Override
+ public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+ List<Long> ret = new ArrayList<Long>();
+
+ List<RangerTag> tags = getTagsForResourceId(resourceId);
+
+ if(CollectionUtils.isNotEmpty(tags)) {
+ for(RangerTag tag : tags) {
+ ret.add(tag.getId());
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagFileStore.getTagsForResourceId(" + resourceId + ")");
@@ -1063,6 +1074,7 @@ public class TagFileStore extends AbstractTagStore {
}
ServiceTags ret = new ServiceTags();
+ ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
boolean tagsChanged = true;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
index e56e942..146c3f9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/ServiceTags.java
@@ -44,6 +44,10 @@ import org.codehaus.jackson.map.annotate.JsonSerialize;
public class ServiceTags implements java.io.Serializable {
private static final long serialVersionUID = 1L;
+ public static final String OP_ADD_OR_UPDATE = "add_or_update";
+ public static final String OP_DELETE = "delete";
+
+ private String op;
private String serviceName;
private Long tagVersion;
private Date tagUpdateTime;
@@ -53,6 +57,13 @@ public class ServiceTags implements java.io.Serializable {
private Map<Long, List<Long>> resourceToTagIds;
/**
+ * @return the op
+ */
+ public String getOp() {
+ return op;
+ }
+
+ /**
* @return the serviceName
*/
public String getServiceName() {
@@ -60,6 +71,13 @@ public class ServiceTags implements java.io.Serializable {
}
/**
+ * @param op the op to set
+ */
+ public void setOp(String op) {
+ this.op = op;
+ }
+
+ /**
* @param serviceName the serviceName to set
*/
public void setServiceName(String serviceName) {
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..feaaa38
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..3c2ecb6
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar
----------------------------------------------------------------------
diff --git a/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar
new file mode 100644
index 0000000..8e5d1bf
Binary files /dev/null and b/lib/local/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar differ
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index aa40882..002885a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -255,6 +255,13 @@
<enabled>false</enabled>
</snapshots>
</repository>
+ <repository>
+ <id>repo</id>
+ <url>file://${user.home}/local-repo</url>
+ <snapshots>
+ <enabled>true</enabled>
+ </snapshots>
+ </repository>
</repositories>
<dependencyManagement>
<dependencies>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
index 962bfde..98a6e87 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/TagDBStore.java
@@ -719,6 +719,21 @@ public class TagDBStore extends AbstractTagStore {
}
@Override
+ public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> TagDBStore.getTagIdsForResourceId(" + resourceId + ")");
+ }
+
+ List<Long> ret = rangerTagResourceMapService.getTagIdsForResourceId(resourceId);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== TagDBStore.getTagIdsForResourceId(" + resourceId + "): count=" + (ret == null ? 0 : ret.size()));
+ }
+
+ return ret;
+ }
+
+ @Override
public List<RangerTagResourceMap> getTagResourceMapsForResourceId(Long resourceId) throws Exception {
if (LOG.isDebugEnabled()) {
LOG.debug("==> TagDBStore.getTagResourceMapsForResourceId(" + resourceId + ")");
@@ -875,6 +890,7 @@ public class TagDBStore extends AbstractTagStore {
}
ret = new ServiceTags();
+ ret.setOp(ServiceTags.OP_ADD_OR_UPDATE);
ret.setServiceName(xxService.getName());
ret.setTagVersion(xxService.getTagVersion());
ret.setTagUpdateTime(xxService.getTagUpdateTime());
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
index 0cac1ef..72e0cd9 100644
--- a/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
+++ b/security-admin/src/main/java/org/apache/ranger/db/XXTagResourceMapDao.java
@@ -70,6 +70,15 @@ public class XXTagResourceMapDao extends BaseDao<XXTagResourceMap> {
}
}
+ public List<Long> findTagIdsForResourceId(Long resourceId) {
+ try {
+ return getEntityManager().createNamedQuery("XXTagResourceMap.getTagIdsForResourceId", Long.class)
+ .setParameter("resourceId", resourceId).getResultList();
+ } catch (NoResultException e) {
+ return new ArrayList<Long>();
+ }
+ }
+
public List<XXTagResourceMap> findByTagId(Long tagId) {
if (tagId == null) {
return new ArrayList<XXTagResourceMap>();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
new file mode 100644
index 0000000..7f0a92c
--- /dev/null
+++ b/security-admin/src/main/java/org/apache/ranger/rest/ServiceTagsProcessor.java
@@ -0,0 +1,370 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.rest;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.collections.MapUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.plugin.model.RangerTag;
+import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.model.RangerTagResourceMap;
+import org.apache.ranger.plugin.store.RangerServiceResourceSignature;
+import org.apache.ranger.plugin.store.TagStore;
+import org.apache.ranger.plugin.util.ServiceTags;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ServiceTagsProcessor {
+ private static final Log LOG = LogFactory.getLog(ServiceTagsProcessor.class);
+
+ private final TagStore tagStore;
+
+ public ServiceTagsProcessor(TagStore tagStore) {
+ this.tagStore = tagStore;
+ }
+
+ public void process(ServiceTags serviceTags) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceTagsProcessor.process()");
+ }
+
+ if (tagStore != null && serviceTags != null) {
+ String op = serviceTags.getOp();
+
+ if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_ADD_OR_UPDATE)) {
+ addOrUpdate(serviceTags);
+ } else if (StringUtils.equalsIgnoreCase(op, ServiceTags.OP_DELETE)) {
+ delete(serviceTags);
+ } else {
+ LOG.error("Unknown op, op=" + op);
+ }
+ } else {
+ if(tagStore == null) {
+ LOG.error("tagStore is null!!");
+ }
+
+ if (serviceTags == null) {
+ LOG.error("No ServiceTags to import!!");
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceTagsProcessor.process()");
+ }
+ }
+
+ // Map tagdef, tag, serviceResource ids to created ids and use them in tag-resource-mapping
+ private void addOrUpdate(ServiceTags serviceTags) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceTagsProcessor.createOrUpdate()");
+ }
+
+ Map<Long, RangerTagDef> tagDefsInStore = new HashMap<Long, RangerTagDef>();
+ Map<Long, RangerTag> tagsInStore = new HashMap<Long, RangerTag>();
+ Map<Long, RangerServiceResource> resourcesInStore = new HashMap<Long, RangerServiceResource>();
+
+ boolean createOrUpdate = true;
+
+ if (MapUtils.isNotEmpty(serviceTags.getTagDefinitions())) {
+ RangerTagDef tagDef = null;
+
+ try {
+ for (Map.Entry<Long, RangerTagDef> entry : serviceTags.getTagDefinitions().entrySet()) {
+ tagDef = entry.getValue();
+
+ RangerTagDef existing = null;
+
+ if(createOrUpdate) {
+ if(StringUtils.isNotEmpty(tagDef.getGuid())) {
+ existing = tagStore.getTagDefByGuid(tagDef.getGuid());
+ }
+
+ if(existing == null && StringUtils.isNotEmpty(tagDef.getName())) {
+ existing = tagStore.getTagDefByName(tagDef.getName());
+ }
+ }
+
+ RangerTagDef tagDefInStore = null;
+
+ if(existing == null) {
+ tagDefInStore = tagStore.createTagDef(tagDef);
+ } else {
+ tagDef.setId(existing.getId());
+ tagDef.setGuid(existing.getGuid());
+
+ tagDefInStore = tagStore.updateTagDef(tagDef);
+ }
+
+ if(tagDefsInStore != null) {
+ tagDefsInStore.put(entry.getKey(), tagDefInStore);
+ }
+ }
+ } catch (Exception exception) {
+ LOG.error("createTagDef failed, tagDef=" + tagDef, exception);
+ throw exception;
+ }
+ }
+
+ if (MapUtils.isNotEmpty(serviceTags.getTags())) {
+ RangerTag tag = null;
+
+ try {
+ for (Map.Entry<Long, RangerTag> entry : serviceTags.getTags().entrySet()) {
+ tag = entry.getValue();
+
+ RangerTag existing = null;
+
+ if(createOrUpdate) {
+ if(StringUtils.isNotEmpty(tag.getGuid())) {
+ existing = tagStore.getTagByGuid(tag.getGuid());
+ }
+ }
+
+ RangerTag tagInStore = null;
+
+ if(existing == null) {
+ tagInStore = tagStore.createTag(tag);
+ } else {
+ tag.setId(existing.getId());
+ tag.setGuid(existing.getGuid());
+
+ tagInStore = tagStore.updateTag(tag);
+ }
+
+ if(tagsInStore != null) {
+ tagsInStore.put(entry.getKey(), tagInStore);
+ }
+ }
+ } catch (Exception exception) {
+ LOG.error("createTag failed, tag=" + tag, exception);
+ throw exception;
+ }
+ }
+
+ List<RangerServiceResource> resources = serviceTags.getServiceResources();
+ if (CollectionUtils.isNotEmpty(resources)) {
+ RangerServiceResource resource = null;
+
+ try {
+ for (int i = 0; i < resources.size(); i++) {
+ resource = resources.get(i);
+
+ RangerServiceResource existing = null;
+ String resourceSignature = null;
+ Long resourceId = resource.getId();
+
+ if(createOrUpdate) {
+ if(StringUtils.isNotEmpty(resource.getGuid())) {
+ existing = tagStore.getServiceResourceByGuid(resource.getGuid());
+ }
+
+ if(existing == null) {
+ if(MapUtils.isNotEmpty(resource.getResourceElements())) {
+ RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
+ resourceSignature = serializer.getSignature();
+
+ existing = tagStore.getServiceResourceByResourceSignature(resourceSignature);
+ }
+ }
+
+ if(existing != null) {
+ resourceSignature = existing.getResourceSignature();
+ }
+ }
+
+ if(StringUtils.isEmpty(resourceSignature)) {
+ RangerServiceResourceSignature serializer = new RangerServiceResourceSignature(resource);
+
+ resourceSignature = serializer.getSignature();
+ }
+
+ RangerServiceResource resourceInStore = null;
+
+ if (existing == null) {
+ resource.setResourceSignature(resourceSignature);
+
+ resourceInStore = tagStore.createServiceResource(resource);
+ } else {
+ resource.setId(existing.getId());
+ resource.setGuid(existing.getGuid());
+ resource.setResourceSignature(resourceSignature);
+
+ resourceInStore = tagStore.updateServiceResource(resource);
+ }
+
+ resourcesInStore.put(resourceId, resourceInStore);
+ }
+ } catch (Exception exception) {
+ LOG.error("createServiceResource failed, resource=" + resource, exception);
+ throw exception;
+ }
+ }
+
+ if (MapUtils.isNotEmpty(serviceTags.getResourceToTagIds())) {
+ for (Map.Entry<Long, List<Long>> entry : serviceTags.getResourceToTagIds().entrySet()) {
+ Long resourceId = entry.getKey();
+
+ RangerServiceResource resourceInStore = resourcesInStore.get(resourceId);
+
+ if (resourceInStore == null) {
+ LOG.error("Resource (id=" + resourceId + ") not found. Skipping tags update");
+ continue;
+ }
+
+ // Get all tags associated with this resourceId
+ List<Long> tagMapsToDelete = null;
+ try {
+ tagMapsToDelete = tagStore.getTagIdsForResourceId(resourceInStore.getId());
+ } catch (Exception exception) {
+ LOG.error("RangerTags cannot be retrieved for resource with guid=" + resourceInStore.getGuid());
+ throw exception;
+ }
+
+ List<Long> tagIds = entry.getValue();
+ try {
+ for (Long tagId : tagIds) {
+ RangerTag tagInStore = tagsInStore.get(tagId);
+
+ if (tagInStore == null) {
+ LOG.error("Tag (id=" + tagId + ") not found. Skipping addition of this tag for resource (id=" + resourceId + ")");
+ continue;
+ }
+
+ RangerTagResourceMap existing = null;
+
+ if(createOrUpdate) {
+ existing = tagStore.getTagResourceMapForTagAndResourceId(tagInStore.getId(), resourceInStore.getId());
+ }
+
+ if(existing == null) {
+ RangerTagResourceMap tagResourceMap = new RangerTagResourceMap();
+
+ tagResourceMap.setTagId(tagInStore.getId());
+ tagResourceMap.setResourceId(resourceInStore.getId());
+
+ tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+ }
+
+ if(tagMapsToDelete != null) {
+ tagMapsToDelete.remove((Long)tagInStore.getId());
+ }
+ }
+ } catch (Exception exception) {
+ LOG.error("createRangerTagResourceMap failed", exception);
+ throw exception;
+ }
+
+ if (CollectionUtils.isNotEmpty(tagMapsToDelete)) {
+ Long tagId = null;
+
+ try {
+ for(int i = 0; i < tagMapsToDelete.size(); i++) {
+ tagId = tagMapsToDelete.get(i);
+
+ RangerTagResourceMap tagResourceMap = tagStore.getTagResourceMapForTagAndResourceId(tagId, resourceInStore.getId());
+
+ if(tagResourceMap != null) {
+ tagStore.deleteTagResourceMap(tagResourceMap.getId());
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Deleted tagResourceMap(tagId=" + tagId + ", resourceId=" + resourceId);
+ }
+ }
+ } catch(Exception exception) {
+ LOG.error("deleteTagResourceMap failed, tagId=" + tagId + ", resourceId=" + resourceId);
+ throw exception;
+ }
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceTagsProcessor.createOrUpdate()");
+ }
+ }
+
+ private void delete(ServiceTags serviceTags) throws Exception {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> ServiceTagsProcessor.delete()");
+ }
+
+ List<RangerServiceResource> serviceResources = serviceTags.getServiceResources();
+ if (CollectionUtils.isNotEmpty(serviceResources)) {
+ for (RangerServiceResource serviceResource : serviceResources) {
+ try {
+ RangerServiceResource objToDelete = tagStore.getServiceResourceByGuid(serviceResource.getGuid());
+
+ if(objToDelete != null) {
+ tagStore.deleteServiceResource(objToDelete.getId());
+ }
+ } catch (Exception exception) {
+ LOG.error("deleteServiceResourceByGuid failed, guid=" + serviceResource.getGuid(), exception);
+ throw exception;
+ }
+ }
+ }
+
+ Map<Long, RangerTag> tagsMap = serviceTags.getTags();
+ if (MapUtils.isNotEmpty(tagsMap)) {
+ for (Map.Entry<Long, RangerTag> entry : tagsMap.entrySet()) {
+ RangerTag tag = entry.getValue();
+ try {
+ RangerTag objToDelete = tagStore.getTagByGuid(tag.getGuid());
+
+ if(objToDelete != null) {
+ tagStore.deleteTag(objToDelete.getId());
+ }
+ } catch (Exception exception) {
+ LOG.error("deleteTag failed, guid=" + tag.getGuid(), exception);
+ throw exception;
+ }
+ }
+ }
+
+ Map<Long, RangerTagDef> tagDefsMap = serviceTags.getTagDefinitions();
+ if (MapUtils.isNotEmpty(tagDefsMap)) {
+ for (Map.Entry<Long, RangerTagDef> entry : tagDefsMap.entrySet()) {
+ RangerTagDef tagDef = entry.getValue();
+ try {
+ RangerTagDef objToDelete = tagStore.getTagDefByGuid(tagDef.getGuid());
+
+ if(objToDelete != null) {
+ tagStore.deleteTagDef(objToDelete.getId());
+ }
+ } catch (Exception exception) {
+ LOG.error("deleteTagDef failed, guid=" + tagDef.getGuid(), exception);
+ throw exception;
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== ServiceTagsProcessor.delete()");
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
index dfa4a64..378ff0b 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/TagREST.java
@@ -29,6 +29,7 @@ import org.apache.ranger.plugin.model.RangerServiceResource;
import org.apache.ranger.plugin.model.RangerTag;
import org.apache.ranger.plugin.model.RangerTagResourceMap;
import org.apache.ranger.plugin.model.RangerTagDef;
+import org.apache.ranger.plugin.store.TagStore;
import org.apache.ranger.plugin.store.TagValidator;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServiceTags;
@@ -75,19 +76,30 @@ public class TagREST {
validator.setTagStore(tagStore);
}
+ TagStore getTagStore() {
+ return tagStore;
+ }
+
@POST
@Path(TagRESTConstants.TAGDEFS_RESOURCE)
@Produces({ "application/json", "application/xml" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
- public RangerTagDef createTagDef(RangerTagDef tagDef) {
+ public RangerTagDef createTagDef(RangerTagDef tagDef, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.createTagDef(" + tagDef + ")");
+ LOG.debug("==> TagREST.createTagDef(" + tagDef + ", " + updateIfExists + ")");
}
RangerTagDef ret;
try {
- ret = tagStore.createTagDef(tagDef);
+ RangerTagDef exist = validator.preCreateTagDef(tagDef, updateIfExists);
+ if (exist == null) {
+ ret = tagStore.createTagDef(tagDef);
+ } else if (updateIfExists) {
+ ret = updateTagDef(exist.getId(), exist);
+ } else {
+ throw new Exception("tag-definition with Id " + exist.getId() + " already exists");
+ }
} catch(Exception excp) {
LOG.error("createTagDef(" + tagDef + ") failed", excp);
@@ -95,12 +107,13 @@ public class TagREST {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== TagREST.createTagDef(" + tagDef + "): " + ret);
+ LOG.debug("<== TagREST.createTagDef(" + tagDef + ", " + updateIfExists + "): " + ret);
}
return ret;
}
+
@PUT
@Path(TagRESTConstants.TAGDEF_RESOURCE + "{id}")
@Produces({ "application/json", "application/xml" })
@@ -327,16 +340,22 @@ public class TagREST {
@Path(TagRESTConstants.TAGS_RESOURCE)
@Produces({ "application/json", "application/xml" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
- public RangerTag createTag(RangerTag tag) {
+ public RangerTag createTag(RangerTag tag, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.createTag(" + tag + ")");
+ LOG.debug("==> TagREST.createTag(" + tag + ", " + updateIfExists + ")");
}
RangerTag ret;
try {
- validator.preCreateTag(tag);
- ret = tagStore.createTag(tag);
+ RangerTag exist = validator.preCreateTag(tag);
+ if (exist == null) {
+ ret = tagStore.createTag(tag);
+ } else if (updateIfExists) {
+ ret = updateTag(exist.getId(), exist);
+ } else {
+ throw new Exception("tag with Id " + exist.getId() + " already exists");
+ }
} catch(Exception excp) {
LOG.error("createTag(" + tag + ") failed", excp);
@@ -344,7 +363,7 @@ public class TagREST {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== TagREST.createTag(" + tag + "): " + ret);
+ LOG.debug("<== TagREST.createTag(" + tag + ", " + updateIfExists + "): " + ret);
}
return ret;
@@ -553,16 +572,22 @@ public class TagREST {
@Path(TagRESTConstants.RESOURCES_RESOURCE)
@Produces({ "application/json", "application/xml" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
- public RangerServiceResource createServiceResource(RangerServiceResource resource) {
+ public RangerServiceResource createServiceResource(RangerServiceResource resource, @DefaultValue("true") @QueryParam("updateIfExists") boolean updateIfExists) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.createServiceResource(" + resource + ")");
+ LOG.debug("==> TagREST.createServiceResource(" + resource + ", " + updateIfExists + ")");
}
RangerServiceResource ret;
try {
- validator.preCreateServiceResource(resource);
- ret = tagStore.createServiceResource(resource);
+ RangerServiceResource exist = validator.preCreateServiceResource(resource);
+ if (exist == null) {
+ ret = tagStore.createServiceResource(resource);
+ } else if (updateIfExists) {
+ ret = updateServiceResource(exist.getId(), exist);
+ } else {
+ throw new Exception("resource with Id " + exist.getId() + " already exists");
+ }
} catch(Exception excp) {
LOG.error("createServiceResource(" + resource + ") failed", excp);
@@ -570,7 +595,7 @@ public class TagREST {
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== TagREST.createServiceResource(" + resource + "): " + ret);
+ LOG.debug("<== TagREST.createServiceResource(" + resource + ", " + updateIfExists + "): " + ret);
}
return ret;
@@ -650,22 +675,30 @@ public class TagREST {
@Path(TagRESTConstants.RESOURCE_RESOURCE + "guid/{guid}")
@Produces({ "application/json", "application/xml" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
- public void deleteServiceResourceByGuid(@PathParam("guid") String guid) {
+ public void deleteServiceResourceByGuid(@PathParam("guid") String guid, @DefaultValue("false") @QueryParam("deleteReferences") boolean deleteReferences) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ")");
+ LOG.debug("==> TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")");
}
try {
- RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid);
+ RangerServiceResource exist = validator.preDeleteServiceResourceByGuid(guid, deleteReferences);
+ if (deleteReferences) {
+ List<RangerTagResourceMap> tagResourceMaps = tagStore.getTagResourceMapsForResourceGuid(exist.getGuid());
+ if (CollectionUtils.isNotEmpty(tagResourceMaps)) {
+ for (RangerTagResourceMap tagResourceMap : tagResourceMaps) {
+ deleteTagResourceMap(tagResourceMap.getId());
+ }
+ }
+ }
tagStore.deleteServiceResource(exist.getId());
} catch(Exception excp) {
- LOG.error("deleteServiceResourceByGuid(" + guid + ") failed", excp);
+ LOG.error("deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ") failed", excp);
throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
}
if(LOG.isDebugEnabled()) {
- LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ")");
+ LOG.debug("<== TagREST.deleteServiceResourceByGuid(" + guid + ", " + deleteReferences + ")");
}
}
@@ -800,24 +833,31 @@ public class TagREST {
@Path(TagRESTConstants.TAGRESOURCEMAPS_RESOURCE)
@Produces({ "application/json", "application/xml" })
@PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
- public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid) {
+ public RangerTagResourceMap createTagResourceMap(@QueryParam("tag-guid") String tagGuid, @QueryParam("resource-guid") String resourceGuid,
+ @DefaultValue("false") @QueryParam("lenient") boolean lenient) {
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ")");
+ LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ")");
}
RangerTagResourceMap tagResourceMap;
try {
- tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid);
- tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+ tagResourceMap = tagStore.getTagResourceMapForTagAndResourceGuid(tagGuid, resourceGuid);
+ if (tagResourceMap == null) {
+ tagResourceMap = validator.preCreateTagResourceMap(tagGuid, resourceGuid);
+
+ tagResourceMap = tagStore.createTagResourceMap(tagResourceMap);
+ } else if (!lenient) {
+ throw new Exception("tagResourceMap with tag-guid=" + tagGuid + " and resource-guid=" + resourceGuid + " already exists");
+ }
} catch(Exception excp) {
- LOG.error("createTagResourceMap(" + tagGuid + ", " + resourceGuid + ") failed", excp);
+ LOG.error("createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ") failed", excp);
throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
}
if(LOG.isDebugEnabled()) {
- LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ")");
+ LOG.debug("==> TagREST.createTagResourceMap(" + tagGuid + ", " + resourceGuid + ", " + lenient + ")");
}
return tagResourceMap;
@@ -998,8 +1038,35 @@ public class TagREST {
return ret;
}
+ // This API is used by tag-sync to upload tag-objects
+
+ @PUT
+ @Path(TagRESTConstants.IMPORT_SERVICETAGS_RESOURCE)
+ @Produces({ "application/json", "application/xml" })
+ @PreAuthorize("hasRole('ROLE_SYS_ADMIN')")
+ public void importServiceTags(ServiceTags serviceTags) {
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("==> TagREST.importServiceTags()");
+ }
+
+ try {
+
+ ServiceTagsProcessor serviceTagsProcessor = new ServiceTagsProcessor(tagStore);
+ serviceTagsProcessor.process(serviceTags);
+
+ } catch (Exception excp) {
+ LOG.error("importServiceTags() failed", excp);
+
+ throw restErrorUtil.createRESTException(HttpServletResponse.SC_BAD_REQUEST, excp.getMessage(), true);
+
+ }
+ if(LOG.isDebugEnabled()) {
+ LOG.debug("<== TagREST.importServiceTags()");
+ }
+ }
// This API is typically used by plug-in to get selected tagged resources from RangerAdmin
+
@GET
@Path(TagRESTConstants.TAGS_DOWNLOAD + "{serviceName}")
@Produces({ "application/json", "application/xml" })
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
index 977e487..919f814 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
@@ -29,6 +29,7 @@ public class TagRESTConstants {
static final String RESOURCES_RESOURCE = "/resources/";
static final String RESOURCE_RESOURCE = "/resource/";
static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/";
+ static final String IMPORT_SERVICETAGS_RESOURCE = "/importservicetags/";
static final String TAGRESOURCEMAP_RESOURCE = "/tagresourcemap/";
static final String TAGTYPES_RESOURCE = "/types/";
static final String TAGTYPES_LOOKUP_RESOURCE = "/types/lookup/";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
index d50a937..40d9d64 100644
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerTagResourceMapService.java
@@ -132,6 +132,12 @@ public class RangerTagResourceMapService extends RangerTagResourceMapServiceBase
return ret;
}
+ public List<Long> getTagIdsForResourceId(Long resourceId) {
+ List<Long> ret = daoMgr.getXXTagResourceMap().findTagIdsForResourceId(resourceId);
+
+ return ret;
+ }
+
public List<RangerTagResourceMap> getByResourceGuid(String resourceGuid) {
List<RangerTagResourceMap> ret = new ArrayList<RangerTagResourceMap>();
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
----------------------------------------------------------------------
diff --git a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
index a5f4a6c..993e7d6 100644
--- a/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
+++ b/security-admin/src/main/resources/META-INF/jpa_named_queries.xml
@@ -599,6 +599,12 @@
where obj.resourceId = res.id and res.serviceId = :serviceId
order by obj.resourceId</query>
</named-query>
+
+ <named-query name="XXTagResourceMap.getTagIdsForResourceId">
+ <query>select obj.tagId from XXTagResourceMap obj
+ where obj.resourceId = :resourceId
+ </query>
+ </named-query>
<named-query name="XXTagDef.findByGuid">
<query>select obj from XXTagDef obj where obj.guid = :guid</query>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/src/main/assembly/tagsync.xml
----------------------------------------------------------------------
diff --git a/src/main/assembly/tagsync.xml b/src/main/assembly/tagsync.xml
index d7262ac..4f43851 100644
--- a/src/main/assembly/tagsync.xml
+++ b/src/main/assembly/tagsync.xml
@@ -47,14 +47,24 @@
<include>org.slf4j:slf4j-api</include>
<include>org.apache.hadoop:hadoop-common</include>
<include>org.apache.commons:commons-csv</include>
- <include>org.apache.ranger:credentialbuilder</include>
- <include>org.apache.ranger:ranger-util</include>
<include>commons-io:commons-io:jar:${commons.io.version}</include>
<include>org.apache.htrace:htrace-core</include>
<include>org.codehaus.jackson:jackson-core-asl</include>
<include>org.codehaus.jackson:jackson-jaxrs</include>
<include>org.codehaus.jackson:jackson-mapper-asl</include>
<include>org.codehaus.jackson:jackson-xc</include>
+ <include>org.apache.atlas:atlas-notification</include>
+ <include>org.apache.atlas:atlas-typesystem</include>
+ <include>org.apache.atlas:atlas-client</include>
+ <include>com.google.inject:guice</include>
+ <include>aopalliance:aopalliance</include>
+ <include>javax.inject:javax.inject</include>
+ <include>org.apache.kafka:kafka_2.10</include>
+ <include>org.apache.kafka:kafka-clients</include>
+ <include>com.yammer.metrics:metrics-core</include>
+ <include>org.scala-lang:scala-library</include>
+ <include>com.101tec:zkclient</include>
+ <include>org.apache.zookeeper:zookeeper</include>
<include>security_plugins.ranger-plugins-common:ranger-plugins-common</include>
</includes>
<unpack>false</unpack>
@@ -66,6 +76,29 @@
<include>org.apache.ranger:ranger-tagsync</include>
</includes>
</moduleSet>
+ <moduleSet>
+ <binaries>
+ <includeDependencies>false</includeDependencies>
+ <unpack>false</unpack>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <dependencySets>
+ <dependencySet>
+ <outputDirectory>/lib</outputDirectory>
+ <includes>
+ <include>org.apache.ranger:credentialbuilder</include>
+ <include>org.apache.ranger:ranger-util</include>
+ </includes>
+ <unpack>false</unpack>
+ </dependencySet>
+ </dependencySets>
+ <outputDirectory>/lib</outputDirectory>
+ </binaries>
+ <includes>
+ <include>org.apache.ranger:credentialbuilder</include>
+ <include>org.apache.ranger:ranger-util</include>
+ </includes>
+ </moduleSet>
</moduleSets>
<fileSets>
<fileSet>
@@ -73,6 +106,15 @@
<fileMode>644</fileMode>
<outputDirectory>/conf.dist</outputDirectory>
<directory>tagsync/conf.dist</directory>
+ <excludes>
+ <exclude>etc</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directoryMode>755</directoryMode>
+ <fileMode>644</fileMode>
+ <outputDirectory>/conf.dist/etc/ranger/data</outputDirectory>
+ <directory>tagsync/src/main/resources/etc/ranger/data</directory>
</fileSet>
<fileSet>
<directoryMode>755</directoryMode>
@@ -98,7 +140,7 @@
<file>
<source>tagsync/scripts/initd</source>
<outputDirectory>/</outputDirectory>
- <destName>ranger-usersync.sh</destName>
+ <destName>ranger-tagsync.sh</destName>
<fileMode>755</fileMode>
</file>
</files>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/log4j.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/log4j.xml b/tagsync/conf.dist/log4j.xml
index fb6986f..d455509 100644
--- a/tagsync/conf.dist/log4j.xml
+++ b/tagsync/conf.dist/log4j.xml
@@ -23,14 +23,14 @@
<param name="file" value="${logdir}/tagsync.log" />
<param name="DatePattern" value="'.'yyyy-MM-dd" />
<layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/>
+ <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/>
</layout>
</appender>
<appender name="console" class="org.apache.log4j.ConsoleAppender">
<param name="Target" value="System.out"/>
<layout class="org.apache.log4j.PatternLayout">
- <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %m%n"/>
+ <param name="ConversionPattern" value="%d{dd MMM yyyy HH:mm:ss} %5p %c{1} [%t] - %L %m%n"/>
</layout>
</appender>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-default-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/ranger-tagsync-default-site.xml b/tagsync/conf.dist/ranger-tagsync-default-site.xml
deleted file mode 100644
index b098740..0000000
--- a/tagsync/conf.dist/ranger-tagsync-default-site.xml
+++ /dev/null
@@ -1,36 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
-<!--
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-<!-- Put site-specific property overrides in this file. -->
-
-<configuration>
- <property>
- <name>ranger.tagsync.port</name>
- <value>6161</value>
- </property>
- <property>
- <name>ranger.tagsync.ssl</name>
- <value>true</value>
- </property>
- <property>
- <name>ranger.tagsync.enabled</name>
- <value>true</value>
- </property>
- <property>
- <name>ranger.tagsync.logdir</name>
- <value>./log</value>
- </property>
-</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/conf.dist/ranger-tagsync-site.xml
----------------------------------------------------------------------
diff --git a/tagsync/conf.dist/ranger-tagsync-site.xml b/tagsync/conf.dist/ranger-tagsync-site.xml
new file mode 100644
index 0000000..21accd4
--- /dev/null
+++ b/tagsync/conf.dist/ranger-tagsync-site.xml
@@ -0,0 +1,67 @@
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+
+<configuration>
+ <property>
+ <name>ranger.authentication.method</name>
+ <value>NONE</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.tagadmin.rest.url</name>
+ <value>http://localhost:6080</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.tagadmin.rest.ssl.config.file</name>
+ <value></value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.policymanager.basicauth.username</name>
+ <value>admin</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.policymanager.basicauth.password</name>
+ <value>admin</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.sleeptimeinmillisbetweensynccycle</name>
+ <value>60000</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.file</name>
+ <value>/etc/ranger/data/tags.json</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.source.impl.class</name>
+ <value>org.apache.ranger.source.file.TagFileSource</value>
+ <description></description>
+ </property>
+ <property>
+ <name>ranger.tagsync.sink.impl.class</name>
+ <value>org.apache.ranger.sink.policymgr.TagRESTSink</value>
+ <description></description>
+ </property>
+ <property>
+ <name>atlas.endpoint</name>
+ <value>http://localhost:21000/</value>
+ <description></description>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/pom.xml
----------------------------------------------------------------------
diff --git a/tagsync/pom.xml b/tagsync/pom.xml
index 83011d2..18c684c 100644
--- a/tagsync/pom.xml
+++ b/tagsync/pom.xml
@@ -86,7 +86,6 @@
<artifactId>ranger-plugins-common</artifactId>
<version>${project.version}</version>
</dependency>
- <!--
<dependency>
<groupId>com.google.inject</groupId>
<artifactId>guice</artifactId>
@@ -96,17 +95,22 @@
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-notification</artifactId>
<version>0.6-incubating-SNAPSHOT</version>
+ <scope>system</scope>
+ <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-notification-0.6-incubating-SNAPSHOT.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-typesystem</artifactId>
<version>0.6-incubating-SNAPSHOT</version>
+ <scope>system</scope>
+ <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-typesystem-0.6-incubating-SNAPSHOT.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.atlas</groupId>
<artifactId>atlas-client</artifactId>
<version>0.6-incubating-SNAPSHOT</version>
+ <scope>system</scope>
+ <systemPath>${local.lib.dir}/apache-atlas-0.6-incubating-SNAPSHOT/atlas-client-0.6-incubating-SNAPSHOT.jar</systemPath>
</dependency>
- -->
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/samples/tags.json
----------------------------------------------------------------------
diff --git a/tagsync/samples/tags.json b/tagsync/samples/tags.json
new file mode 100644
index 0000000..1b24fc8
--- /dev/null
+++ b/tagsync/samples/tags.json
@@ -0,0 +1,55 @@
+{
+ "op": "add_or_update",
+ "serviceName": "cl1_hive",
+ "tagVersion": 1,
+ "tagUpdateTime": "20150924-22:26:33.000-+0000",
+ "tagDefinitions": {
+ "1":{"id":1, "guid":"tagdef-1", "name":"PII", "attributeDefs":[]},
+ "2":{"id":2, "guid":"tagdef-2", "name":"EXPIRES_ON", "attributeDefs":[{"name":"expiry_date", "type":"date"}]},
+ "3":{"id":3, "guid":"tagdef-3", "name":"FINANCE", "attributeDefs":[]},
+ "4":{"id":4, "guid":"tagdef-4", "name":"AUDIT", "attributeDefs":[]}
+ },
+ "tags": {
+ "1":{"id":1, "guid":"tag-1", "type":"PII", "attributes":{}},
+ "2":{"id":2, "guid":"tag-2", "type":"EXPIRES_ON", "attributes":{"expiry_date":"2015/08/31"}},
+ "3":{"id":3, "guid":"tag-3", "type":"FINANCE", "attributes":{}},
+ "4":{"id":4, "guid":"tag-4", "type":"AUDIT", "attributes":{}}
+ },
+ "serviceResources": [
+ {
+ "id":1,
+ "guid":"resource-1",
+ "serviceName":"cl1_hive",
+ "resourceElements":{
+ "database":{"values":["hr"]},
+ "table":{"values":["employee"]},
+ "column":{"values":["ssn"]}
+ }
+ },
+ {
+ "id":2,
+ "guid":"resource-2",
+ "serviceName":"cl1_hive",
+ "resourceElements":{
+ "database":{"values":["finance"]},
+ "table":{"values":["tax_2010"]}
+ }
+ },
+ {
+ "id":3,
+ "guid":"resource-3",
+ "serviceName":"cl1_hive",
+ "resourceElements":{
+ "database":{"values":["finance"]},
+ "table":{"values":["tax_2010"]},
+ "column":{"values":["ssn"]}
+ }
+ }
+ ],
+
+ "resourceToTagIds": {
+ "1":[1],
+ "2":[2],
+ "3":[3]
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/scripts/ranger-tagsync-services.sh
----------------------------------------------------------------------
diff --git a/tagsync/scripts/ranger-tagsync-services.sh b/tagsync/scripts/ranger-tagsync-services.sh
index 5ca5b95..2f4b255 100755
--- a/tagsync/scripts/ranger-tagsync-services.sh
+++ b/tagsync/scripts/ranger-tagsync-services.sh
@@ -27,8 +27,13 @@ realScriptDir=`dirname $realScriptPath`
cd $realScriptDir
cdir=`pwd`
-pidf=/var/run/ranger/tagsync.pid
+pidd=/var/run/ranger
+if [ -d $pidd ]; then
+ mkdir -p $pidd
+fi
+
+pidf=${pidd}/tagsync.pid
if [ "${action}" == "START" ]; then
@@ -47,9 +52,13 @@ if [ "${action}" == "START" ]; then
export PATH=$JAVA_HOME/bin:$PATH
fi
- logdir=/var/log/ranger/tagsync
+ logdir=/var/log/ranger/tagsync
+
+ if [ ! -d $logdir ]; then
+ mkdir -p $logdir
+ fi
- cp="${cdir}/dist/*:${cdir}/lib/*:${cdir}/conf"
+ cp="${cdir}/conf.dist:${cdir}/dist/*:${cdir}/lib/*"
if [ -f $pidf ]; then
PID=`cat $pidf`
@@ -81,7 +90,6 @@ if [ "${action}" == "START" ]; then
elif [ "${action}" == "STOP" ]; then
if [ -f $pidf ]; then
- pidf=/var/run/ranger/tagsync.pid
PID=`cat $pidf` > /dev/null 2>&1
kill -9 $PID > /dev/null 2>&1
rm -f $pidf
@@ -103,8 +111,8 @@ elif [ "${action}" == "VERSION" ]; then
java -cp ranger-util-*.jar org.apache.ranger.common.RangerVersionInfo
exit
else
- echo "Invalid argument [$1];"
- echo "Usage: Only start | stop | restart | version, are supported."
- exit;
+ echo "Invalid argument [$1];"
+ echo "Usage: Only start | stop | restart | version, are supported."
+ exit;
fi
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
index 74ea5cf..5256570 100644
--- a/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
+++ b/tagsync/src/main/java/org/apache/ranger/model/TagSink.java
@@ -26,9 +26,6 @@ import java.util.Map;
import java.util.Properties;
-/**
- * Created by akulkarni on 9/10/15.
- */
public interface TagSink extends TagStore {
boolean initialize(Properties properties);
void uploadServiceTags(ServiceTags serviceTags) throws Exception;
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
index 568d3c7..0ccceae 100644
--- a/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
+++ b/tagsync/src/main/java/org/apache/ranger/model/TagSource.java
@@ -26,9 +26,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-/**
- * Created by akulkarni on 9/10/15.
- */
public interface TagSource {
boolean initialize(Properties properties);
@@ -37,16 +34,8 @@ public interface TagSource {
void updateSink() throws Exception;
- void start();
+ Thread start();
boolean isChanged();
- List<RangerTagDef> fetchAllTagDefs(String syncSentinel) throws Exception;
-
- List<RangerTagDef> receiveUpdatesToTagDefs() throws Exception;
-
- List<RangerTagResourceMap> fetchAllTaggedEntities() throws Exception;
-
- List<RangerTagResourceMap> receiveUpdatesToTaggedEntities() throws Exception;
-
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
index 1c995b1..c80eff9 100644
--- a/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
+++ b/tagsync/src/main/java/org/apache/ranger/process/TagSyncConfig.java
@@ -20,27 +20,20 @@
package org.apache.ranger.process;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.log4j.Logger;
-import org.w3c.dom.Document;
-import org.w3c.dom.Element;
-import org.w3c.dom.Node;
-import org.w3c.dom.NodeList;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
import java.io.*;
+import java.net.MalformedURLException;
import java.net.URL;
import java.util.Properties;
-/**
- * Created by akulkarni on 9/11/15.
- */
-public class TagSyncConfig {
+public class TagSyncConfig extends Configuration {
private static final Logger LOG = Logger.getLogger(TagSyncConfig.class) ;
public static final String CONFIG_FILE = "ranger-tagsync-site.xml";
- public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default-site.xml";
+ public static final String DEFAULT_CONFIG_FILE = "ranger-tagsync-default.xml";
public static final String TAGSYNC_ENABLED_PROP = "ranger.tagsync.enabled" ;
@@ -50,9 +43,9 @@ public class TagSyncConfig {
public static final String TAGSYNC_LOGDIR_PROP = "ranger.tagsync.logdir" ;
- private static final String TAGSYNC_PM_URL_PROP = "ranger.tagsync.policymanager.baseURL";
+ private static final String TAGSYNC_TAGADMIN_REST_URL_PROP = "ranger.tagsync.tagadmin.rest.url";
- private static final String TAGSYNC_PM_SSL_CONFIG_FILE_PROP = "ranger.tagsync.policymanager.ssl.config.file";
+ private static final String TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP = "ranger.tagsync.tagadmin.rest.ssl.config.file";
private static final String TAGSYNC_PM_SSL_BASICAUTH_USERNAME_PROP = "ranger.tagsync.policymanager.basicauth.username";
@@ -68,9 +61,13 @@ public class TagSyncConfig {
private static final String TAGSYNC_SOURCE_ATLAS_PROP = "atlas.endpoint";
- private static volatile TagSyncConfig instance = null;
+ private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX = "ranger.tagsync.atlas.";
+
+ private static final String TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX = ".ranger.service";
- private Properties prop = new Properties() ;
+ private static final String TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR = "_";
+
+ private static volatile TagSyncConfig instance = null;
public static TagSyncConfig getInstance() {
/*
@@ -85,12 +82,11 @@ public class TagSyncConfig {
}
*/
TagSyncConfig newConfig = new TagSyncConfig();
- newConfig.init();
return newConfig;
}
public Properties getProperties() {
- return prop;
+ return getProps();
}
public static InputStream getFileInputStream(String path) throws FileNotFoundException {
@@ -158,7 +154,7 @@ public class TagSyncConfig {
LOG.error(path + " is not a file", exception);
}
} else {
- LOG.error("URL not found for " + path + " or no privilege for reading file " + path);
+ LOG.warn("URL not found for " + path + " or no privilege for reading file " + path);
}
}
}
@@ -170,16 +166,10 @@ public class TagSyncConfig {
public String toString() {
StringBuffer sb = new StringBuffer();
- sb.append("CONFIG_FILE=").append(CONFIG_FILE).append(", ")
- .append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append("\n");
-
- ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
- PrintStream printStream = new PrintStream(outputStream);
- prop.list(printStream);
- printStream.close();
- sb.append(outputStream.toString());
+ sb.append("DEFAULT_CONFIG_FILE=").append(DEFAULT_CONFIG_FILE).append(", ")
+ .append("CONFIG_FILE=").append(CONFIG_FILE).append("\n\n");
- return sb.toString();
+ return sb.toString() + super.toString();
}
static public boolean isTagSyncEnabled(Properties prop) {
@@ -217,13 +207,13 @@ public class TagSyncConfig {
return val;
}
- static public String getPolicyMgrUrl(Properties prop) {
- String val = prop.getProperty(TAGSYNC_PM_URL_PROP);
+ static public String getTagAdminRESTUrl(Properties prop) {
+ String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_URL_PROP);
return val;
}
- static public String getPolicyMgrSslConfigFile(Properties prop) {
- String val = prop.getProperty(TAGSYNC_PM_SSL_CONFIG_FILE_PROP);
+ static public String getTagAdminRESTSslConfigFile(Properties prop) {
+ String val = prop.getProperty(TAGSYNC_TAGADMIN_REST_SSL_CONFIG_FILE_PROP);
return val;
}
@@ -251,75 +241,39 @@ public class TagSyncConfig {
return "";
}
+ static public String getServiceName(String componentName, String instanceName, Properties prop) {
+ String propName = TAGSYNC_SERVICENAME_MAPPER_PROP_PREFIX + componentName
+ + ".instance." + instanceName
+ + TAGSYNC_SERVICENAME_MAPPER_PROP_SUFFIX;
+ String val = prop.getProperty(propName);
+ if (StringUtils.isBlank(val)) {
+ val = instanceName + TAGSYNC_DEFAULT_CLUSTERNAME_AND_COMPONENTNAME_SEPARATOR + componentName;
+ }
+ return val;
+ }
+
private TagSyncConfig() {
+ super(false);
init() ;
}
private void init() {
- readConfigFile(CONFIG_FILE);
readConfigFile(DEFAULT_CONFIG_FILE);
+ readConfigFile(CONFIG_FILE);
}
private void readConfigFile(String fileName) {
- try {
- InputStream in = getFileInputStream(fileName);
- if (in != null) {
- try {
- DocumentBuilderFactory xmlDocumentBuilderFactory = DocumentBuilderFactory
- .newInstance();
- xmlDocumentBuilderFactory.setIgnoringComments(true);
- xmlDocumentBuilderFactory.setNamespaceAware(true);
- DocumentBuilder xmlDocumentBuilder = xmlDocumentBuilderFactory
- .newDocumentBuilder();
- Document xmlDocument = xmlDocumentBuilder.parse(in);
- xmlDocument.getDocumentElement().normalize();
-
- NodeList nList = xmlDocument
- .getElementsByTagName("property");
-
- for (int temp = 0; temp < nList.getLength(); temp++) {
-
- Node nNode = nList.item(temp);
-
- if (nNode.getNodeType() == Node.ELEMENT_NODE) {
-
- Element eElement = (Element) nNode;
-
- String propertyName = "";
- String propertyValue = "";
- if (eElement.getElementsByTagName("name").item(
- 0) != null) {
- propertyName = eElement
- .getElementsByTagName("name")
- .item(0).getTextContent().trim();
- }
- if (eElement.getElementsByTagName("value")
- .item(0) != null) {
- propertyValue = eElement
- .getElementsByTagName("value")
- .item(0).getTextContent().trim();
- }
-
- if (prop.get(propertyName) != null) {
- prop.remove(propertyName) ;
- }
-
- prop.put(propertyName, propertyValue);
- }
- }
- }
- finally {
- try {
- in.close() ;
- }
- catch(IOException ioe) {
- // Ignore IOE when closing stream
- }
- }
+ if (StringUtils.isNotBlank(fileName)) {
+ String fName = getResourceFileName(fileName);
+ if (StringUtils.isBlank(fName)) {
+ LOG.warn("Cannot find configuration file " + fileName + " in the classpath");
+ } else {
+ LOG.info("Loading configuration from " + fName);
+ addResource(fileName);
}
- } catch (Throwable e) {
- throw new RuntimeException("Unable to load configuration file [" + CONFIG_FILE + "]", e) ;
+ } else {
+ LOG.error("Configuration fileName is null");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
index c7e8d6e..8ab2a1a 100644
--- a/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
+++ b/tagsync/src/main/java/org/apache/ranger/process/TagSynchronizer.java
@@ -24,16 +24,14 @@ import org.apache.log4j.Logger;
import org.apache.ranger.model.TagSink;
import org.apache.ranger.model.TagSource;
+import java.util.Map;
import java.util.Properties;
-/**
- * Created by akulkarni on 9/11/15.
- */
public class TagSynchronizer implements Runnable {
private static final Logger LOG = Logger.getLogger(TagSynchronizer.class);
- private final static int MAX_INIT_RETRIES = 1;
+ private final static int MAX_INIT_RETRIES = 5;
private boolean shutdownFlag = false;
private TagSink tagSink = null;
@@ -46,12 +44,6 @@ public class TagSynchronizer implements Runnable {
TagSyncConfig config = TagSyncConfig.getInstance();
Properties props = config.getProperties();
- LOG.info("--------------------------------");
- LOG.info("");
- LOG.info("Ranger-TagSync Configuration: {\n" + config + "\n}");
- LOG.info("");
- LOG.info("--------------------------------");
-
TagSynchronizer tagSynchronizer = new TagSynchronizer(props);
tagSynchronizer.run();
@@ -86,15 +78,28 @@ public class TagSynchronizer implements Runnable {
if (initDone) {
- tagSource.start();
+ Thread tagSourceThread = tagSource.start();
- while (!shutdownFlag) {
- try {
- LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
- Thread.sleep(sleepTimeBetweenCycleInMillis);
- } catch (InterruptedException e) {
- LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e);
+ if (tagSourceThread != null) {
+ while (!shutdownFlag) {
+ try {
+ LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
+ Thread.sleep(sleepTimeBetweenCycleInMillis);
+ } catch (InterruptedException e) {
+ LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to synchronize tag information", e);
+ }
}
+ if (shutdownFlag) {
+ LOG.info("Interrupting tagSourceThread...");
+ tagSourceThread.interrupt();
+ try {
+ tagSourceThread.join();
+ } catch (InterruptedException interruptedException) {
+ LOG.error("tagSourceThread.join() was interrupted");
+ }
+ }
+ } else {
+ LOG.error("Could not start tagSource monitoring thread");
}
} else {
LOG.error("Failed to initialize TagSynchonizer after " + MAX_INIT_RETRIES + " retries. Exiting thread");
@@ -121,6 +126,8 @@ public class TagSynchronizer implements Runnable {
for (int initRetries = 0; initRetries < MAX_INIT_RETRIES && !ret; initRetries++) {
+ printConfigurationProperties();
+
ret = init();
if (!ret) {
@@ -128,6 +135,7 @@ public class TagSynchronizer implements Runnable {
try {
LOG.debug("Sleeping for [" + sleepTimeBetweenCycleInMillis + "] milliSeconds");
Thread.sleep(sleepTimeBetweenCycleInMillis);
+ properties = TagSyncConfig.getInstance().getProperties();
} catch (Exception e) {
LOG.error("Failed to wait for [" + sleepTimeBetweenCycleInMillis + "] milliseconds before attempting to initialize tag source/sink", e);
}
@@ -181,7 +189,23 @@ public class TagSynchronizer implements Runnable {
}
public void shutdown(String reason) {
- LOG.error("Received shutdown(), reason=" + reason);
+ LOG.info("Received shutdown(), reason=" + reason);
this.shutdownFlag = true;
}
+
+ public void printConfigurationProperties() {
+ LOG.info("--------------------------------");
+ LOG.info("");
+ LOG.info("Ranger-TagSync Configuration: {\n");
+ if (MapUtils.isNotEmpty(properties)) {
+ for (Map.Entry<Object, Object> entry : properties.entrySet()) {
+ LOG.info("\tProperty-Name:" + entry.getKey());
+ LOG.info("\tProperty-Value:" + entry.getValue());
+ LOG.info("\n");
+ }
+ }
+ LOG.info("\n}");
+ LOG.info("");
+ LOG.info("--------------------------------");
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/eb874038/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
----------------------------------------------------------------------
diff --git a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
index 98410d6..e934ce4 100644
--- a/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
+++ b/tagsync/src/main/java/org/apache/ranger/sink/policymgr/TagRESTSink.java
@@ -39,9 +39,6 @@ import java.util.List;
import java.util.Map;
import java.util.Properties;
-/**
- * Created by akulkarni on 9/11/15.
- */
public class TagRESTSink implements TagSink {
private static final Log LOG = LogFactory.getLog(TagRESTSink.class);
@@ -57,8 +54,6 @@ public class TagRESTSink implements TagSink {
private static final String REST_URL_TAG_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tag/" ;
private static final String REST_URL_TAGRESOURCEMAP_IDS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/tagresourcemapids/";
private static final String REST_URL_IMPORT_SERVICETAGS_RESOURCE = REST_PREFIX + MODULE_PREFIX + "/importservicetags/";
- public static final String REST_URL_IMPORT_SERVICETAGS_PARAM = "op";
-
private RangerRESTClient tagRESTClient = null;
@@ -73,8 +68,8 @@ public class TagRESTSink implements TagSink {
boolean ret = false;
- String restUrl = TagSyncConfig.getPolicyMgrUrl(properties);
- String sslConfigFile = TagSyncConfig.getPolicyMgrSslConfigFile(properties);
+ String restUrl = TagSyncConfig.getTagAdminRESTUrl(properties);
+ String sslConfigFile = TagSyncConfig.getTagAdminRESTSslConfigFile(properties);
String userName = TagSyncConfig.getPolicyMgrUserName(properties);
String password = TagSyncConfig.getPolicyMgrPassword(properties);
@@ -267,6 +262,11 @@ public class TagRESTSink implements TagSink {
}
@Override
+ public List<Long> getTagIdsForResourceId(Long resourceId) throws Exception {
+ throw new Exception("Not implemented");
+ }
+
+ @Override
public List<RangerTag> getTagsForResourceId(Long resourceId) throws Exception {
throw new Exception("Not implemented");
}