You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2015/08/29 03:30:10 UTC
incubator-ranger git commit: RANGER-274: Fixed minor bugs,
potential NPE, REST resource path. Fixed service creation
Repository: incubator-ranger
Updated Branches:
refs/heads/tag-policy 5876e2272 -> f8bea62bc
RANGER-274: Fixed minor bugs, potential NPE, REST resource path. Fixed service creation
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-ranger/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ranger/commit/f8bea62b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ranger/tree/f8bea62b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ranger/diff/f8bea62b
Branch: refs/heads/tag-policy
Commit: f8bea62bccbca1eaf16083dec4cf1b43ac9b439d
Parents: 5876e22
Author: Abhay Kulkarni <ak...@hortonworks.com>
Authored: Fri Aug 28 15:31:02 2015 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Aug 28 18:11:08 2015 -0700
----------------------------------------------------------------------
.../admin/client/RangerAdminRESTClient.java | 3 +-
.../RangerAdminTagRetriever.java | 24 +++-----
.../contextenricher/RangerTagProvider.java | 7 +--
.../ranger/plugin/store/file/TagFileStore.java | 8 +--
.../ranger/plugin/util/RangerRESTUtils.java | 4 +-
.../ranger/plugin/store/TestTagStore.java | 64 ++++++++++----------
.../org/apache/ranger/biz/ServiceDBStore.java | 1 +
.../apache/ranger/rest/TagRESTConstants.java | 4 +-
.../service/RangerServiceServiceBase.java | 4 ++
9 files changed, 58 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
index 5ec7996..de138ed 100644
--- a/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
+++ b/agents-common/src/main/java/org/apache/ranger/admin/client/RangerAdminRESTClient.java
@@ -204,8 +204,7 @@ public class RangerAdminRESTClient implements RangerAdminClient {
ServiceTags ret;
- WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED)
- .queryParam(RangerRESTUtils.SERVICE_NAME_PARAM, serviceName)
+ WebResource webResource = createWebResource(RangerRESTUtils.REST_URL_GET_SERVICE_TAGS_IF_UPDATED + serviceName)
.queryParam(RangerRESTUtils.LAST_KNOWN_TAG_VERSION_PARAM, Long.toString(lastKnownVersion))
.queryParam(RangerRESTUtils.REST_PARAM_PLUGIN_ID, pluginId);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
index 987ed45..889fbbc 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerAdminTagRetriever.java
@@ -69,29 +69,25 @@ public class RangerAdminTagRetriever extends RangerTagRefresher {
@Override
public void retrieveTags() {
- if (adminClient != null) {
+ if (adminClient != null && receiver != null) {
ServiceTags serviceTags = null;
- long savedLastKnownVersion = lastKnownVersion;
try {
serviceTags = adminClient.getServiceTagsIfUpdated(lastKnownVersion);
- lastKnownVersion = serviceTags.getTagVersion();
} catch (Exception exp) {
- LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources");
+ LOG.error("RangerAdminTagRetriever.retrieveTags() - Error retrieving resources, exception=", exp);
}
- if (receiver != null && serviceTags != null) {
- if (serviceTags.getTagVersion() != null && serviceTags.getTagVersion().longValue() > savedLastKnownVersion) {
- receiver.setServiceTags(serviceTags);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("RangerAdminTagRetriever.retrieveTags() - no updates to tags !!");
- }
- }
+ if (serviceTags != null) {
+ LOG.info("RangerAdminTagRetriever.retrieveTags() - Updating tags-cache to new version of tags, lastKnownVersion=" + lastKnownVersion + "; newVersion=" + serviceTags.getTagVersion());
+ lastKnownVersion = serviceTags.getTagVersion();
+ receiver.setServiceTags(serviceTags);
} else {
- LOG.error("RangerAdminTagRetriever.retrieveTags() - No receiver to send resources to ");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("RangerAdminTagRetriever.retrieveTags() - No need to update tags-cache. lastKnownVersion=" + lastKnownVersion);
+ }
}
} else {
- LOG.error("RangerAdminTagRetriever.retrieveTags() - No Tag Provider ...");
+ LOG.error("RangerAdminTagRetriever.retrieveTags() - No admin client to get tags from or no tag receiver to update tag-cache");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java
index f78515c..83600c9 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/contextenricher/RangerTagProvider.java
@@ -161,17 +161,14 @@ public class RangerTagProvider extends RangerAbstractContextEnricher implements
for (RangerServiceResourceMatcher resourceMatcher : resourceMatchers) {
- RangerServiceResource serviceResource = resourceMatcher.getServiceResource();
- RangerPolicyResourceMatcher matcher = resourceMatcher.getPolicyResourceMatcher();
-
- boolean matchResult = matcher.isMatch(resource);
+ boolean matchResult = resourceMatcher.isMatch(resource);
if (matchResult) {
if (ret == null) {
ret = new ArrayList<RangerTag>();
}
// Find tags from serviceResource
- ret.addAll(getTagsForServiceResource(serviceTags, serviceResource));
+ ret.addAll(getTagsForServiceResource(serviceTags, resourceMatcher.getServiceResource()));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
index 6117034..0b4f0ca 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/store/file/TagFileStore.java
@@ -1026,6 +1026,7 @@ public class TagFileStore extends AbstractTagStore {
filter.setParam(SearchFilter.TAG_RESOURCE_SERVICE_NAME, serviceName);
List<RangerServiceResource> serviceResources = getServiceResources(filter);
+ List<RangerServiceResource> filteredServiceResources = new ArrayList<RangerServiceResource>();
Map<Long, RangerTag> tagsMap = new HashMap<Long, RangerTag>();
Map<Long, List<Long>> resourceToTagIdsMap = new HashMap<Long, List<Long>>();
@@ -1040,14 +1041,11 @@ public class TagFileStore extends AbstractTagStore {
tagIdList.add(tag.getId());
}
resourceToTagIdsMap.put(serviceResource.getId(), tagIdList);
+ filteredServiceResources.add(serviceResource);
}
}
- if (MapUtils.isEmpty(resourceToTagIdsMap)) {
- serviceResources.clear();
- }
-
- ret.setServiceResources(serviceResources);
+ ret.setServiceResources(filteredServiceResources);
ret.setResourceToTagIds(resourceToTagIdsMap);
ret.setTags(tagsMap);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
----------------------------------------------------------------------
diff --git a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
index e162c5c..ad113fe 100644
--- a/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
+++ b/agents-common/src/main/java/org/apache/ranger/plugin/util/RangerRESTUtils.java
@@ -40,9 +40,9 @@ public class RangerRESTUtils {
public static final String REST_URL_SERVICE_GRANT_ACCESS = "/service/plugins/services/grant/";
public static final String REST_URL_SERVICE_REVOKE_ACCESS = "/service/plugins/services/revoke/";
- public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download";
+ public static final String REST_URL_GET_SERVICE_TAGS_IF_UPDATED = "/service/tags/download/";
public static final String SERVICE_NAME_PARAM = "serviceName";
- public static final String LAST_KNOWN_TAG_VERSION_PARAM = "tagVersion";
+ public static final String LAST_KNOWN_TAG_VERSION_PARAM = "lastKnownVersion";
public static final String PATTERN_PARAM = "pattern";
public static final String REST_URL_LOOKUP_TAG_NAMES = "/service/tags/lookup";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java
----------------------------------------------------------------------
diff --git a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java
index e0f4865..5424158 100644
--- a/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java
+++ b/agents-common/src/test/java/org/apache/ranger/plugin/store/TestTagStore.java
@@ -40,68 +40,66 @@ import org.apache.ranger.plugin.store.file.TagFileStore;
import org.apache.ranger.plugin.util.SearchFilter;
import org.apache.ranger.plugin.util.ServiceTags;
import org.junit.BeforeClass;
+import org.junit.AfterClass;
import org.junit.Test;
public class TestTagStore {
static TagStore tagStore = null;
static TagValidator validator = null;
-
+ static RangerServiceDef serviceDef = null;
+ static RangerService service = null;
static SearchFilter filter = null;
- static Path filePath = new Path("file:///etc/ranger/data/ranger-admin-test-site.xml");
+
+ static String tmpDir = null;
+ static Path filePath = null;
+
static Configuration config = new Configuration();
static final String serviceDefJsonFile = "/admin/service-defs/test-hive-servicedef.json";
static final String serviceName = "tag-unit-test-TestTagStore";
- static final String crcSuffix = ".crc";
- static final String jsonSuffix = ".json";
-
static Gson gsonBuilder = null;
- static RangerServiceDef serviceDef = null;
- static RangerService service = null;
@BeforeClass
public static void setupTest() throws Exception {
- tearDownAfterClass(crcSuffix);
- tearDownAfterClass(jsonSuffix);
+ tmpDir = "file://" + System.getProperty("java.io.tmpdir");
- FileSystem fs = filePath.getFileSystem(config);
-
- FSDataOutputStream outStream = fs.create(filePath, true);
- OutputStreamWriter writer = null;
-
- writer = new OutputStreamWriter(outStream);
-
- writer.write("<configuration>\n" +
+ String textTemplate = "<configuration>\n" +
" <property>\n" +
" <name>ranger.tag.store.file.dir</name>\n" +
- " <value>file:///etc/ranger/data</value>\n" +
+ " <value>%s</value>\n" +
" </property>\n" +
" <property>\n" +
" <name>ranger.service.store.file.dir</name>\n" +
- " <value>file:///etc/ranger/data</value>\n" +
+ " <value>%s</value>\n" +
" </property>\n" +
- "</configuration>\n");
+ "</configuration>\n";
+
+ String text = String.format(textTemplate, tmpDir, tmpDir);
+ String fileName = tmpDir + "/ranger-admin-test-site.xml";
+ filePath = new Path(fileName);
+ FileSystem fs = filePath.getFileSystem(config);
+
+ FSDataOutputStream outStream = fs.create(filePath, true);
+
+ OutputStreamWriter writer = new OutputStreamWriter(outStream);
+ writer.write(text);
writer.close();
RangerConfiguration config = RangerConfiguration.getInstance();
config.addResource(filePath);
- tagStore = TagFileStore.getInstance();
- tagStore.init();
-
- ServiceStore svcStore;
-
- svcStore = new ServiceFileStore();
+ ServiceStore svcStore = new ServiceFileStore();
svcStore.init();
+ tagStore = TagFileStore.getInstance();
+ tagStore.init();
tagStore.setServiceStore(svcStore);
validator = new TagValidator();
-
validator.setTagStore(tagStore);
gsonBuilder = new GsonBuilder().setDateFormat("yyyyMMdd-HH:mm:ss.SSS-Z")
@@ -110,17 +108,20 @@ public class TestTagStore {
InputStream inStream = TestTagStore.class.getResourceAsStream(serviceDefJsonFile);
InputStreamReader reader = new InputStreamReader(inStream);
+
serviceDef = gsonBuilder.fromJson(reader, RangerServiceDef.class);
+
service = svcStore.createService(new RangerService(serviceDef.getName(), serviceName, serviceName, null, null));
+
reader.close();
inStream.close();
}
- //@AfterClass
- public static void tearDownAfterClass(String suffix) throws Exception {
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
- Path dirPath = new Path("file:///etc/ranger/data");
+ Path dirPath = new Path(tmpDir);
FileSystem fs = dirPath.getFileSystem(config);
try {
@@ -132,7 +133,7 @@ public class TestTagStore {
while (files.hasNext()) {
LocatedFileStatus fileStatus = files.next();
Path path = fileStatus.getPath();
- if (fs.isFile(path) && path.getName().endsWith(suffix)) {
+ if (fs.isFile(path)) {
fs.delete(path, true);
}
}
@@ -142,6 +143,7 @@ public class TestTagStore {
}
fs.delete(filePath, true);
+
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
index 1fec611..a4765ac 100644
--- a/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
+++ b/security-admin/src/main/java/org/apache/ranger/biz/ServiceDBStore.java
@@ -1027,6 +1027,7 @@ public class ServiceDBStore extends AbstractServiceStore {
// While creating, value of version should be 1.
service.setVersion(new Long(1));
+ service.setTagVersion(new Long(1));
if (populateExistingBaseFields) {
svcServiceWithAssignedId.setPopulateExistingBaseFields(true);
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
index 07a559c..91525df 100644
--- a/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
+++ b/security-admin/src/main/java/org/apache/ranger/rest/TagRESTConstants.java
@@ -29,8 +29,8 @@ public class TagRESTConstants {
static final String RESOURCES_RESOURCE = "/resources/";
static final String RESOURCE_RESOURCE = "/resource/";
static final String TAGRESOURCEMAPS_RESOURCE = "/tagresourcemaps/";
- static final String TAGNAMES_RESOURCE = "/tags/names/";
- static final String LOOKUP_TAGS_RESOURCE = "/tags/lookup/";
+ static final String TAGNAMES_RESOURCE = "/names/";
+ static final String LOOKUP_TAGS_RESOURCE = "/lookup/";
static final String TAGS_DOWNLOAD = "/download/";
public static final String SERVICE_NAME_PARAM = "serviceName";
http://git-wip-us.apache.org/repos/asf/incubator-ranger/blob/f8bea62b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java
----------------------------------------------------------------------
diff --git a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java
index e594bb5..3cbeaf0 100755
--- a/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java
+++ b/security-admin/src/main/java/org/apache/ranger/service/RangerServiceServiceBase.java
@@ -95,7 +95,9 @@ public abstract class RangerServiceServiceBase<T extends XXServiceBase, V extend
xObj.setName(vObj.getName());
xObj.setTagService(tagServiceId);
xObj.setPolicyVersion(vObj.getPolicyVersion());
+ xObj.setTagVersion(vObj.getTagVersion());
xObj.setPolicyUpdateTime(vObj.getPolicyUpdateTime());
+ xObj.setTagUpdateTime(vObj.getTagUpdateTime());
xObj.setDescription(vObj.getDescription());
xObj.setIsEnabled(vObj.getIsEnabled());
return xObj;
@@ -113,7 +115,9 @@ public abstract class RangerServiceServiceBase<T extends XXServiceBase, V extend
vObj.setDescription(xObj.getDescription());
vObj.setTagService(xTagService != null ? xTagService.getName() : null);
vObj.setPolicyVersion(xObj.getPolicyVersion());
+ vObj.setTagVersion(xObj.getTagVersion());
vObj.setPolicyUpdateTime(xObj.getPolicyUpdateTime());
+ vObj.setTagUpdateTime(xObj.getTagUpdateTime());
vObj.setIsEnabled(xObj.getIsenabled());
return vObj;
}