You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/04/09 19:50:50 UTC
[ranger] branch master updated: RANGER-2779: updated ADLS-Gen2
resource mapper for Atlas changes in qualifiedName - #2
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
The following commit(s) were added to refs/heads/master by this push:
new bb05c64 RANGER-2779: updated ADLS-Gen2 resource mapper for Atlas changes in qualifiedName - #2
bb05c64 is described below
commit bb05c645fccb45dcadeaf557fa7dd174413d7af3
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Thu Apr 9 09:07:28 2020 -0700
RANGER-2779: updated ADLS-Gen2 resource mapper for Atlas changes in qualifiedName - #2
---
.../source/atlas/AtlasAdlsResourceMapper.java | 37 +++++++++-------------
.../tagsync/process/TestAdlsResourceMapper.java | 34 +++++++++++++++++---
2 files changed, 44 insertions(+), 27 deletions(-)
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java
index e38f5fa..331a70c 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java
@@ -127,33 +127,26 @@ public class AtlasAdlsResourceMapper extends AtlasResourceMapper {
if (idxClusterNameSep != -1) {
ret[IDX_CLUSTER_NAME] = qualifiedName.substring(idxClusterNameSep + CLUSTER_DELIMITER.length());
- }
-
- int idxProtocolStart = qualifiedName.indexOf(SEP_PROTOCOL);
-
- if (idxProtocolStart != -1) {
- int idxResourceStart = idxProtocolStart + SEP_PROTOCOL.length();
- int idxContainerSep = qualifiedName.indexOf(SEP_CONTAINER, idxResourceStart);
- if (idxContainerSep != -1) {
- if (idxContainerSep == idxClusterNameSep) { // this is adls_gen2_account, so no containerName
- ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxResourceStart, idxContainerSep);
- } else {
- ret[IDX_RESOURCE_CONTAINER] = qualifiedName.substring(idxResourceStart, idxContainerSep);
+ int idxProtocolSep = qualifiedName.indexOf(SEP_PROTOCOL);
- int idxAccountSep = qualifiedName.indexOf(SEP_ACCOUNT, idxContainerSep + SEP_CONTAINER.length());
+ if (idxProtocolSep != -1) {
+ int idxResourceStart = idxProtocolSep + SEP_PROTOCOL.length();
+ int idxContainerAccountSep = qualifiedName.indexOf(SEP_CONTAINER, idxResourceStart);
- if (idxAccountSep != -1) {
- ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxContainerSep + SEP_CONTAINER.length(), idxAccountSep);
+ if (idxContainerAccountSep != -1) {
+ if (idxContainerAccountSep == idxClusterNameSep) { // this is adls_gen2_account, so no containerName
+ ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxResourceStart, idxClusterNameSep);
+ } else {
+ ret[IDX_RESOURCE_CONTAINER] = qualifiedName.substring(idxResourceStart, idxContainerAccountSep);
- int idxRelativePath = qualifiedName.indexOf(SEP_RELATIVE_PATH, idxAccountSep + SEP_ACCOUNT.length());
+ int idxRelativePath = qualifiedName.indexOf(SEP_RELATIVE_PATH, idxContainerAccountSep + SEP_CONTAINER.length());
- if (idxRelativePath != -1) {
- if (idxClusterNameSep == -1) {
- ret[IDX_RESOURCE_RELATIVE_PATH] = qualifiedName.substring(idxRelativePath);
- } else {
- ret[IDX_RESOURCE_RELATIVE_PATH] = qualifiedName.substring(idxRelativePath, idxClusterNameSep);
- }
+ if (idxRelativePath == -1) { // this is adls_gen2_container, so no relativePath
+ ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxContainerAccountSep + SEP_CONTAINER.length(), idxClusterNameSep);
+ } else {
+ ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxContainerAccountSep + SEP_CONTAINER.length(), idxRelativePath);
+ ret[IDX_RESOURCE_RELATIVE_PATH] = qualifiedName.substring(idxRelativePath, idxClusterNameSep);
}
}
}
diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java
index d0b4339..871c9e1 100644
--- a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java
+++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java
@@ -32,12 +32,15 @@ import static org.apache.ranger.tagsync.source.atlas.AtlasResourceMapper.ENTITY_
public class TestAdlsResourceMapper {
- private static final String ACCOUNT_QUALIFIED_NAME = "abfs://myaccount@cl1";
- private static final String CONTAINER_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net@cl1";
- private static final String RELATIVE_PATH_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net/tmp@cl1";
+ private static final String ACCOUNT_QUALIFIED_NAME = "abfs://myaccount@cl1";
+ private static final String CONTAINER_QUALIFIED_NAME = "abfs://mycontainer@myaccount@cl1";
+ private static final String RELATIVE_PATH_QUALIFIED_NAME = "abfs://mycontainer@myaccount/tmp@cl1";
+ private static final String CONTAINER2_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net@cl1";
+ private static final String RELATIVE_PATH2_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net/tmp@cl1";
private static final String SERVICE_NAME = "cl1_adls";
private static final String ACCOUNT_NAME = "myaccount";
+ private static final String ACCOUNT2_NAME = "myaccount.dfs.core.windows.net";
private static final String CONTAINER_NAME = "mycontainer";
private static final String RELATIVE_PATH_NAME = "/tmp";
@@ -77,6 +80,29 @@ public class TestAdlsResourceMapper {
}
@Test
+ public void testContainer2Entity() throws Exception {
+ RangerAtlasEntity entity = getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, CONTAINER2_QUALIFIED_NAME);
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
+ assertResourceElementCount(resource, 2);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT2_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_CONTAINER, CONTAINER_NAME);
+ }
+
+ @Test
+ public void testDirectory2Entity() throws Exception {
+ RangerAtlasEntity entity = getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, RELATIVE_PATH2_QUALIFIED_NAME);
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
+ assertResourceElementCount(resource, 3);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT2_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_CONTAINER, CONTAINER_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_RELATIVE_PATH, RELATIVE_PATH_NAME);
+ }
+
+ @Test
public void testInvalidEntityType() {
assertException(getEntity("Unknown", RELATIVE_PATH_QUALIFIED_NAME), "unrecognized entity-type");
}
@@ -96,7 +122,6 @@ public class TestAdlsResourceMapper {
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "test"), "cluster-name not found");
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "test@cl1"), "account-name not found");
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "abfs://test@cl1"), "container-name not found");
- assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "abfs://a@test@cl1"), "account-name not found");
}
@Test
@@ -106,7 +131,6 @@ public class TestAdlsResourceMapper {
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "test"), "cluster-name not found");
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "test@cl1"), "account-name not found");
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://test@cl1"), "container-name not found");
- assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://a@test@cl1"), "account-name not found");
assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://a@test.dfs.core.windows.net@cl1"), "relative-path not found");
}