You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ranger.apache.org by ma...@apache.org on 2020/04/06 17:03:19 UTC
[ranger] 01/02: RANGER-2779: updated tag-sync to process Atlas
notifications for ADLS-Gen2 entities
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ranger.git
commit f0797138f6bdf9432400d1c2743b5ea7351a0c8d
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Sat Apr 4 17:31:31 2020 -0700
RANGER-2779: updated tag-sync to process Atlas notifications for ADLS-Gen2 entities
---
.../source/atlas/AtlasAdlsResourceMapper.java | 166 +++++++++++++++++++++
.../source/atlas/AtlasResourceMapperUtil.java | 2 +
.../tagsync/process/TestAdlsResourceMapper.java | 140 +++++++++++++++++
3 files changed, 308 insertions(+)
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java
new file mode 100644
index 0000000..e38f5fa
--- /dev/null
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasAdlsResourceMapper.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.tagsync.source.atlas;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.ranger.plugin.model.RangerPolicy.RangerPolicyResource;
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.tagsync.source.atlasrest.RangerAtlasEntity;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class AtlasAdlsResourceMapper extends AtlasResourceMapper {
+ public static final String ENTITY_TYPE_ADLS_GEN2_ACCOUNT = "adls_gen2_account";
+ public static final String ENTITY_TYPE_ADLS_GEN2_CONTAINER = "adls_gen2_container";
+ public static final String ENTITY_TYPE_ADLS_GEN2_DIRECTORY = "adls_gen2_directory";
+
+ public static final String RANGER_TYPE_ADLS_GEN2_ACCOUNT = "storageaccount";
+ public static final String RANGER_TYPE_ADLS_GEN2_CONTAINER = "container";
+ public static final String RANGER_TYPE_ADLS_GEN2_RELATIVE_PATH = "relativepath";
+
+ public static final String[] SUPPORTED_ENTITY_TYPES = { ENTITY_TYPE_ADLS_GEN2_ACCOUNT, ENTITY_TYPE_ADLS_GEN2_CONTAINER, ENTITY_TYPE_ADLS_GEN2_DIRECTORY };
+
+ private static final String SEP_PROTOCOL = "://";
+ private static final String SEP_CONTAINER = "@";
+ private static final String SEP_ACCOUNT = ".";
+ private static final String SEP_RELATIVE_PATH = "/";
+ private static final int IDX_RESOURCE_ACCOUNT = 0;
+ private static final int IDX_RESOURCE_CONTAINER = 1;
+ private static final int IDX_RESOURCE_RELATIVE_PATH = 2;
+ private static final int IDX_CLUSTER_NAME = 3;
+ private static final int RESOURCE_COUNT = 4;
+
+
+ public AtlasAdlsResourceMapper() {
+ super("adls", SUPPORTED_ENTITY_TYPES);
+ }
+
+ @Override
+ public RangerServiceResource buildResource(final RangerAtlasEntity entity) throws Exception {
+ String qualifiedName = (String)entity.getAttributes().get(AtlasResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME);
+
+ if (StringUtils.isEmpty(qualifiedName)) {
+ throw new Exception("attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "' not found in entity");
+ }
+
+ String[] resources = parseQualifiedName(qualifiedName);
+ String clusterName = resources[IDX_CLUSTER_NAME];
+ String accountName = resources[IDX_RESOURCE_ACCOUNT];
+
+ if (StringUtils.isEmpty(clusterName)) {
+ throwExceptionWithMessage("cluster-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName);
+ }
+
+ if (StringUtils.isEmpty(accountName)) {
+ throwExceptionWithMessage("account-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName);
+ }
+
+ String entityType = entity.getTypeName();
+ String entityGuid = entity.getGuid();
+ String serviceName = getRangerServiceName(clusterName);
+
+ Map<String, RangerPolicyResource> elements = new HashMap<String, RangerPolicyResource>();
+
+ if (StringUtils.equals(entityType, ENTITY_TYPE_ADLS_GEN2_ACCOUNT)) {
+ elements.put(RANGER_TYPE_ADLS_GEN2_ACCOUNT, new RangerPolicyResource(accountName));
+ } else if (StringUtils.equals(entityType, ENTITY_TYPE_ADLS_GEN2_CONTAINER)) {
+ String containerName = resources[IDX_RESOURCE_CONTAINER];
+
+ if (StringUtils.isEmpty(containerName)) {
+ throwExceptionWithMessage("container-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName);
+ }
+
+ elements.put(RANGER_TYPE_ADLS_GEN2_ACCOUNT, new RangerPolicyResource(accountName));
+ elements.put(RANGER_TYPE_ADLS_GEN2_CONTAINER, new RangerPolicyResource(containerName));
+ } else if (StringUtils.equals(entityType, ENTITY_TYPE_ADLS_GEN2_DIRECTORY)) {
+ String containerName = resources[IDX_RESOURCE_CONTAINER];
+ String relativePath = resources[IDX_RESOURCE_RELATIVE_PATH];
+
+ if (StringUtils.isEmpty(containerName)) {
+ throwExceptionWithMessage("container-name not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName);
+ }
+
+ if (StringUtils.isEmpty(relativePath)) {
+ throwExceptionWithMessage("relative-path not found in attribute '" + ENTITY_ATTRIBUTE_QUALIFIED_NAME + "': " + qualifiedName);
+ }
+
+ elements.put(RANGER_TYPE_ADLS_GEN2_ACCOUNT, new RangerPolicyResource(accountName));
+ elements.put(RANGER_TYPE_ADLS_GEN2_CONTAINER, new RangerPolicyResource(containerName));
+ elements.put(RANGER_TYPE_ADLS_GEN2_RELATIVE_PATH, new RangerPolicyResource(relativePath));
+ } else {
+ throwExceptionWithMessage("unrecognized entity-type: " + entityType);
+ }
+
+ RangerServiceResource ret = new RangerServiceResource(entityGuid, serviceName, elements);
+
+ return ret;
+ }
+
+ /* qualifiedName can be of format, depending upon the entity-type:
+ adls_gen2_account: abfs://<accountName>@<clusterName>
+ adls_gen2_container: abfs://<containerName>@<accountName>.dfs.core.windows.net@<clusterName>
+ adls_gen2_directory: abfs://<containerName>@<accountName>.dfs.core.windows.net/<relativePath>@<clusterName>
+ */
+ private String[] parseQualifiedName(String qualifiedName) {
+ String[] ret = new String[RESOURCE_COUNT];
+
+ if(StringUtils.isNotBlank(qualifiedName)) {
+ int idxClusterNameSep = qualifiedName.lastIndexOf(CLUSTER_DELIMITER);
+
+ if (idxClusterNameSep != -1) {
+ ret[IDX_CLUSTER_NAME] = qualifiedName.substring(idxClusterNameSep + CLUSTER_DELIMITER.length());
+ }
+
+ int idxProtocolStart = qualifiedName.indexOf(SEP_PROTOCOL);
+
+ if (idxProtocolStart != -1) {
+ int idxResourceStart = idxProtocolStart + SEP_PROTOCOL.length();
+ int idxContainerSep = qualifiedName.indexOf(SEP_CONTAINER, idxResourceStart);
+
+ if (idxContainerSep != -1) {
+ if (idxContainerSep == idxClusterNameSep) { // this is adls_gen2_account, so no containerName
+ ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxResourceStart, idxContainerSep);
+ } else {
+ ret[IDX_RESOURCE_CONTAINER] = qualifiedName.substring(idxResourceStart, idxContainerSep);
+
+ int idxAccountSep = qualifiedName.indexOf(SEP_ACCOUNT, idxContainerSep + SEP_CONTAINER.length());
+
+ if (idxAccountSep != -1) {
+ ret[IDX_RESOURCE_ACCOUNT] = qualifiedName.substring(idxContainerSep + SEP_CONTAINER.length(), idxAccountSep);
+
+ int idxRelativePath = qualifiedName.indexOf(SEP_RELATIVE_PATH, idxAccountSep + SEP_ACCOUNT.length());
+
+ if (idxRelativePath != -1) {
+ if (idxClusterNameSep == -1) {
+ ret[IDX_RESOURCE_RELATIVE_PATH] = qualifiedName.substring(idxRelativePath);
+ } else {
+ ret[IDX_RESOURCE_RELATIVE_PATH] = qualifiedName.substring(idxRelativePath, idxClusterNameSep);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return ret;
+ }
+}
diff --git a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java
index cd2cb63..999c206 100644
--- a/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java
+++ b/tagsync/src/main/java/org/apache/ranger/tagsync/source/atlas/AtlasResourceMapperUtil.java
@@ -90,6 +90,8 @@ public class AtlasResourceMapperUtil {
mapperNames.add("org.apache.ranger.tagsync.source.atlas.AtlasHbaseResourceMapper");
mapperNames.add("org.apache.ranger.tagsync.source.atlas.AtlasKafkaResourceMapper");
+ mapperNames.add(AtlasAdlsResourceMapper.class.getName());
+
if (StringUtils.isNotBlank(customMapperNames)) {
for (String customMapperName : customMapperNames.split(MAPPER_NAME_DELIMITER)) {
mapperNames.add(customMapperName.trim());
diff --git a/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java
new file mode 100644
index 0000000..d0b4339
--- /dev/null
+++ b/tagsync/src/test/java/org/apache/ranger/tagsync/process/TestAdlsResourceMapper.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.ranger.tagsync.process;
+
+import org.apache.ranger.plugin.model.RangerServiceResource;
+import org.apache.ranger.tagsync.source.atlas.AtlasAdlsResourceMapper;
+import org.apache.ranger.tagsync.source.atlasrest.RangerAtlasEntity;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Collections;
+
+import static org.apache.ranger.tagsync.source.atlas.AtlasAdlsResourceMapper.*;
+import static org.apache.ranger.tagsync.source.atlas.AtlasResourceMapper.ENTITY_ATTRIBUTE_QUALIFIED_NAME;
+
+
+public class TestAdlsResourceMapper {
+ private static final String ACCOUNT_QUALIFIED_NAME = "abfs://myaccount@cl1";
+ private static final String CONTAINER_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net@cl1";
+ private static final String RELATIVE_PATH_QUALIFIED_NAME = "abfs://mycontainer@myaccount.dfs.core.windows.net/tmp@cl1";
+
+ private static final String SERVICE_NAME = "cl1_adls";
+ private static final String ACCOUNT_NAME = "myaccount";
+ private static final String CONTAINER_NAME = "mycontainer";
+ private static final String RELATIVE_PATH_NAME = "/tmp";
+
+ AtlasAdlsResourceMapper resourceMapper = new AtlasAdlsResourceMapper();
+
+ @Test
+ public void testAccountEntity() throws Exception {
+ RangerAtlasEntity entity = getEntity(ENTITY_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT_QUALIFIED_NAME);
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
+ assertResourceElementCount(resource, 1);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT_NAME);
+ }
+
+ @Test
+ public void testContainerEntity() throws Exception {
+ RangerAtlasEntity entity = getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, CONTAINER_QUALIFIED_NAME);
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
+ assertResourceElementCount(resource, 2);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_CONTAINER, CONTAINER_NAME);
+ }
+
+ @Test
+ public void testDirectoryEntity() throws Exception {
+ RangerAtlasEntity entity = getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, RELATIVE_PATH_QUALIFIED_NAME);
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertEquals(SERVICE_NAME, resource.getServiceName());
+ assertResourceElementCount(resource, 3);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_ACCOUNT, ACCOUNT_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_CONTAINER, CONTAINER_NAME);
+ assertResourceElementValue(resource, RANGER_TYPE_ADLS_GEN2_RELATIVE_PATH, RELATIVE_PATH_NAME);
+ }
+
+ @Test
+ public void testInvalidEntityType() {
+ assertException(getEntity("Unknown", RELATIVE_PATH_QUALIFIED_NAME), "unrecognized entity-type");
+ }
+
+ @Test
+ public void testInvalidAccountEntity() {
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_ACCOUNT, null), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_ACCOUNT, ""), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_ACCOUNT, "test"), "cluster-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_ACCOUNT, "test@cl1"), "account-name not found");
+ }
+
+ @Test
+ public void testInvalidContainerEntity() {
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, null), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, ""), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "test"), "cluster-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "test@cl1"), "account-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "abfs://test@cl1"), "container-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_CONTAINER, "abfs://a@test@cl1"), "account-name not found");
+ }
+
+ @Test
+ public void testInvalidDirectoryEntity() {
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, null), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, ""), "attribute 'qualifiedName' not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "test"), "cluster-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "test@cl1"), "account-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://test@cl1"), "container-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://a@test@cl1"), "account-name not found");
+ assertException(getEntity(ENTITY_TYPE_ADLS_GEN2_DIRECTORY, "abfs://a@test.dfs.core.windows.net@cl1"), "relative-path not found");
+ }
+
+ private RangerAtlasEntity getEntity(String entityType, String qualifiedName) {
+ return new RangerAtlasEntity(entityType, "guid-" + entityType, Collections.singletonMap(ENTITY_ATTRIBUTE_QUALIFIED_NAME, qualifiedName));
+ }
+
+ private void assertResourceElementCount(RangerServiceResource resource, int count) {
+ Assert.assertNotNull(resource);
+ Assert.assertNotNull(resource.getResourceElements());
+ Assert.assertEquals(count, resource.getResourceElements().size());
+ }
+
+ private void assertResourceElementValue(RangerServiceResource resource, String resourceName, String value) {
+ Assert.assertTrue(resource.getResourceElements().containsKey(resourceName));
+ Assert.assertNotNull(resource.getResourceElements().get(resourceName).getValues());
+ Assert.assertEquals(1, resource.getResourceElements().get(resourceName).getValues().size());
+ Assert.assertEquals(value, resource.getResourceElements().get(resourceName).getValues().get(0));
+ }
+
+ private void assertException(RangerAtlasEntity entity, String exceptionMessage) {
+ try {
+ RangerServiceResource resource = resourceMapper.buildResource(entity);
+
+ Assert.assertFalse("Expected buildResource() to fail. But it returned " + resource, true);
+ } catch (Exception excp) {
+ Assert.assertTrue("Unexpected exception message: expected=" + exceptionMessage + "; found " + excp.getMessage(),
+ excp.getMessage().startsWith(exceptionMessage));
+ }
+ }
+}