You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ni...@apache.org on 2020/07/01 09:24:42 UTC
[atlas] branch master updated (8b50ac0 -> 337575c)
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from 8b50ac0 ATLAS-3836 Add Apache Ozone support in hive hook
new f932e52 ATLAS-3867 : Relationship search API should have a provision to fetch custom attributes in search results
new e72aa10 ATLAS-3871 Add unit tests to path extractor utility for s3, s3 v2, adls path entities
new 337575c ATLAS-3869 Upgrade httpclient from 4.5.3 to 4.5.4
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../atlas/utils/AtlasPathExtractorUtilTest.java | 285 ++++++++++++++++++---
pom.xml | 2 +-
.../atlas/discovery/AtlasDiscoveryService.java | 5 +-
.../atlas/discovery/EntityDiscoveryService.java | 4 +-
.../org/apache/atlas/web/rest/DiscoveryREST.java | 19 +-
5 files changed, 269 insertions(+), 46 deletions(-)
[atlas] 03/03: ATLAS-3869 Upgrade httpclient from 4.5.3 to 4.5.4
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 337575c81b993e2cdcd6dc462dcf614940c783c2
Author: nixonrodrigues <ni...@apache.org>
AuthorDate: Tue Jun 30 18:06:07 2020 +0530
ATLAS-3869 Upgrade httpclient from 4.5.3 to 4.5.4
Signed-off-by: nixonrodrigues <ni...@apache.org>
---
pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/pom.xml b/pom.xml
index 115e0e1..5e0442a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1574,7 +1574,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
- <version>4.5.3</version>
+ <version>4.5.4</version>
</dependency>
<dependency>
[atlas] 01/03: ATLAS-3867 : Relationship search API should have a
provision to fetch custom attributes in search results
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit f932e52c687479c77b9967165353b0fffa6f2fa0
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Wed Jul 1 10:53:18 2020 +0530
ATLAS-3867 : Relationship search API should have a provision to fetch custom attributes in search results
Signed-off-by: nixonrodrigues <ni...@apache.org>
---
.../apache/atlas/discovery/AtlasDiscoveryService.java | 5 +++--
.../atlas/discovery/EntityDiscoveryService.java | 4 ++--
.../java/org/apache/atlas/web/rest/DiscoveryREST.java | 19 +++++++++++--------
3 files changed, 16 insertions(+), 12 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index e64c315..8657259 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -19,13 +19,13 @@
package org.apache.atlas.discovery;
-import com.sun.xml.bind.v2.model.annotation.Quick;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.*;
import org.apache.atlas.model.profile.AtlasUserSavedSearch;
import java.util.List;
+import java.util.Set;
public interface AtlasDiscoveryService {
/**
@@ -83,6 +83,7 @@ public interface AtlasDiscoveryService {
*
* @param guid unique ID of the entity.
* @param relation relation name.
+ * @param attributes set of attributes in search result.
* @param sortByAttribute sort the result using this attribute name, default value is 'name'
* @param sortOrder sorting order
* @param excludeDeletedEntities exclude deleted entities in search result.
@@ -90,7 +91,7 @@ public interface AtlasDiscoveryService {
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult
*/
- AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
+ AtlasSearchResult searchRelatedEntities(String guid, String relation, Set<String> attributes, String sortByAttribute, SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
/**
*
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 4169e14..4b95642 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -563,7 +563,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortBy, SortOrder sortOrder,
+ public AtlasSearchResult searchRelatedEntities(String guid, String relation, Set<String> attributes, String sortBy, SortOrder sortOrder,
boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
@@ -685,7 +685,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
List<AtlasEntityHeader> resultList = new ArrayList<>(vertices.size());
for (AtlasVertex vertex : vertices) {
- resultList.add(entityRetriever.toAtlasEntityHeader(vertex));
+ resultList.add(entityRetriever.toAtlasEntityHeader(vertex, attributes));
}
ret.setEntities(resultList);
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 076284e..140ed9a 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -55,6 +55,7 @@ import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import java.io.IOException;
import java.util.List;
+import java.util.Set;
/**
* REST interface for data discovery using dsl or full text search
@@ -344,6 +345,7 @@ public class DiscoveryREST {
*
* @param guid Attribute name
* @param relation relationName
+ * @param attributes set of attributes in search result.
* @param sortByAttribute sort the result using this attribute name, default value is 'name'
* @param sortOrder sorting order
* @param limit limit the result set to only include the specified number of entries
@@ -355,13 +357,14 @@ public class DiscoveryREST {
*/
@GET
@Path("relationship")
- public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid,
- @QueryParam("relation") String relation,
- @QueryParam("sortBy") String sortByAttribute,
- @QueryParam("sortOrder") SortOrder sortOrder,
- @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
- @QueryParam("limit") int limit,
- @QueryParam("offset") int offset) throws AtlasBaseException {
+ public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid,
+ @QueryParam("relation") String relation,
+ @QueryParam("attributes") Set<String> attributes,
+ @QueryParam("sortBy") String sortByAttribute,
+ @QueryParam("sortOrder") SortOrder sortOrder,
+ @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
+ @QueryParam("limit") int limit,
+ @QueryParam("offset") int offset) throws AtlasBaseException {
Servlets.validateQueryParamLength("guid", guid);
Servlets.validateQueryParamLength("relation", relation);
Servlets.validateQueryParamLength("sortBy", sortByAttribute);
@@ -374,7 +377,7 @@ public class DiscoveryREST {
", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")");
}
- return discoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
+ return discoveryService.searchRelatedEntities(guid, relation, attributes, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
} finally {
AtlasPerfTracer.log(perf);
}
[atlas] 02/03: ATLAS-3871 Add unit tests to path extractor utility
for s3, s3 v2, adls path entities
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit e72aa1076a5a1272935fed4dedaf9e29e41fa30b
Author: Nikhil Bonte <ni...@freestoneinfotech.com>
AuthorDate: Tue Jun 30 18:15:24 2020 +0530
ATLAS-3871 Add unit tests to path extractor utility for s3, s3 v2, adls path entities
Signed-off-by: nixonrodrigues <ni...@apache.org>
---
.../atlas/utils/AtlasPathExtractorUtilTest.java | 285 ++++++++++++++++++---
1 file changed, 252 insertions(+), 33 deletions(-)
diff --git a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
index 664bfb7..4abdca4 100644
--- a/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
+++ b/common/src/test/java/org/apache/atlas/utils/AtlasPathExtractorUtilTest.java
@@ -22,6 +22,7 @@ import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import org.apache.hadoop.fs.Path;
@@ -53,56 +54,68 @@ public class AtlasPathExtractorUtilTest {
private static final String OZONE_KEY = "ozone_key";
private static final String OZONE_SCHEME = "ofs" + SCHEME_SEPARATOR;
private static final String OZONE_3_SCHEME = "o3fs" + SCHEME_SEPARATOR;
- private static final String OZONE_PATH = OZONE_SCHEME + "bucket1.volume1.ozone1/files/file.txt";
- private static final String OZONE_3_PATH = OZONE_3_SCHEME + "bucket1.volume1.ozone1/files/file.txt";
// HDFS
private static final String HDFS_SCHEME = "hdfs" + SCHEME_SEPARATOR;
private static final String HDFS_PATH = HDFS_SCHEME + "host_name:8020/warehouse/tablespace/external/hive/taBlE_306";
- @Test
- public void testGetPathEntityOzone3Path() {
- PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
-
- Path path = new Path(OZONE_3_PATH);
- AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
- AtlasEntity entity = entityWithExtInfo.getEntity();
-
- assertNotNull(entity);
- assertEquals(entity.getTypeName(), OZONE_KEY);
- verifyOzoneKeyEntity(OZONE_3_PATH, entity);
-
- assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
- verifyOzoneEntities(OZONE_3_SCHEME, OZONE_3_PATH, extractorContext.getKnownEntities());
-
- assertEquals(extractorContext.getKnownEntities().size(), 3);
- verifyOzoneEntities(OZONE_3_SCHEME, OZONE_3_PATH, extractorContext.getKnownEntities());
+ // ADLS Gen2
+ private static final String ADLS_GEN2_ACCOUNT = "adls_gen2_account";
+ private static final String ADLS_GEN2_CONTAINER = "adls_gen2_container";
+ private static final String ADLS_GEN2_DIRECTORY = "adls_gen2_directory";
+ private static final String ABFS_SCHEME = "abfs" + SCHEME_SEPARATOR;
+ private static final String ABFSS_SCHEME = "abfss" + SCHEME_SEPARATOR;
+ private static final String ABFS_PATH = ABFS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv";
+ private static final String ABFSS_PATH = ABFSS_SCHEME + "data@razrangersan.dfs.core.windows.net/tmp/cdp-demo/sample.csv";
+
+ // AWS S3
+ private static final String AWS_S3_ATLAS_MODEL_VERSION_V2 = "V2";
+ private static final String AWS_S3_BUCKET = "aws_s3_bucket";
+ private static final String AWS_S3_PSEUDO_DIR = "aws_s3_pseudo_dir";
+ private static final String AWS_S3_V2_BUCKET = "aws_s3_v2_bucket";
+ private static final String AWS_S3_V2_PSEUDO_DIR = "aws_s3_v2_directory";
+ private static final String S3_SCHEME = "s3" + SCHEME_SEPARATOR;
+ private static final String S3A_SCHEME = "s3a" + SCHEME_SEPARATOR;
+ private static final String ATTRIBUTE_OBJECT_PREFIX = "objectPrefix";
+ private static final String S3_PATH = S3_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv";
+ private static final String S3A_PATH = S3A_SCHEME + "aws_my_bucket1/1234567890/renders/Irradiance_A.csv";
+
+ @DataProvider(name = "ozonePathProvider")
+ private Object[][] ozonePathProvider(){
+ return new Object[][]{
+ { OZONE_SCHEME, "bucket1.volume1.ozone1/files/file.txt", "/files/file.txt" },
+ { OZONE_SCHEME, "bucket1.volume1.ozone1/file21.txt", "/file21.txt" },
+ { OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", "/quarter_one/sales" },
+ { OZONE_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", "/quarter_one/sales" },
+ { OZONE_3_SCHEME, "bucket1.volume1.ozone1/files/file.txt", "/files/file.txt" },
+ { OZONE_3_SCHEME, "bucket1.volume1.ozone1/file21.txt", "/file21.txt"},
+ { OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales", "/quarter_one/sales" },
+ { OZONE_3_SCHEME, "bucket1.volume1.ozone1/quarter_one/sales/", "/quarter_one/sales" },
+ };
}
- @Test
- public void testGetPathEntityOzonePath() {
+ @Test(dataProvider = "ozonePathProvider")
+ public void testGetPathEntityOzone3Path(String scheme, String location, String keyName) {
+ String ozonePath = scheme + location;
PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
- Path path = new Path(OZONE_PATH);
+ Path path = new Path(ozonePath);
AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
AtlasEntity entity = entityWithExtInfo.getEntity();
assertNotNull(entity);
assertEquals(entity.getTypeName(), OZONE_KEY);
- verifyOzoneKeyEntity(OZONE_PATH, entity);
+ verifyOzoneKeyEntity(ozonePath, keyName, entity);
assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
- verifyOzoneEntities(OZONE_SCHEME, OZONE_PATH, extractorContext.getKnownEntities());
+ verifyOzoneEntities(scheme, ozonePath, keyName, entityWithExtInfo.getReferredEntities());
assertEquals(extractorContext.getKnownEntities().size(), 3);
- verifyOzoneEntities(OZONE_SCHEME, OZONE_PATH, extractorContext.getKnownEntities());
+ verifyOzoneEntities(scheme, ozonePath, keyName, extractorContext.getKnownEntities());
}
@Test
public void testGetPathEntityHdfsPath() {
- Map<String, AtlasEntity> knownEntities = new HashMap<>();
- AtlasEntityWithExtInfo extInfo = new AtlasEntityWithExtInfo();
-
PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
Path path = new Path(HDFS_PATH);
@@ -113,7 +126,7 @@ public class AtlasPathExtractorUtilTest {
assertEquals(entity.getTypeName(), HDFS_PATH_TYPE);
verifyHDFSEntity(entity, false);
- assertNull(extInfo.getReferredEntities());
+ assertNull(entityWithExtInfo.getReferredEntities());
assertEquals(extractorContext.getKnownEntities().size(), 1);
extractorContext.getKnownEntities().values().forEach(x -> verifyHDFSEntity(x, false));
}
@@ -135,11 +148,107 @@ public class AtlasPathExtractorUtilTest {
extractorContext.getKnownEntities().values().forEach(x -> verifyHDFSEntity(x, true));
}
- private void verifyOzoneEntities(String scheme, String path, Map<String, AtlasEntity> knownEntities) {
+ @Test
+ public void testGetPathEntityABFSPath() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
+
+ Path path = new Path(ABFS_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), ADLS_GEN2_DIRECTORY);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
+
+ verifyABFSAdlsGen2Dir(ABFS_SCHEME, ABFS_PATH, entity);
+ verifyABFSKnownEntities(ABFS_SCHEME, ABFS_PATH, extractorContext.getKnownEntities());
+ }
+
+ @Test
+ public void testGetPathEntityABFSSPath() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
+
+ Path path = new Path(ABFSS_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), ADLS_GEN2_DIRECTORY);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 2);
+
+ verifyABFSAdlsGen2Dir(ABFSS_SCHEME, ABFSS_PATH, entity);
+ verifyABFSKnownEntities(ABFSS_SCHEME, ABFSS_PATH, extractorContext.getKnownEntities());
+ }
+
+ @Test
+ public void testGetPathEntityS3V2Path() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE, AWS_S3_ATLAS_MODEL_VERSION_V2);
+
+ Path path = new Path(S3_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), AWS_S3_V2_PSEUDO_DIR);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 1);
+
+ verifyS3V2PseudoDir(S3A_SCHEME, S3_PATH, entity);
+ verifyS3V2KnownEntities(S3_SCHEME, S3_PATH, extractorContext.getKnownEntities());
+ }
+
+ @Test
+ public void testGetPathEntityS3AV2Path() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE, AWS_S3_ATLAS_MODEL_VERSION_V2);
+
+ Path path = new Path(S3A_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), AWS_S3_V2_PSEUDO_DIR);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 1);
+
+ verifyS3V2PseudoDir(S3A_SCHEME, S3A_PATH, entity);
+ verifyS3V2KnownEntities(S3A_SCHEME, S3A_PATH, extractorContext.getKnownEntities());
+ }
+
+ @Test
+ public void testGetPathEntityS3Path() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
+
+ Path path = new Path(S3_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), AWS_S3_PSEUDO_DIR);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 1);
+
+ verifyS3PseudoDir(S3_PATH, entity);
+ verifyS3KnownEntities(S3_SCHEME, S3_PATH, extractorContext.getKnownEntities());
+ }
+
+ @Test
+ public void testGetPathEntityS3APath() {
+ PathExtractorContext extractorContext = new PathExtractorContext(METADATA_NAMESPACE);
+
+ Path path = new Path(S3A_PATH);
+ AtlasEntityWithExtInfo entityWithExtInfo = AtlasPathExtractorUtil.getPathEntity(path, extractorContext);
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+
+ assertNotNull(entity);
+ assertEquals(entity.getTypeName(), AWS_S3_PSEUDO_DIR);
+ assertEquals(entityWithExtInfo.getReferredEntities().size(), 1);
+
+ verifyS3PseudoDir(S3A_PATH, entity);
+ verifyS3KnownEntities(S3A_SCHEME, S3A_PATH, extractorContext.getKnownEntities());
+ }
+
+ private void verifyOzoneEntities(String scheme, String path, String keyName, Map<String, AtlasEntity> knownEntities) {
for (AtlasEntity knownEntity : knownEntities.values()) {
switch (knownEntity.getTypeName()){
case OZONE_KEY:
- verifyOzoneKeyEntity(path, knownEntity);
+ verifyOzoneKeyEntity(path, keyName, knownEntity);
break;
case OZONE_VOLUME:
@@ -155,9 +264,11 @@ public class AtlasPathExtractorUtilTest {
}
}
- private void verifyOzoneKeyEntity(String path, AtlasEntity entity) {
+ private void verifyOzoneKeyEntity(String path, String name, AtlasEntity entity) {
+ //remove trailing "/" if present from path
+ path = (path.charAt(path.length()-1) == '/') ? path.substring(0, path.length()-1) : path;
assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), path + QNAME_METADATA_NAMESPACE);
- assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "/files/file.txt");
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), name);
}
private void verifyHDFSEntity(AtlasEntity entity, boolean toLowerCase) {
@@ -173,4 +284,112 @@ public class AtlasPathExtractorUtilTest {
assertEquals(entity.getAttribute(ATTRIBUTE_CLUSTER_NAME), METADATA_NAMESPACE);
}
}
+
+ private void verifyABFSAdlsGen2Dir(String abfsScheme, String path, AtlasEntity entity){
+ String pathQName = abfsScheme + "data@razrangersan/tmp/cdp-demo/sample.csv/" + QNAME_METADATA_NAMESPACE;
+ String entityQName = (String) entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME);
+
+ if (pathQName.equalsIgnoreCase(entityQName)){
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "sample.csv");
+ } else {
+ pathQName = abfsScheme + "data@razrangersan/tmp/cdp-demo/" + QNAME_METADATA_NAMESPACE;
+ if (pathQName.equalsIgnoreCase(entityQName)){
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "cdp-demo");
+ } else {
+ assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), abfsScheme + "data@razrangersan/tmp/" + QNAME_METADATA_NAMESPACE);
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "tmp");
+ }
+ }
+ }
+
+ private void verifyABFSKnownEntities(String scheme, String path, Map<String, AtlasEntity> knownEntities) {
+ assertEquals(knownEntities.size(), 5);
+ int directoryCount = 0;
+ for (AtlasEntity knownEntity : knownEntities.values()) {
+ switch (knownEntity.getTypeName()){
+ case ADLS_GEN2_DIRECTORY:
+ verifyABFSAdlsGen2Dir(scheme, path, knownEntity);
+ directoryCount++;
+ break;
+
+ case ADLS_GEN2_CONTAINER:
+ assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "data@razrangersan" + QNAME_METADATA_NAMESPACE);
+ assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "data");
+ break;
+
+ case ADLS_GEN2_ACCOUNT:
+ assertEquals(knownEntity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "razrangersan" + QNAME_METADATA_NAMESPACE);
+ assertEquals(knownEntity.getAttribute(ATTRIBUTE_NAME), "razrangersan");
+ break;
+ }
+ }
+ assertEquals(directoryCount, 3);
+ }
+
+ private void verifyS3V2PseudoDir(String s3Scheme, String path, AtlasEntity entity){
+ String pathQName = path + "/" + QNAME_METADATA_NAMESPACE;
+ String entityQName = (String) entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME);
+
+ if (pathQName.equalsIgnoreCase(entityQName)){
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "Irradiance_A.csv");
+ assertEquals(entity.getAttribute(ATTRIBUTE_OBJECT_PREFIX), "/1234567890/renders/Irradiance_A.csv/");
+ } else {
+ pathQName = s3Scheme + "aws_my_bucket1/1234567890/" + QNAME_METADATA_NAMESPACE;
+ if (pathQName.equalsIgnoreCase(entityQName)){
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "1234567890");
+ assertEquals(entity.getAttribute(ATTRIBUTE_OBJECT_PREFIX), "/1234567890/");
+ } else {
+ assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), s3Scheme + "aws_my_bucket1/1234567890/renders/" + QNAME_METADATA_NAMESPACE);
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "renders");
+ assertEquals(entity.getAttribute(ATTRIBUTE_OBJECT_PREFIX), "/1234567890/renders/");
+ }
+ }
+ }
+
+ private void verifyS3V2KnownEntities(String scheme, String path, Map<String, AtlasEntity> knownEntities) {
+ assertEquals(knownEntities.size(), 4);
+ int dirCount = 0;
+ for (AtlasEntity knownEntity : knownEntities.values()) {
+ switch (knownEntity.getTypeName()){
+ case AWS_S3_V2_PSEUDO_DIR:
+ verifyS3V2PseudoDir(scheme, path, knownEntity);
+ dirCount++;
+ break;
+
+ case AWS_S3_V2_BUCKET:
+ verifyS3BucketEntity(scheme, knownEntity);
+ break;
+ }
+ }
+ assertEquals(dirCount, 3);
+ }
+
+ private void verifyS3PseudoDir(String path, AtlasEntity entity){
+ assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), path.toLowerCase() + QNAME_METADATA_NAMESPACE);
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "/1234567890/renders/irradiance_a.csv");
+ assertEquals(entity.getAttribute(ATTRIBUTE_OBJECT_PREFIX), "/1234567890/renders/irradiance_a.csv");
+ }
+
+ private void verifyS3KnownEntities(String scheme, String path, Map<String, AtlasEntity> knownEntities) {
+ assertEquals(knownEntities.size(), 2);
+ int dirCount = 0;
+ for (AtlasEntity knownEntity : knownEntities.values()) {
+ switch (knownEntity.getTypeName()){
+ case AWS_S3_PSEUDO_DIR:
+ verifyS3PseudoDir(path, knownEntity);
+ dirCount++;
+ break;
+
+ case AWS_S3_BUCKET:
+ verifyS3BucketEntity(scheme, knownEntity);
+ break;
+ }
+ }
+ assertEquals(dirCount, 1);
+ }
+
+ private void verifyS3BucketEntity(String scheme, AtlasEntity entity) {
+ assertEquals(entity.getAttribute(ATTRIBUTE_QUALIFIED_NAME), scheme + "aws_my_bucket1" + QNAME_METADATA_NAMESPACE);
+ assertEquals(entity.getAttribute(ATTRIBUTE_NAME), "aws_my_bucket1");
+ }
}