You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/04/25 17:51:47 UTC
atlas git commit: ATLAS-2602: Walkthrough includes relationship
attribute.
Repository: atlas
Updated Branches:
refs/heads/master 540129f5c -> 65f7da17e
ATLAS-2602: Walkthrough includes relationship attribute.
Signed-off-by: Ashutosh Mestry <am...@hortonworks.com>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/65f7da17
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/65f7da17
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/65f7da17
Branch: refs/heads/master
Commit: 65f7da17e444dff86ecbcbcab17251b66c067aa2
Parents: 540129f
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Tue Apr 24 15:15:20 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Wed Apr 25 10:50:20 2018 -0700
----------------------------------------------------------------------
.../graph/v1/AtlasEntityGraphDiscoveryV1.java | 4 +-
.../store/graph/v1/BulkImporterImpl.java | 4 +-
.../repository/impexp/ImportServiceTest.java | 63 ++++++++++++++++---
.../src/test/resources/stocks-glossary.zip | Bin 0 -> 3876 bytes
4 files changed, 57 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/65f7da17/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
index f092d18..e31ca4d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityGraphDiscoveryV1.java
@@ -305,9 +305,7 @@ public class AtlasEntityGraphDiscoveryV1 implements EntityGraphDiscovery {
List<String> visitedAttributes = new ArrayList<>();
// visit relationship attributes
- if(!(this.discoveryContext.getEntityStream() instanceof EntityImportStream)) {
- visitRelationships(entityType, entity, visitedAttributes);
- }
+ visitRelationships(entityType, entity, visitedAttributes);
// visit struct attributes
for (AtlasAttribute attribute : entityType.getAllAttributes().values()) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/65f7da17/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/BulkImporterImpl.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/BulkImporterImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/BulkImporterImpl.java
index af31095..e17daf6 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/BulkImporterImpl.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/BulkImporterImpl.java
@@ -73,10 +73,10 @@ public class BulkImporterImpl implements BulkImporter {
AtlasEntityWithExtInfo entityWithExtInfo = entityImportStreamWithResidualList.getNextEntityWithExtInfo();
AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
- if (entity == null || processedGuids.contains(entity.getGuid())) {
+ if (entity == null) {
continue;
}
-
+
AtlasEntityStreamForImport oneEntityStream = new AtlasEntityStreamForImport(entityWithExtInfo, entityStream);
try {
http://git-wip-us.apache.org/repos/asf/atlas/blob/65f7da17/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index 91d56ab..5089070 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -22,13 +22,16 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.discovery.EntityDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.slf4j.Logger;
@@ -42,6 +45,7 @@ import org.testng.annotations.Test;
import java.io.IOException;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
@@ -54,6 +58,7 @@ import static org.testng.Assert.assertNotNull;
@Guice(modules = TestModules.TestOnlyModule.class)
public class ImportServiceTest {
private static final Logger LOG = LoggerFactory.getLogger(ImportServiceTest.class);
+ private static final int DEFAULT_LIMIT = 25;
private final ImportService importService;
@Inject
@@ -63,6 +68,9 @@ public class ImportServiceTest {
private AtlasTypeDefStore typeDefStore;
@Inject
+ private EntityDiscoveryService discoveryService;
+
+ @Inject
public ImportServiceTest(ImportService importService) {
this.importService = importService;
}
@@ -104,10 +112,6 @@ public class ImportServiceTest {
runAndVerifyQuickStart_v1_Import(importService, zipSource);
}
- private void loadBaseModel() throws IOException, AtlasBaseException {
- loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
- }
-
@DataProvider(name = "logging")
public static Object[][] getDataFromLogging(ITestContext context) throws IOException {
return getZipSource("logging-v1-full.zip");
@@ -180,6 +184,28 @@ public class ImportServiceTest {
runImportWithNoParameters(importService, zipSource);
}
+ @DataProvider(name = "stocks-glossary")
+ public static Object[][] getDataFromGlossary(ITestContext context) throws IOException {
+ return getZipSource("stocks-glossary.zip");
+ }
+
+ @Test(dataProvider = "stocks-glossary")
+ public void importGlossary(ZipSource zipSource) throws IOException, AtlasBaseException {
+ loadBaseModel();
+ loadGlossary();
+ runImportWithNoParameters(importService, zipSource);
+
+ assertEntityCount("__AtlasGlossary", "40c80052-3129-4f7c-8f2f-391677935416", 1);
+ assertEntityCount("__AtlasGlossaryTerm", "e93ac426-de04-4d54-a7c9-d76c1e96369b", 1);
+ assertEntityCount("__AtlasGlossaryTerm", "93ad3bf6-23dc-4e3f-b70e-f8fad6438203", 1);
+ assertEntityCount("__AtlasGlossaryTerm", "105533b6-c125-4a87-bed5-cdf67fb68c39", 1);
+ }
+
+ private List<AtlasEntityHeader> getEntitiesFromDB(String query, String guid) throws AtlasBaseException {
+ String q = StringUtils.isEmpty(guid) ? query : String.format("%s where __guid = '%s'", query, guid);
+ return discoveryService.searchUsingDslQuery(q, DEFAULT_LIMIT, 0).getEntities();
+ }
+
@DataProvider(name = "hdfs_path1")
public static Object[][] getDataFromHdfsPath1(ITestContext context) throws IOException {
return getZipSource("hdfs_path1.zip");
@@ -226,6 +252,20 @@ public class ImportServiceTest {
loadHiveModel();
AtlasImportRequest request = getDefaultImportRequest();
runImportWithParameters(importService, request, zipSource);
+
+ assertEntityCount("hive_db", "d7dc0848-fbba-4d63-9264-a460798361f5", 1);
+ assertEntityCount("hive_table", "2fb31eaa-4bb2-4eb8-b333-a888ba7c84fe", 1);
+ assertEntityCount("hive_column", "13422f0c-9265-4960-91a9-290ffd83b7f1",1);
+ assertEntityCount("hive_column", "c1ae870f-ce0c-44ae-832f-ff77035b1f7e",1);
+ assertEntityCount("hive_column", "b84baab3-0664-4f13-82f1-e81d043db02f",1);
+ assertEntityCount("hive_column", "53ea1991-6ca8-44f2-a75e-61b8d4866fc8",1);
+ assertEntityCount("hive_column", "a973c04c-aa42-49f4-877c-66fbe6754fb5",1);
+ assertEntityCount("hive_column", "a4550803-f18e-4072-a1e8-1201e6022a58",1);
+ assertEntityCount("hive_column", "6c4f196a-4046-493b-8c3a-2b1a9ef255a2",1);
+ }
+
+ private void assertEntityCount(String entityType, String guid, int expectedCount) throws AtlasBaseException {
+ assertEquals(getEntitiesFromDB(entityType, guid).size(), expectedCount);
}
@Test
@@ -233,11 +273,8 @@ public class ImportServiceTest {
ImportService importService = new ImportService(typeDefStore, typeRegistry, null);
AtlasImportRequest req = mock(AtlasImportRequest.class);
- Answer<Map> answer = new Answer<Map>() {
- @Override
- public Map answer(InvocationOnMock invocationOnMock) throws Throwable {
- throw new IOException("file is read only");
- }
+ Answer<Map> answer = invocationOnMock -> {
+ throw new IOException("file is read only");
};
when(req.getFileName()).thenReturn("some-file.zip");
@@ -258,4 +295,12 @@ public class ImportServiceTest {
private void loadHiveModel() throws IOException, AtlasBaseException {
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
}
+
+ private void loadBaseModel() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ }
+
+ private void loadGlossary() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0011-glossary_model.json", typeDefStore, typeRegistry);
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/65f7da17/repository/src/test/resources/stocks-glossary.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/stocks-glossary.zip b/repository/src/test/resources/stocks-glossary.zip
new file mode 100644
index 0000000..dc8039f
Binary files /dev/null and b/repository/src/test/resources/stocks-glossary.zip differ