You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/11/10 05:48:21 UTC
[5/5] atlas git commit: ATLAS-2251: restored bunch of deleted tests
ATLAS-2251: restored bunch of deleted tests
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/042fc557
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/042fc557
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/042fc557
Branch: refs/heads/ATLAS-2251
Commit: 042fc55755e17b3b0112a7a08d30c296300b95d9
Parents: 5f40541
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Thu Nov 9 21:44:26 2017 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Nov 9 21:44:26 2017 -0800
----------------------------------------------------------------------
.../org/apache/atlas/query/QueryParser.java | 11 +-
.../impexp/AtlasImportRequestTest.java | 145 +++
.../repository/impexp/ExportServiceTest.java | 358 ++++++
.../repository/impexp/ImportServiceTest.java | 224 ++++
.../impexp/ImportServiceTestUtils.java | 116 ++
.../impexp/ImportTransformerJSONTest.java | 47 +
.../impexp/ImportTransformerTest.java | 131 +++
.../repository/impexp/ImportTransformsTest.java | 151 +++
.../impexp/TypeAttributeDifferenceTest.java | 150 +++
.../atlas/repository/impexp/UniqueListTest.java | 69 ++
.../impexp/ZipFileResourceTestUtils.java | 220 ++++
.../atlas/repository/impexp/ZipSinkTest.java | 210 ++++
.../atlas/repository/impexp/ZipSourceTest.java | 189 ++++
.../store/graph/AtlasTypeDefGraphStoreTest.java | 644 +++++++++++
.../graph/v1/AtlasEntityDefStoreV1Test.java | 73 ++
...AtlasEntityStoreV1BulkImportPercentTest.java | 167 +++
.../store/graph/v1/AtlasEntityStoreV1Test.java | 1043 ++++++++++++++++++
.../v1/AtlasRelationshipDefStoreV1Test.java | 330 ++++++
.../AtlasRelationshipStoreHardDeleteV1Test.java | 112 ++
.../AtlasRelationshipStoreSoftDeleteV1Test.java | 116 ++
.../graph/v1/AtlasRelationshipStoreV1Test.java | 623 +++++++++++
.../InverseReferenceUpdateHardDeleteV1Test.java | 75 ++
.../InverseReferenceUpdateSoftDeleteV1Test.java | 78 ++
.../graph/v1/InverseReferenceUpdateV1Test.java | 374 +++++++
.../userprofile/UserProfileServiceTest.java | 280 +++++
.../atlas/services/MetricsServiceTest.java | 123 +++
.../utils/ObjectUpdateSynchronizerTest.java | 218 ++++
.../org/apache/atlas/util/RestUtilsTest.java | 195 ++++
28 files changed, 6471 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/main/java/org/apache/atlas/query/QueryParser.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/query/QueryParser.java b/repository/src/main/java/org/apache/atlas/query/QueryParser.java
index be2d9b1..1e5e5ff 100644
--- a/repository/src/main/java/org/apache/atlas/query/QueryParser.java
+++ b/repository/src/main/java/org/apache/atlas/query/QueryParser.java
@@ -19,11 +19,20 @@ package org.apache.atlas.query;
import org.apache.atlas.query.Expressions.Expression;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
public class QueryParser {
+ private static final Set<String> RESERVED_KEYWORDS =
+ new HashSet<>(Arrays.asList("[", "]", "(", ")", "=", "<", ">", "!=", "<=", ">=", ",", "and", "or", "+", "-",
+ "*", "/", ".", "select", "from", "where", "groupby", "loop", "isa", "is", "has",
+ "as", "times", "withPath", "limit", "offset", "orderby", "count", "max", "min",
+ "sum", "by", "order", "like"));
+
public static boolean isKeyword(String word) {
- return false; // TODO:
+ return RESERVED_KEYWORDS.contains(word);
}
public static Expression apply(String queryStr, QueryParams params) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
new file mode 100644
index 0000000..b70b181
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/AtlasImportRequestTest.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.type.AtlasType;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.*;
+
+public class AtlasImportRequestTest {
+ @Test
+ public void serializeAtlasImportRequstFromJsonWithEmptyOptions() {
+ String jsonData = "{ \"options\": {} }";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNotNull(request.getOptions());
+ assertNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ assertNull(tr);
+ }
+
+ @Test
+ public void serializeOptions_VerifyAccessors() {
+ String guid = "\"abcd\"";
+ String pos = "\"1\"";
+ String trueVal = "\"true\"";
+
+ String jsonData = "{ \"options\": " +
+ " {" +
+ "\"startGuid\":" + guid + "," +
+ "\"startPosition\":" + pos + "," +
+ "\"updateTypeDefinition\":" + trueVal +
+ "}" +
+ "}";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNotNull(request.getStartGuid());
+ assertNotNull(request.getStartPosition());
+ assertNotNull(request.getUpdateTypeDefs());
+
+ assertEquals(request.getStartGuid(), guid.replace("\"", ""));
+ assertEquals(request.getStartPosition(), pos.replace("\"", ""));
+ assertEquals(request.getUpdateTypeDefs(), trueVal.replace("\"", ""));
+ }
+
+ @Test
+ public void optionsDefaultsTest() {
+ String jsonData = "{ \"options\": " +
+ " {" +
+ "}" +
+ "}";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNull(request.getStartGuid());
+ assertNull(request.getStartPosition());
+ assertNull(request.getUpdateTypeDefs());
+ }
+
+ @Test
+ public void serializeAtlasImportRequstFromJsonWithEmptyTransforms() {
+ String jsonData = "{ \"options\": { \"transforms\": \"{ }\" } }";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNotNull(request.getOptions());
+ assertNotNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ assertNotNull(tr);
+ assertNotNull(tr.getTransforms());
+ assertEquals(tr.getTransforms().size(), 0);
+ }
+
+ @Test
+ public void serializeAtlasImportRequstFromJsonWith1Transform() {
+ String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] } }\" } }";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNotNull(request.getOptions());
+ assertNotNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ assertNotNull(tr);
+ assertNotNull(tr.getTransforms());
+ assertEquals(tr.getTransforms().size(), 1);
+ assertTrue(tr.getTransforms().containsKey("hive_db"));
+ assertEquals(tr.getTransforms("hive_db").entrySet().size(), 1);
+ assertTrue(tr.getTransforms("hive_db").containsKey("qualifiedName"));
+ assertEquals(tr.getTransforms("hive_db").get("qualifiedName").size(), 1);
+ }
+
+ @Test
+ public void serializeAtlasImportRequstFromJson() {
+ String jsonData = "{ \"options\": { \"transforms\": \"{ \\\"hive_db\\\": { \\\"qualifiedName\\\": [ \\\"replace:@cl1:@cl2\\\" ] }, \\\"hive_table\\\": { \\\"qualifiedName\\\": [ \\\"lowercase\\\", \\\"replace:@cl1:@cl2\\\" ] } }\" } } }";
+
+ AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
+
+ assertNotNull(request);
+ assertNotNull(request.getOptions());
+ assertNotNull(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ ImportTransforms tr = ImportTransforms.fromJson(request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY));
+
+ assertNotNull(tr);
+ assertNotNull(tr.getTransforms());
+ assertEquals(tr.getTransforms().size(), 2);
+ assertTrue(tr.getTransforms().containsKey("hive_db"));
+ assertEquals(tr.getTransforms("hive_db").entrySet().size(), 1);
+ assertTrue(tr.getTransforms("hive_db").containsKey("qualifiedName"));
+ assertEquals(tr.getTransforms("hive_db").get("qualifiedName").size(), 1);
+ assertTrue(tr.getTransforms().containsKey("hive_table"));
+ assertEquals(tr.getTransforms("hive_table").entrySet().size(), 1);
+ assertTrue(tr.getTransforms("hive_table").containsKey("qualifiedName"));
+ assertEquals(tr.getTransforms("hive_table").get("qualifiedName").size(), 2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
new file mode 100644
index 0000000..2a8bdfa
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -0,0 +1,358 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityChangeNotifier;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
+import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
+import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ExportServiceTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ExportServiceTest.class);
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private EntityGraphMapper graphMapper;
+ @Inject
+ ExportService exportService;
+ private DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class);;
+ private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
+ private AtlasEntityStoreV1 entityStore;
+
+ @BeforeTest
+ public void setupTest() {
+ RequestContextV1.clear();
+ RequestContextV1.get().setUser(TestUtilsV2.TEST_USER);
+ }
+
+ @BeforeClass
+ public void setupSampleData() throws AtlasBaseException {
+ entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);;
+
+ AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes();
+ AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry);
+
+ if (!typesToCreate.isEmpty()) {
+ typeDefStore.createTypesDef(typesToCreate);
+ }
+
+ AtlasEntity.AtlasEntitiesWithExtInfo hrDept = TestUtilsV2.createDeptEg2();
+
+ AtlasEntityStream entityStream = new AtlasEntityStream(hrDept);
+ entityStore.createOrUpdate(entityStream, false);
+ LOG.debug("==> setupSampleData: ", AtlasEntity.dumpObjects(hrDept.getEntities(), null).toString());
+ }
+
+ @AfterClass
+ public void clear() {
+ AtlasGraphProvider.cleanup();
+ }
+
+ private AtlasExportRequest getRequestForFullFetch() {
+ AtlasExportRequest request = new AtlasExportRequest();
+
+ List<AtlasObjectId> itemsToExport = new ArrayList<>();
+ itemsToExport.add(new AtlasObjectId("hive_db", "qualifiedName", "default@cl1"));
+ request.setItemsToExport(itemsToExport);
+
+ return request;
+ }
+
+ private AtlasExportRequest getRequestForDept(boolean addFetchType, String fetchTypeValue, boolean addMatchType, String matchTypeValue) {
+ AtlasExportRequest request = new AtlasExportRequest();
+
+ List<AtlasObjectId> itemsToExport = new ArrayList<>();
+ itemsToExport.add(new AtlasObjectId("Department", "name", "hr"));
+ request.setItemsToExport(itemsToExport);
+
+ setOptionsMap(request, addFetchType, fetchTypeValue, addMatchType, matchTypeValue);
+ return request;
+ }
+
+ private AtlasExportRequest getRequestForEmployee() {
+ AtlasExportRequest request = new AtlasExportRequest();
+
+ List<AtlasObjectId> itemsToExport = new ArrayList<>();
+ itemsToExport.add(new AtlasObjectId("Employee", "name", "Max"));
+ request.setItemsToExport(itemsToExport);
+
+ setOptionsMap(request, true, "CONNECTED", false, "");
+ return request;
+ }
+
+ private void setOptionsMap(AtlasExportRequest request,
+ boolean addFetchType, String fetchTypeValue, boolean addMatchType, String matchTypeValue) {
+ Map<String, Object> optionsMap = null;
+ if(addFetchType) {
+ if(optionsMap == null) {
+ optionsMap = new HashMap<>();
+ }
+
+ optionsMap.put("fetchType", fetchTypeValue);
+ request.setOptions(optionsMap);
+ }
+
+ if(addMatchType) {
+ if(optionsMap == null) {
+ optionsMap = new HashMap<>();
+ }
+
+ optionsMap.put("matchType", matchTypeValue);
+ }
+
+ if(optionsMap != null) {
+ request.setOptions(optionsMap);
+ }
+ }
+
+ private ZipSource runExportWithParameters(AtlasExportRequest request) throws AtlasBaseException, IOException {
+ final String requestingIP = "1.0.0.0";
+ final String hostName = "localhost";
+ final String userName = "admin";
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ZipSink zipSink = new ZipSink(baos);
+ AtlasExportResult result = exportService.run(zipSink, request, userName, hostName, requestingIP);
+
+ zipSink.close();
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(baos.toByteArray());
+ ZipSource zipSource = new ZipSource(bis);
+ return zipSource;
+ }
+
+ @Test
+ public void exportType_Succeeds() throws AtlasBaseException, FileNotFoundException {
+ String requestingIP = "1.0.0.0";
+ String hostName = "root";
+
+ AtlasExportRequest request = getRequestForFullFetch();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ZipSink zipSink = new ZipSink(baos);
+ AtlasExportResult result = exportService.run(zipSink, request, "admin", hostName, requestingIP);
+
+ assertNotNull(exportService);
+ assertEquals(result.getHostName(), hostName);
+ assertEquals(result.getClientIpAddress(), requestingIP);
+ assertEquals(request, result.getRequest());
+ }
+
+ @Test
+ public void requestingEntityNotFound_NoData() throws AtlasBaseException, IOException {
+ String requestingIP = "1.0.0.0";
+ String hostName = "root";
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ ZipSink zipSink = new ZipSink(baos);
+ AtlasExportResult result = exportService.run(
+ zipSink, getRequestForFullFetch(), "admin", hostName, requestingIP);
+
+ Assert.assertNull(result.getData());
+
+ ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
+ ZipSource zipSource = new ZipSource(bais);
+
+ assertNotNull(exportService);
+ assertNotNull(zipSource.getCreationOrder());
+ Assert.assertFalse(zipSource.hasNext());
+ }
+
+ @Test
+ public void requestingEntityFoundDefaultFetch_ContainsData() throws Exception {
+ ZipSource source = runExportWithParameters(
+ getRequestForDept(false, "", false, ""));
+ verifyExportForHrData(source);
+ }
+
+ @Test
+ public void requestingHrEntityWithMatchTypeContains_ContainsData() throws Exception {
+ ZipSource source = runExportWithParameters(
+ getRequestForDept(false, "", true, "CONTAINS"));
+ verifyExportForHrData(source);
+ }
+
+ @Test
+ public void requestingHrEntityWithMatchTypeEndsWith_ContainsData() throws Exception {
+ ZipSource source = runExportWithParameters(
+ getRequestForDept(false, "", true, "ENDSWITH"));
+ verifyExportForHrData(source);
+ }
+
+ @Test
+ public void requestingDeptEntityFoundFullFetch_ContainsData() throws Exception {
+ ZipSource source = runExportWithParameters(
+ getRequestForDept(true, "FULL", false, ""));
+ verifyExportForHrData(source);
+ }
+
+ @Test
+ public void requestingDeptEntityFoundConnectedFetch_ContainsData() throws Exception {
+ ZipSource source = runExportWithParameters(
+ getRequestForDept(true, "CONNECTED", false, ""));
+ verifyExportForHrDataForConnected(source);
+ }
+
+ @Test
+ public void requestingEmployeeEntityFoundConnectedFetch_ContainsData() throws Exception {
+ ZipSource zipSource = runExportWithParameters(getRequestForEmployee());
+ verifyExportForEmployeeData(zipSource);
+ }
+
+ @Test
+ public void verifyOverallStatus() throws Exception {
+
+// ExportService service = new ExportService(typeRegistry);
+ assertEquals(AtlasExportResult.OperationStatus.FAIL, exportService.getOverallOperationStatus());
+
+ assertEquals(AtlasExportResult.OperationStatus.SUCCESS, exportService.getOverallOperationStatus(AtlasExportResult.OperationStatus.SUCCESS));
+
+ assertEquals(AtlasExportResult.OperationStatus.SUCCESS, exportService.getOverallOperationStatus(
+ AtlasExportResult.OperationStatus.SUCCESS,
+ AtlasExportResult.OperationStatus.SUCCESS,
+ AtlasExportResult.OperationStatus.SUCCESS));
+
+ assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, exportService.getOverallOperationStatus(
+ AtlasExportResult.OperationStatus.FAIL,
+ AtlasExportResult.OperationStatus.PARTIAL_SUCCESS,
+ AtlasExportResult.OperationStatus.SUCCESS));
+
+ assertEquals(AtlasExportResult.OperationStatus.PARTIAL_SUCCESS, exportService.getOverallOperationStatus(
+ AtlasExportResult.OperationStatus.FAIL,
+ AtlasExportResult.OperationStatus.FAIL,
+ AtlasExportResult.OperationStatus.PARTIAL_SUCCESS));
+
+ assertEquals(AtlasExportResult.OperationStatus.FAIL, exportService.getOverallOperationStatus(
+ AtlasExportResult.OperationStatus.FAIL,
+ AtlasExportResult.OperationStatus.FAIL,
+ AtlasExportResult.OperationStatus.FAIL));
+ }
+
+ @Test
+ public void requestingExportOfNonExistentEntity_ReturnsFailure() throws Exception {
+ AtlasExportRequest request = getRequestForEmployee();
+ tamperEmployeeRequest(request);
+ ZipSource zipSource = runExportWithParameters(request);
+
+ assertNotNull(zipSource.getCreationOrder());
+ assertEquals(zipSource.getCreationOrder().size(), 0);
+ assertEquals(AtlasExportResult.OperationStatus.FAIL, zipSource.getExportResult().getOperationStatus());
+ }
+
+ private void tamperEmployeeRequest(AtlasExportRequest request) {
+ AtlasObjectId objectId = request.getItemsToExport().get(0);
+ objectId.getUniqueAttributes().remove("name");
+ objectId.getUniqueAttributes().put("qualifiedName", "XXX@121");
+ }
+
+ private void verifyExportForEmployeeData(ZipSource zipSource) throws AtlasBaseException {
+ final List<String> expectedEntityTypes = Arrays.asList(new String[]{"Manager", "Employee", "Department"});
+
+ assertNotNull(zipSource.getCreationOrder());
+ assertEquals(zipSource.getCreationOrder().size(), 2);
+ assertTrue(zipSource.hasNext());
+
+ while (zipSource.hasNext()) {
+ AtlasEntity entity = zipSource.next();
+
+ assertNotNull(entity);
+ assertEquals(AtlasEntity.Status.ACTIVE, entity.getStatus());
+ assertTrue(expectedEntityTypes.contains(entity.getTypeName()));
+ }
+
+ verifyTypeDefs(zipSource);
+ }
+
+ private void verifyExportForHrData(ZipSource zipSource) throws IOException, AtlasBaseException {
+ assertNotNull(zipSource.getCreationOrder());
+ assertTrue(zipSource.getCreationOrder().size() == 1);
+ assertTrue(zipSource.hasNext());
+
+ AtlasEntity entity = zipSource.next();
+
+ assertNotNull(entity);
+ assertTrue(entity.getTypeName().equals("Department"));
+ assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+ verifyTypeDefs(zipSource);
+ }
+
+ private void verifyExportForHrDataForConnected(ZipSource zipSource) throws IOException, AtlasBaseException {
+ assertNotNull(zipSource.getCreationOrder());
+ assertTrue(zipSource.getCreationOrder().size() == 2);
+ assertTrue(zipSource.hasNext());
+
+ AtlasEntity entity = zipSource.next();
+
+ assertNotNull(entity);
+ assertTrue(entity.getTypeName().equals("Department"));
+ assertEquals(entity.getStatus(), AtlasEntity.Status.ACTIVE);
+ verifyTypeDefs(zipSource);
+ }
+
+ private void verifyTypeDefs(ZipSource zipSource) throws AtlasBaseException {
+ assertEquals(zipSource.getTypesDef().getEnumDefs().size(), 1);
+ assertEquals(zipSource.getTypesDef().getClassificationDefs().size(), 0);
+ assertEquals(zipSource.getTypesDef().getStructDefs().size(), 1);
+ assertEquals(zipSource.getTypesDef().getEntityDefs().size(), 4);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
new file mode 100644
index 0000000..b24774d
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import com.google.inject.Inject;
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasClassificationType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.ITestContext;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ImportServiceTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ImportServiceTest.class);
+ private final ImportService importService;
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ public ImportServiceTest(ImportService importService) {
+ this.importService = importService;
+ }
+
+ @BeforeTest
+ public void setupTest() {
+ RequestContextV1.clear();
+ RequestContextV1.get().setUser(TestUtilsV2.TEST_USER);
+ }
+
+ @DataProvider(name = "sales")
+ public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException {
+ return getZipSource("sales-v1-full.zip");
+ }
+
+ @Test(dataProvider = "sales")
+ public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException {
+ loadBaseModel();
+ runAndVerifyQuickStart_v1_Import(importService, zipSource);
+ }
+
+ @DataProvider(name = "reporting")
+ public static Object[][] getDataFromReporting() throws IOException {
+ return getZipSource("reporting-v1-full.zip");
+ }
+
+ @Test(dataProvider = "reporting")
+ public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException {
+ loadBaseModel();
+ runAndVerifyQuickStart_v1_Import(importService, zipSource);
+ }
+
+ private void loadBaseModel() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ }
+
+ @DataProvider(name = "logging")
+ public static Object[][] getDataFromLogging(ITestContext context) throws IOException {
+ return getZipSource("logging-v1-full.zip");
+ }
+
+ @Test(dataProvider = "logging")
+ public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException {
+ loadBaseModel();
+ runAndVerifyQuickStart_v1_Import(importService, zipSource);
+ }
+
+ @DataProvider(name = "salesNewTypeAttrs")
+ public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException {
+ return getZipSource("salesNewTypeAttrs.zip");
+ }
+
+ @Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1")
+ public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException {
+ loadBaseModel();
+ runImportWithParameters(importService, getDefaultImportRequest(), zipSource);
+ }
+
+ @DataProvider(name = "salesNewTypeAttrs-next")
+ public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException {
+ return getZipSource("salesNewTypeAttrs-next.zip");
+ }
+
+
+ @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
+ public void importDB5(ZipSource zipSource) throws AtlasBaseException, IOException {
+ final String newEnumDefName = "database_action";
+
+ assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+
+ AtlasImportRequest request = getDefaultImportRequest();
+ Map<String, String> options = new HashMap<>();
+ options.put("updateTypeDefinition", "false");
+ request.setOptions(options);
+
+ runImportWithParameters(importService, request, zipSource);
+ assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+ assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 4);
+ }
+
+ @Test(dataProvider = "salesNewTypeAttrs-next", dependsOnMethods = "importDB4")
+ public void importDB6(ZipSource zipSource) throws AtlasBaseException, IOException {
+ final String newEnumDefName = "database_action";
+
+ assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+
+ AtlasImportRequest request = getDefaultImportRequest();
+ Map<String, String> options = new HashMap<>();
+ options.put("updateTypeDefinition", "true");
+ request.setOptions(options);
+
+ runImportWithParameters(importService, request, zipSource);
+ assertNotNull(typeDefStore.getEnumDefByName(newEnumDefName));
+ assertEquals(typeDefStore.getEnumDefByName(newEnumDefName).getElementDefs().size(), 8);
+ }
+
+ @DataProvider(name = "ctas")
+ public static Object[][] getDataFromCtas(ITestContext context) throws IOException {
+ return getZipSource("ctas.zip");
+ }
+
+ @Test(dataProvider = "ctas")
+ public void importCTAS(ZipSource zipSource) throws IOException, AtlasBaseException {
+ loadBaseModel();
+ loadHiveModel();
+
+ runImportWithNoParameters(importService, zipSource);
+ }
+
+ @DataProvider(name = "hdfs_path1")
+ public static Object[][] getDataFromHdfsPath1(ITestContext context) throws IOException {
+ return getZipSource("hdfs_path1.zip");
+ }
+
+
+ @Test(dataProvider = "hdfs_path1", expectedExceptions = AtlasBaseException.class)
+ public void importHdfs_path1(ZipSource zipSource) throws IOException, AtlasBaseException {
+ loadBaseModel();
+ loadFsModel();
+ loadModelFromResourcesJson("tag1.json", typeDefStore, typeRegistry);
+
+ try {
+ runImportWithNoParameters(importService, zipSource);
+ } catch (AtlasBaseException e) {
+ assertEquals(e.getAtlasErrorCode(), AtlasErrorCode.INVALID_IMPORT_ATTRIBUTE_TYPE_CHANGED);
+ AtlasClassificationType tag1 = typeRegistry.getClassificationTypeByName("tag1");
+ assertNotNull(tag1);
+ assertEquals(tag1.getAllAttributes().size(), 2);
+ throw e;
+ }
+ }
+
+ @Test
+ public void importServiceProcessesIOException() {
+ ImportService importService = new ImportService(typeDefStore, typeRegistry, null);
+ AtlasImportRequest req = mock(AtlasImportRequest.class);
+
+ Answer<Map> answer = new Answer<Map>() {
+ @Override
+ public Map answer(InvocationOnMock invocationOnMock) throws Throwable {
+ throw new IOException("file is read only");
+ }
+ };
+
+ when(req.getFileName()).thenReturn("some-file.zip");
+ when(req.getOptions()).thenAnswer(answer);
+
+ try {
+ importService.run(req, "a", "b", "c");
+ }
+ catch (AtlasBaseException ex) {
+ assertEquals(ex.getAtlasErrorCode().getErrorCode(), AtlasErrorCode.INVALID_PARAMETERS.getErrorCode());
+ }
+ }
+
+ private void loadFsModel() throws IOException, AtlasBaseException {
+ loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
+ }
+
+ private void loadHiveModel() throws IOException, AtlasBaseException {
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java
new file mode 100644
index 0000000..7289512
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTestUtils.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import com.google.common.collect.Sets;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.Assert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ImportServiceTestUtils {
+
+ public static void verifyImportedEntities(List<String> creationOrder, List<String> processedEntities) {
+ Set<String> lhs = com.google.common.collect.Sets.newHashSet(creationOrder);
+ Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
+ Set<String> difference = Sets.difference(lhs, rhs);
+
+ Assert.assertNotNull(difference);
+ Assert.assertEquals(difference.size(), 0);
+ }
+
+ public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
+ Map<String, Integer> metricsForCompare = getImportMetricsForCompare(importResult);
+ for (Map.Entry<String, Integer> entry : exportResult.getMetrics().entrySet()) {
+ if(entry.getKey().startsWith("entity") == false ||
+ entry.getKey().contains("withExtInfo") ||
+ entry.getKey().contains("Column") ||
+ entry.getKey().contains("StorageDesc")) continue;
+
+ Assert.assertTrue(metricsForCompare.containsKey(entry.getKey()));
+ Assert.assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()));
+ }
+ }
+
+ private static Map<String,Integer> getImportMetricsForCompare(AtlasImportResult result) {
+ Map<String, Integer> r = new HashMap<>();
+ for (Map.Entry<String, Integer> entry : result.getMetrics().entrySet()) {
+ r.put(entry.getKey().replace(":updated", "").replace(":created", ""), entry.getValue());
+ }
+
+ return r;
+ }
+
+
+ public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
+ AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
+ createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
+ }
+
+ private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry);
+
+ if (!typesToCreate.isEmpty()) {
+ typeDefStore.createTypesDef(typesToCreate);
+ }
+ }
+
+ private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName) throws IOException {
+ String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
+ return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
+ }
+
+ public static AtlasImportRequest getDefaultImportRequest() {
+ return new AtlasImportRequest();
+ }
+
+
+ public static AtlasImportResult runImportWithParameters(ImportService importService, AtlasImportRequest request, ZipSource source) throws AtlasBaseException, IOException {
+ final String requestingIP = "1.0.0.0";
+ final String hostName = "localhost";
+ final String userName = "admin";
+
+ AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
+ Assert.assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
+ return result;
+ }
+
+ public static void runAndVerifyQuickStart_v1_Import(ImportService importService, ZipSource zipSource) throws AtlasBaseException, IOException {
+ AtlasExportResult exportResult = zipSource.getExportResult();
+ List<String> creationOrder = zipSource.getCreationOrder();
+
+ AtlasImportRequest request = getDefaultImportRequest();
+ AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
+
+ Assert.assertNotNull(result);
+ verifyImportedMetrics(exportResult, result);
+ verifyImportedEntities(creationOrder, result.getProcessedEntities());
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
new file mode 100644
index 0000000..7044e71
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerJSONTest.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+import static org.testng.Assert.*;
+
+public class ImportTransformerJSONTest {
+
+
+ @Test
+ public void createAtlasImportTransformFromJson() throws Exception {
+ String hiveTableType = "hive_table";
+ String qualifiedName = "qualifiedName";
+ String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+
+ ImportTransforms transforms = ImportTransforms.fromJson(jsonTransforms);
+
+ assertNotNull(transforms);
+ assertEquals(transforms.getTransforms().entrySet().size(), 1);
+ assertEquals(transforms.getTransforms().get(hiveTableType).entrySet().size(), 1);
+ assertEquals(transforms.getTransforms().get(hiveTableType).get(qualifiedName).size(), 2);
+ Assert.assertEquals(transforms.getTransforms().get(hiveTableType).get(qualifiedName).get(0).getTransformType(), "lowercase");
+ assertEquals(transforms.getTransforms().get(hiveTableType).get(qualifiedName).get(1).getTransformType(), "replace");
+ assertTrue(transforms.getTransforms().get(hiveTableType).get(qualifiedName).get(1) instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)transforms.getTransforms().get(hiveTableType).get(qualifiedName).get(1)).getToFindStr(), "@cl1");
+ assertEquals(((ImportTransformer.Replace)transforms.getTransforms().get(hiveTableType).get(qualifiedName).get(1)).getReplaceStr(), "@cl2");
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
new file mode 100644
index 0000000..7ce34c8
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformerTest.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+public class ImportTransformerTest {
+
+ @Test
+ public void createWithCorrectParameters() throws AtlasBaseException, IllegalAccessException {
+ String param1 = "@cl1";
+ String param2 = "@cl2";
+
+ ImportTransformer e = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
+
+ assertTrue(e instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)e).getToFindStr(), param1);
+ assertEquals(((ImportTransformer.Replace)e).getReplaceStr(), param2);
+ }
+
+ @Test
+ public void createSeveralWithCorrectParameters() throws AtlasBaseException, IllegalAccessException {
+ String param1 = "@cl1";
+ String param2 = "@cl2";
+
+ ImportTransformer e1 = ImportTransformer.getTransformer(String.format("%s:%s:%s", "replace", param1, param2));
+ ImportTransformer e2 = ImportTransformer.getTransformer(String.format("replace:tt1:tt2"));
+
+ assertTrue(e1 instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)e1).getToFindStr(), param1);
+ assertEquals(((ImportTransformer.Replace)e1).getReplaceStr(), param2);
+
+ assertTrue(e2 instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)e2).getToFindStr(), "tt1");
+ assertEquals(((ImportTransformer.Replace)e2).getReplaceStr(), "tt2");
+ }
+
+ @Test
+ public void createWithDefaultParameters() throws AtlasBaseException {
+ ImportTransformer e1 = ImportTransformer.getTransformer("replace:@cl1");
+ ImportTransformer e2 = ImportTransformer.getTransformer("replace");
+
+ assertTrue(e1 instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)e1).getToFindStr(), "@cl1");
+ assertEquals(((ImportTransformer.Replace)e1).getReplaceStr(), "");
+
+ assertTrue(e2 instanceof ImportTransformer.Replace);
+ assertEquals(((ImportTransformer.Replace)e2).getToFindStr(), "");
+ assertEquals(((ImportTransformer.Replace)e2).getReplaceStr(), "");
+ }
+
+ @Test
+ public void applyLowercaseTransformer() throws AtlasBaseException {
+ ImportTransformer e = ImportTransformer.getTransformer("lowercase");
+
+ assertEquals(e.apply("@CL1"), "@cl1");
+ assertEquals(e.apply("@cl1"), "@cl1");
+ assertEquals(e.apply(""), ""); // empty string
+ assertEquals(e.apply(null), null); // null value: no change
+ assertEquals(e.apply(Integer.valueOf(5)), Integer.valueOf(5)); // non-string value: no change
+ }
+
+ @Test
+ public void applyUppercaseTransformer() throws AtlasBaseException {
+ ImportTransformer e = ImportTransformer.getTransformer("uppercase");
+
+ assertEquals(e.apply("@CL1"), "@CL1");
+ assertEquals(e.apply("@cl1"), "@CL1");
+ assertEquals(e.apply(""), ""); // empty string
+ assertEquals(e.apply(null), null); // null value: no change
+ assertEquals(e.apply(Integer.valueOf(5)), Integer.valueOf(5)); // non-string value: no change
+ }
+
+ @Test
+ public void applyReplaceTransformer1() throws AtlasBaseException {
+ ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1:@cl2");
+
+ assertEquals(e.apply("@cl1"), "@cl2");
+ assertEquals(e.apply("default@cl1"), "default@cl2");
+ assertEquals(e.apply("@cl11"), "@cl21");
+ assertEquals(e.apply("@cl2"), "@cl2");
+ assertEquals(e.apply(""), ""); // empty string
+ assertEquals(e.apply(null), null); // null value
+ assertEquals(e.apply(Integer.valueOf(5)), Integer.valueOf(5)); // non-string value: no change
+ }
+
+ @Test
+ public void applyReplaceTransformer2() throws AtlasBaseException {
+ ImportTransformer e = ImportTransformer.getTransformer("replace:@cl1");
+
+ assertEquals(e.apply("@cl1"), "");
+ assertEquals(e.apply("default@cl1"), "default");
+ assertEquals(e.apply("@cl11"), "1");
+ assertEquals(e.apply("@cl2"), "@cl2");
+ assertEquals(e.apply(""), ""); // empty string
+ assertEquals(e.apply(null), null); // null value
+ assertEquals(e.apply(Integer.valueOf(5)), Integer.valueOf(5)); // non-string value: no change
+ }
+
+ @Test
+ public void applyReplaceTransformer3() throws AtlasBaseException {
+ ImportTransformer e = ImportTransformer.getTransformer("replace");
+
+ assertEquals(e.apply("@cl1"), "@cl1");
+ assertEquals(e.apply("default@cl1"), "default@cl1");
+ assertEquals(e.apply("@cl11"), "@cl11");
+ assertEquals(e.apply("@cl2"), "@cl2");
+ assertEquals(e.apply(""), ""); // empty string
+ assertEquals(e.apply(null), null); // null value
+ assertEquals(e.apply(Integer.valueOf(5)), Integer.valueOf(5)); // non-string value: no change
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
new file mode 100644
index 0000000..ccedeb1
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportTransformsTest.java
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+public class ImportTransformsTest {
+ private final String qualifiedName = "qualifiedName";
+ private final String lowerCaseCL1 = "@cl1";
+ private final String lowerCaseCL2 = "@cl2";
+ private final String jsonTransforms = "{ \"hive_table\": { \"qualifiedName\":[ \"lowercase\", \"replace:@cl1:@cl2\" ] } }";
+
+ private ImportTransforms transform;
+
+ @BeforeTest
+ public void setup() throws AtlasBaseException {
+ transform = ImportTransforms.fromJson(jsonTransforms);
+ }
+
+ @Test
+ public void transformEntityWith2Transforms() throws AtlasBaseException {
+ AtlasEntity entity = getHiveTableAtlasEntity();
+ String attrValue = (String) entity.getAttribute(qualifiedName);
+
+ transform.apply(entity);
+
+ assertEquals(entity.getAttribute(qualifiedName), applyDefaultTransform(attrValue));
+ }
+
+ @Test
+ public void transformEntityWithExtInfo() throws AtlasBaseException {
+ addColumnTransform(transform);
+
+ AtlasEntityWithExtInfo entityWithExtInfo = getAtlasEntityWithExtInfo();
+ AtlasEntity entity = entityWithExtInfo.getEntity();
+ String attrValue = (String) entity.getAttribute(qualifiedName);
+ String[] expectedValues = getExtEntityExpectedValues(entityWithExtInfo);
+
+ transform.apply(entityWithExtInfo);
+
+ assertEquals(entityWithExtInfo.getEntity().getAttribute(qualifiedName), applyDefaultTransform(attrValue));
+
+ for (int i = 0; i < expectedValues.length; i++) {
+ assertEquals(entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName), expectedValues[i]);
+ }
+ }
+
+ @Test
+ public void transformEntityWithExtInfoNullCheck() throws AtlasBaseException {
+ addColumnTransform(transform);
+
+ AtlasEntityWithExtInfo entityWithExtInfo = getAtlasEntityWithExtInfo();
+
+ entityWithExtInfo.setReferredEntities(null);
+
+ AtlasEntityWithExtInfo transformedEntityWithExtInfo = transform.apply(entityWithExtInfo);
+
+ assertNotNull(transformedEntityWithExtInfo);
+ assertEquals(entityWithExtInfo.getEntity().getGuid(), transformedEntityWithExtInfo.getEntity().getGuid());
+ }
+
+ private String[] getExtEntityExpectedValues(AtlasEntityWithExtInfo entityWithExtInfo) {
+ String[] ret = new String[entityWithExtInfo.getReferredEntities().size()];
+
+ for (int i = 0; i < ret.length; i++) {
+ String attrValue = (String) entityWithExtInfo.getReferredEntities().get(Integer.toString(i)).getAttribute(qualifiedName);
+
+ ret[i] = attrValue.replace(lowerCaseCL1, lowerCaseCL2);
+ }
+
+ return ret;
+ }
+
+ private void addColumnTransform(ImportTransforms transform) throws AtlasBaseException {
+ Map<String, List<ImportTransformer>> tr = new HashMap<>();
+ List<ImportTransformer> trList = new ArrayList<>();
+
+ trList.add(ImportTransformer.getTransformer(String.format("replace:%s:%s", lowerCaseCL1, lowerCaseCL2)));
+
+ tr.put(qualifiedName, trList);
+
+ transform.getTransforms().put("hive_column", tr);
+ }
+
+ private String applyDefaultTransform(String attrValue) {
+ return attrValue.toLowerCase().replace(lowerCaseCL1, lowerCaseCL2);
+ }
+
+ private AtlasEntity getHiveTableAtlasEntity() {
+ AtlasEntity entity = new AtlasEntity("hive_table");
+
+ Map<String, Object> attributes = new HashMap<>();
+ attributes.put(qualifiedName, "TABLE1.default" + lowerCaseCL1);
+ attributes.put("dbname", "someDB");
+ attributes.put("name", "somename");
+
+ entity.setAttributes(attributes);
+ return entity;
+ }
+
+ private AtlasEntity getHiveColumnAtlasEntity(int index) {
+ AtlasEntity entity = new AtlasEntity("hive_column");
+
+ Map<String, Object> attributes = new HashMap<>();
+ attributes.put(qualifiedName, String.format("col%s.TABLE1.default@cl1", index));
+ attributes.put("name", "col" + index);
+
+ entity.setAttributes(attributes);
+ return entity;
+ }
+
+ private AtlasEntityWithExtInfo getAtlasEntityWithExtInfo() {
+ AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(getHiveTableAtlasEntity());
+
+ Map<String, AtlasEntity> referredEntities = new HashMap<>();
+ referredEntities.put("0", getHiveColumnAtlasEntity(1));
+ referredEntities.put("1", getHiveColumnAtlasEntity(2));
+ referredEntities.put("2", getHiveColumnAtlasEntity(3));
+
+ ret.setReferredEntities(referredEntities);
+
+ return ret;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
new file mode 100644
index 0000000..87cb1d4
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class TypeAttributeDifferenceTest {
+ private TypeAttributeDifference typeAttributeDifference;
+
+ @BeforeClass
+ public void setup() {
+ typeAttributeDifference = new TypeAttributeDifference(null, null);
+ }
+
+ private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) {
+ int i = startIndex;
+ List<AtlasEnumDef.AtlasEnumElementDef> list = new ArrayList<>();
+ for (String s: names) {
+ list.add(new AtlasEnumDef.AtlasEnumElementDef(s, s, i++));
+ }
+
+ return list;
+ }
+
+ private List<AtlasStructDef.AtlasAttributeDef> getAtlasAttributeDefs(String... names) {
+ List<AtlasStructDef.AtlasAttributeDef> list = new ArrayList<>();
+ for (String s : names) {
+ list.add(new AtlasStructDef.AtlasAttributeDef(s, AtlasBaseTypeDef.ATLAS_TYPE_STRING));
+ }
+
+ return list;
+ }
+
+ private AtlasEntityDef getAtlasEntityDefWithAttributes(String... attributeNames) {
+ AtlasEntityDef e = new AtlasEntityDef();
+ for (AtlasStructDef.AtlasAttributeDef a : getAtlasAttributeDefs(attributeNames)) {
+ e.addAttribute(a);
+ }
+
+ return e;
+ }
+
+ @Test
+ public void entityDefWithNoAttributes() throws Exception {
+ AtlasEntityDef existing = new AtlasEntityDef();
+ AtlasEntityDef incoming = new AtlasEntityDef();
+ List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = new ArrayList<>();
+ List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+ Assert.assertEquals(actualAttributes, expectedAttributes);
+ }
+
+ private List<AtlasStructDef.AtlasAttributeDef> invokeGetAttributesAbsentInExisting(AtlasStructDef existing, AtlasStructDef incoming) throws Exception {
+ return typeAttributeDifference.getElementsAbsentInExisting(existing, incoming);
+ }
+
+ private List<AtlasEnumDef.AtlasEnumElementDef> invokeGetAttributesAbsentInExisting(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
+ return typeAttributeDifference.getElementsAbsentInExisting(existing, incoming);
+ }
+
+ private AtlasEnumDef getAtlasEnumWithAttributes(String... elements) {
+ AtlasEnumDef enumDef = new AtlasEnumDef();
+ for (AtlasEnumDef.AtlasEnumElementDef ed : getEnumElementDefs(0, elements)) {
+ enumDef.addElement(ed);
+ }
+
+ return enumDef;
+ }
+
+ @Test
+ public void bothSame_DifferenceIsEmptyList() throws Exception {
+ AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+ AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+
+ List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
+ List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+ Assert.assertEquals(actualAttributes, expectedAttributes);
+ }
+
+ @Test
+ public void different_ReturnsDifference() throws Exception {
+ AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name");
+ AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+ List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs( "qualifiedName");
+
+ List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+ Assert.assertEquals(actualAttributes, expectedAttributes);
+ }
+
+ @Test
+ public void differentSubset_ReturnsDifference() throws Exception {
+ AtlasEntityDef existing = getAtlasEntityDefWithAttributes("name", "qualifiedName");
+ AtlasEntityDef incoming = getAtlasEntityDefWithAttributes("name");
+ List<AtlasStructDef.AtlasAttributeDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+ List<AtlasStructDef.AtlasAttributeDef> expectedAttributes = getAtlasAttributeDefs();
+ Assert.assertEquals(actualAttributes, expectedAttributes);
+ }
+
+ @Test
+ public void differentEnumDef_ReturnsDifference () throws Exception {
+ AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
+ AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
+ List<AtlasEnumDef.AtlasEnumElementDef> actualAttributes = invokeGetAttributesAbsentInExisting(existing, incoming);
+
+ List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(2, "Delta", "Echo");
+ Assert.assertEquals(actualAttributes, expectedAttributes);
+ }
+
+ @Test
+ public void differentEnumDefs_ReturnsDifference () throws Exception {
+ AtlasEnumDef existing = getAtlasEnumWithAttributes("Alpha", "Bravo");
+ AtlasEnumDef incoming = getAtlasEnumWithAttributes("Alpha", "Bravo", "Delta", "Echo");
+ boolean ret = invokeUpdate(existing, incoming);
+
+ List<AtlasEnumDef.AtlasEnumElementDef> expectedAttributes = getEnumElementDefs(0, "Alpha", "Bravo", "Delta", "Echo");
+
+ Assert.assertTrue(ret, "Update took place");
+ Assert.assertEquals(existing.getElementDefs(), expectedAttributes);
+ }
+
+ private boolean invokeUpdate(AtlasEnumDef existing, AtlasEnumDef incoming) throws Exception {
+ return typeAttributeDifference.addElements(existing, incoming);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java
new file mode 100644
index 0000000..93aa518
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/UniqueListTest.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+
+public class UniqueListTest {
+ private final String firstElement = "firstElement";
+ private ExportService.UniqueList<String> uniqueList;
+
+ @BeforeClass
+ public void setup() {
+ uniqueList = new ExportService.UniqueList();
+ uniqueList.add(firstElement);
+ uniqueList.add("def");
+ uniqueList.add("firstElement");
+ uniqueList.add("ghi");
+ }
+
+ @Test
+ public void add3Elements_ListHas2() {
+ assertEquals(3, uniqueList.size());
+ }
+
+ @Test
+ public void addAllList_ListHas2() {
+ ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+ uniqueList2.addAll(uniqueList);
+
+ assertEquals(3, uniqueList2.size());
+ }
+
+ @Test
+ public void attemptClear_SizeIsZero() {
+ ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+ uniqueList2.addAll(uniqueList);
+ uniqueList2.clear();
+
+ assertEquals(0, uniqueList2.size());
+ }
+
+ @Test
+ public void attemptOneRemove_SizeIsReduced() {
+ ExportService.UniqueList<String> uniqueList2 = new ExportService.UniqueList<>();
+ uniqueList2.addAll(uniqueList);
+ String removedElement = uniqueList2.remove(0);
+
+ assertEquals(2, uniqueList2.size());
+ assertEquals(firstElement, removedElement);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
new file mode 100644
index 0000000..5ab8c01
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -0,0 +1,220 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+import com.google.common.collect.Sets;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class ZipFileResourceTestUtils {
+ public static final Logger LOG = LoggerFactory.getLogger(ZipFileResourceTestUtils.class);
+
+ public static FileInputStream getFileInputStream(String fileName) {
+ return TestResourceFileUtils.getFileInputStream(fileName);
+ }
+
+ public static String getModelJson(String fileName) throws IOException {
+ String ret = null;
+ File topModelsDir = new File(System.getProperty("user.dir") + "/../addons/models");
+ File[] topModelsDirContents = topModelsDir.exists() ? topModelsDir.listFiles() : null;
+
+ assertTrue(topModelsDirContents != null, topModelsDir.getAbsolutePath() + ": unable to find/read directory");
+ if(topModelsDirContents != null) {
+ Arrays.sort(topModelsDirContents);
+ for (File modelDir : topModelsDirContents) {
+ if (modelDir.exists() && modelDir.isDirectory()) {
+ ret = getFileContents(modelDir, fileName);
+
+ if (ret != null) {
+ break;
+ }
+ }
+ }
+
+ if (ret == null) {
+ ret = getFileContents(topModelsDir, fileName);
+ }
+
+ assertTrue(ret != null, fileName + ": unable to find model file");
+ } else {
+ throw new IOException("Unable to retrieve model contents.");
+ }
+
+ return ret;
+ }
+
+ public static String getFileContents(File dir, String fileName) throws IOException {
+ if (dir.exists() && dir.isDirectory()) {
+ File file = new File(dir, fileName);
+
+ if (file.exists() && file.isFile()) {
+ return FileUtils.readFileToString(file);
+ }
+ }
+
+ return null;
+ }
+
+ public static String getModelJsonFromResources(String fileName) throws IOException {
+ String filePath = TestResourceFileUtils.getTestFilePath(fileName);
+ File f = new File(filePath);
+ String s = FileUtils.readFileToString(f);
+ assertFalse(StringUtils.isEmpty(s), "Model file read correctly from resources!");
+
+ return s;
+ }
+
+ public static Object[][] getZipSource(String fileName) throws IOException {
+ FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream(fileName);
+
+ return new Object[][]{{new ZipSource(fs)}};
+ }
+
+
+ public static void verifyImportedEntities(List<String> creationOrder, List<String> processedEntities) {
+ Set<String> lhs = com.google.common.collect.Sets.newHashSet(creationOrder);
+ Set<String> rhs = com.google.common.collect.Sets.newHashSet(processedEntities);
+ Set<String> difference = Sets.difference(lhs, rhs);
+
+ assertNotNull(difference);
+ assertEquals(difference.size(), 0);
+ }
+
+ public static void verifyImportedMetrics(AtlasExportResult exportResult, AtlasImportResult importResult) {
+ Map<String, Integer> metricsForCompare = getImportMetricsForCompare(importResult);
+ for (Map.Entry<String, Integer> entry : exportResult.getMetrics().entrySet()) {
+ if(entry.getKey().startsWith("entity") == false ||
+ entry.getKey().contains("withExtInfo") ||
+ entry.getKey().contains("Column") ||
+ entry.getKey().contains("StorageDesc")) continue;
+
+ assertTrue(metricsForCompare.containsKey(entry.getKey()), entry.getKey());
+ assertEquals(entry.getValue(), metricsForCompare.get(entry.getKey()), entry.getKey());
+ }
+ }
+
+ private static Map<String,Integer> getImportMetricsForCompare(AtlasImportResult result) {
+ Map<String, Integer> r = new HashMap<>();
+ for (Map.Entry<String, Integer> entry : result.getMetrics().entrySet()) {
+ r.put(entry.getKey().replace(":updated", "").replace(":created", ""), entry.getValue());
+ }
+
+ return r;
+ }
+
+
+ public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
+ AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
+ createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
+ }
+
+ public static void loadModelFromResourcesJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
+ AtlasTypesDef typesFromJson = getAtlasTypesDefFromResourceFile(fileName);
+ createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
+ }
+
+ private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ if(typesFromJson == null) {
+ return;
+ }
+
+ AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(typesFromJson, typeRegistry);
+ if (typesToCreate != null && !typesToCreate.isEmpty()) {
+ typeDefStore.createTypesDef(typesToCreate);
+ }
+ }
+
+ private static AtlasTypesDef getAtlasTypesDefFromFile(String fileName) throws IOException {
+ String sampleTypes = ZipFileResourceTestUtils.getModelJson(fileName);
+ if(sampleTypes == null) return null;
+ return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
+ }
+
+ private static AtlasTypesDef getAtlasTypesDefFromResourceFile(String fileName) throws IOException {
+ String sampleTypes = getModelJsonFromResources(fileName);
+ return AtlasType.fromJson(sampleTypes, AtlasTypesDef.class);
+ }
+
+ public static AtlasImportRequest getDefaultImportRequest() {
+ return new AtlasImportRequest();
+ }
+
+
+ public static AtlasImportResult runImportWithParameters(ImportService importService, AtlasImportRequest request, ZipSource source) throws AtlasBaseException, IOException {
+ final String requestingIP = "1.0.0.0";
+ final String hostName = "localhost";
+ final String userName = "admin";
+
+ AtlasImportResult result = importService.run(source, request, userName, hostName, requestingIP);
+ assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
+ return result;
+ }
+
+ public static AtlasImportResult runImportWithNoParameters(ImportService importService, ZipSource source) throws AtlasBaseException, IOException {
+ final String requestingIP = "1.0.0.0";
+ final String hostName = "localhost";
+ final String userName = "admin";
+
+ AtlasImportResult result = importService.run(source, userName, hostName, requestingIP);
+ assertEquals(result.getOperationStatus(), AtlasImportResult.OperationStatus.SUCCESS);
+ return result;
+ }
+
+ public static void runAndVerifyQuickStart_v1_Import(ImportService importService, ZipSource zipSource) throws AtlasBaseException, IOException {
+ AtlasExportResult exportResult = zipSource.getExportResult();
+ List<String> creationOrder = zipSource.getCreationOrder();
+
+ RequestContextV1.clear();
+ RequestContextV1.get().setUser(TestUtilsV2.TEST_USER);
+
+ AtlasImportRequest request = getDefaultImportRequest();
+ AtlasImportResult result = runImportWithParameters(importService, request, zipSource);
+
+ assertNotNull(result);
+ verifyImportedMetrics(exportResult, result);
+ verifyImportedEntities(creationOrder, result.getProcessedEntities());
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/042fc557/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
new file mode 100644
index 0000000..e8bbeb5
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSinkTest.java
@@ -0,0 +1,210 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.impexp;
+
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.type.AtlasType;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipInputStream;
+
+import static org.testng.Assert.*;
+
+public class ZipSinkTest {
+ private ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ private ZipSink zipSink;
+ private List<String> defaultExportOrder = new ArrayList<>(Arrays.asList("a", "b", "c", "d"));
+ private AtlasExportResult defaultExportResult;
+ private String knownEntityGuidFormat = "111-222-333-%s";
+
+
+ private void initZipSinkWithExportOrder() throws AtlasBaseException {
+ zipSink = new ZipSink(byteArrayOutputStream);
+ zipSink.setExportOrder(defaultExportOrder);
+ zipSink.close();
+ }
+
+ private AtlasExportResult getDefaultExportResult() {
+ AtlasExportRequest request = new AtlasExportRequest();
+
+ List<AtlasObjectId> itemsToExport = new ArrayList<>();
+ itemsToExport.add(new AtlasObjectId("hive_db", "qualifiedName", "default"));
+ request.setItemsToExport(itemsToExport);
+
+ defaultExportResult = new AtlasExportResult(request, "admin", "1.0.0.0", "root", 100);
+ return defaultExportResult;
+ }
+
+ private ZipInputStream getZipInputStreamForDefaultExportOrder() throws AtlasBaseException {
+ initZipSinkWithExportOrder();
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(byteArrayOutputStream.toByteArray());
+ return new ZipInputStream(bis);
+ }
+
+ private String getZipEntryAsStream(ZipInputStream zis) throws IOException {
+ byte[] buf = new byte[1024];
+ int n = 0;
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ while ((n = zis.read(buf, 0, 1024)) > -1) {
+ bos.write(buf, 0, n);
+ }
+
+ Assert.assertNotNull(bos);
+ return bos.toString();
+ }
+
+ @Test
+ public void correctInit_succeeds() throws AtlasBaseException {
+ initZipSinkWithExportOrder();
+ assertTrue(true);
+ Assert.assertNotNull(zipSink);
+ }
+
+ @Test
+ public void zipWithExactlyOneEntry_succeeds() {
+
+ try {
+ ZipInputStream zis = getZipInputStreamForDefaultExportOrder();
+
+ try {
+ Assert.assertNotNull(zis.getNextEntry());
+ Assert.assertNull(zis.getNextEntry());
+ } catch (IOException e) {
+
+ assertTrue(false);
+ }
+ } catch (AtlasBaseException e) {
+
+ assertTrue(false, "No exception should be thrown.");
+ }
+ }
+
+ @Test
+ public void verifyExportOrderEntryName_verifies() throws AtlasBaseException, IOException {
+
+ ZipInputStream zis = getZipInputStreamForDefaultExportOrder();
+ ZipEntry ze = zis.getNextEntry();
+
+ assertEquals(ze.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
+ }
+
+ @Test
+ public void zipWithExactlyOneEntry_ContentsVerified() throws AtlasBaseException, IOException {
+
+ ZipInputStream zis = getZipInputStreamForDefaultExportOrder();
+ zis.getNextEntry();
+
+ assertEquals(getZipEntryAsStream(zis).replace("\"", "'"), "['a','b','c','d']");
+ }
+
+ @Test
+ public void zipWithExactlyTwoEntries_ContentsVerified() throws AtlasBaseException, IOException {
+
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ useZipSinkToCreateEntries(byteOutputStream);
+
+ ByteArrayInputStream bis = new ByteArrayInputStream(byteOutputStream.toByteArray());
+ ZipInputStream zipStream = new ZipInputStream(bis);
+ ZipEntry entry = zipStream.getNextEntry();
+
+ assertEquals(getZipEntryAsStream(zipStream), "[\"a\",\"b\",\"c\",\"d\"]");
+ assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString());
+
+ entry = zipStream.getNextEntry();
+ assertEquals(entry.getName().replace(".json", ""), ZipExportFileNames.ATLAS_EXPORT_INFO_NAME.toString());
+ assertTrue(compareJsonWithObject(getZipEntryAsStream(zipStream), defaultExportResult));
+ }
+
+ @Test
+ public void recordsEntityEntries() throws AtlasBaseException {
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ ZipSink zs = new ZipSink(byteOutputStream);
+
+ AtlasEntity entity = new AtlasEntity();
+ entity.setGuid(String.format(knownEntityGuidFormat, 0));
+
+ zs.add(entity);
+ assertTrue(zs.hasEntity(String.format(knownEntityGuidFormat, 0)));
+
+ zs.close();
+ }
+
+ @Test
+ public void recordsEntityWithExtInfoEntries() throws AtlasBaseException {
+ final int max_entries = 3;
+ ByteArrayOutputStream byteOutputStream = new ByteArrayOutputStream();
+ ZipSink zs = new ZipSink(byteOutputStream);
+
+ AtlasEntity entity = new AtlasEntity();
+ entity.setGuid(String.format(knownEntityGuidFormat, 0));
+
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo(entity);
+ addReferredEntities(entityWithExtInfo, max_entries);
+
+ zs.add(entityWithExtInfo);
+ for (int i = 0; i <= max_entries; i++) {
+ String g = String.format(knownEntityGuidFormat, i);
+ assertTrue(zs.hasEntity(g));
+ }
+
+ zs.close();
+ }
+
+ private void addReferredEntities(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, int maxEntries) {
+
+ for (int i = 1; i <= maxEntries; i++) {
+ AtlasEntity entity1 = new AtlasEntity();
+ entity1.setGuid(String.format(knownEntityGuidFormat, i));
+ entityWithExtInfo.addReferredEntity(entity1);
+ }
+ }
+
+ @Test
+ public void recordsDoesNotRecordEntityEntries() throws AtlasBaseException {
+ initZipSinkWithExportOrder();
+
+ assertNotNull(zipSink);
+ assertFalse(zipSink.hasEntity(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()));
+ }
+
+ private void useZipSinkToCreateEntries(ByteArrayOutputStream byteOutputStream) throws AtlasBaseException {
+ ZipSink zs = new ZipSink(byteOutputStream);
+ zs.setExportOrder(defaultExportOrder);
+ zs.setResult(getDefaultExportResult());
+ zs.close();
+ }
+
+ private boolean compareJsonWithObject(String s, AtlasExportResult defaultExportResult) {
+ String json = AtlasType.toJson(defaultExportResult);
+ return json.equals(s);
+ }
+}