You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/10/12 00:27:55 UTC
[01/17] atlas git commit: ATLAS-2874: Include handling of Atlas
Entity Transformers in current Import logic
Repository: atlas
Updated Branches:
refs/heads/master 8746b3063 -> 1eb995434
ATLAS-2874: Include handling of Atlas Entity Transformers in current Import logic
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/4b3c078c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/4b3c078c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/4b3c078c
Branch: refs/heads/master
Commit: 4b3c078c0efe41d8a5a3fe073bd24c87622a409a
Parents: 8746b30
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Mon Sep 17 21:57:40 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 15:38:23 2018 -0700
----------------------------------------------------------------------
.../atlas/model/impexp/AtlasImportRequest.java | 1 +
.../atlas/repository/impexp/ImportService.java | 50 +++++++++++++++++--
.../atlas/repository/impexp/ZipSource.java | 51 ++++++++++++++++----
3 files changed, 89 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/4b3c078c/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
index 2989fbe..06bc231 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasImportRequest.java
@@ -40,6 +40,7 @@ public class AtlasImportRequest implements Serializable {
private static final long serialVersionUID = 1L;
public static final String TRANSFORMS_KEY = "transforms";
+ public static final String TRANSFORMERS_KEY = "transformers";
public static final String OPTION_KEY_REPLICATED_FROM = "replicatedFrom";
private static final String START_POSITION_KEY = "startPosition";
private static final String START_GUID_KEY = "startGuid";
http://git-wip-us.apache.org/repos/asf/atlas/blob/4b3c078c/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index a88ba2b..a09385e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -19,15 +19,17 @@ package org.apache.atlas.repository.impexp;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.entitytransform.BaseEntityHandler;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.graph.BulkImporter;
import org.apache.atlas.store.AtlasTypeDefStore;
-import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
@@ -40,6 +42,11 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.atlas.model.impexp.AtlasImportRequest.TRANSFORMERS_KEY;
+import static org.apache.atlas.model.impexp.AtlasImportRequest.TRANSFORMS_KEY;
@Component
public class ImportService {
@@ -82,9 +89,12 @@ public class ImportService {
try {
LOG.info("==> import(user={}, from={}, request={})", userName, requestingIP, request);
- String transforms = MapUtils.isNotEmpty(request.getOptions()) ? request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY) : null;
-
+ String transforms = MapUtils.isNotEmpty(request.getOptions()) ? request.getOptions().get(TRANSFORMS_KEY) : null;
setImportTransform(source, transforms);
+
+ String transformers = MapUtils.isNotEmpty(request.getOptions()) ? request.getOptions().get(TRANSFORMERS_KEY) : null;
+ setEntityTransformerHandlers(source, transformers);
+
startTimestamp = System.currentTimeMillis();
processTypes(source.getTypesDef(), result);
setStartPosition(request, source);
@@ -121,6 +131,38 @@ public class ImportService {
}
+ private void setEntityTransformerHandlers(ZipSource source, String transformersString) {
+ if (StringUtils.isEmpty(transformersString)) {
+ return;
+ }
+
+ Object transformersObj = AtlasType.fromJson(transformersString, Object.class);
+ List transformers = (transformersObj != null && transformersObj instanceof List) ? (List) transformersObj : null;
+
+ List<AttributeTransform> attributeTransforms = new ArrayList<>();
+
+ if (CollectionUtils.isNotEmpty(transformers)) {
+ for (Object transformer : transformers) {
+ String transformerStr = AtlasType.toJson(transformer);
+ AttributeTransform attributeTransform = AtlasType.fromJson(transformerStr, AttributeTransform.class);
+
+ if (attributeTransform == null) {
+ continue;
+ }
+
+ attributeTransforms.add(attributeTransform);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(attributeTransforms)) {
+ List<BaseEntityHandler> entityHandlers = BaseEntityHandler.createEntityHandlers(attributeTransforms);
+
+ if (CollectionUtils.isNotEmpty(entityHandlers)) {
+ source.setEntityHandlers(entityHandlers);
+ }
+ }
+ }
+
private void debugLog(String s, Object... params) {
if(!LOG.isDebugEnabled()) return;
@@ -148,7 +190,7 @@ public class ImportService {
try {
LOG.info("==> import(user={}, from={}, fileName={})", userName, requestingIP, fileName);
- String transforms = MapUtils.isNotEmpty(request.getOptions()) ? request.getOptions().get(AtlasImportRequest.TRANSFORMS_KEY) : null;
+ String transforms = MapUtils.isNotEmpty(request.getOptions()) ? request.getOptions().get(TRANSFORMS_KEY) : null;
File file = new File(fileName);
ZipSource source = new ZipSource(new ByteArrayInputStream(FileUtils.readFileToByteArray(file)), ImportTransforms.fromJson(transforms));
result = run(source, request, userName, hostName, requestingIP);
http://git-wip-us.apache.org/repos/asf/atlas/blob/4b3c078c/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index 60cd9f8..1f436ce 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -17,6 +17,7 @@
*/
package org.apache.atlas.repository.impexp;
+import org.apache.atlas.entitytransform.BaseEntityHandler;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
@@ -24,6 +25,7 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.graph.v2.EntityImportStream;
import org.apache.atlas.type.AtlasType;
+import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -42,12 +44,13 @@ import java.util.zip.ZipInputStream;
public class ZipSource implements EntityImportStream {
private static final Logger LOG = LoggerFactory.getLogger(ZipSource.class);
- private final InputStream inputStream;
- private List<String> creationOrder;
- private Iterator<String> iterator;
- private Map<String, String> guidEntityJsonMap;
- private ImportTransforms importTransform;
- private int currentPosition;
+ private final InputStream inputStream;
+ private List<String> creationOrder;
+ private Iterator<String> iterator;
+ private Map<String, String> guidEntityJsonMap;
+ private ImportTransforms importTransform;
+ private List<BaseEntityHandler> entityHandlers;
+ private int currentPosition;
public ZipSource(InputStream inputStream) throws IOException {
this(inputStream, null);
@@ -68,6 +71,14 @@ public class ZipSource implements EntityImportStream {
this.importTransform = importTransform;
}
+ public List<BaseEntityHandler> getEntityHandlers() {
+ return entityHandlers;
+ }
+
+ public void setEntityHandlers(List<BaseEntityHandler> entityHandlers) {
+ this.entityHandlers = entityHandlers;
+ }
+
public AtlasTypesDef getTypesDef() throws AtlasBaseException {
final String fileName = ZipExportFileNames.ATLAS_TYPESDEF_NAME.toString();
@@ -123,17 +134,39 @@ public class ZipSource implements EntityImportStream {
return this.creationOrder;
}
- public AtlasEntity.AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) throws AtlasBaseException {
+ public AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) throws AtlasBaseException {
String s = getFromCache(guid);
- AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = convertFromJson(AtlasEntity.AtlasEntityWithExtInfo.class, s);
+ AtlasEntityWithExtInfo entityWithExtInfo = convertFromJson(AtlasEntityWithExtInfo.class, s);
- if (importTransform != null) {
+ if (entityHandlers != null) {
+ applyTransformers(entityWithExtInfo);
+ } else if (importTransform != null) {
entityWithExtInfo = importTransform.apply(entityWithExtInfo);
}
return entityWithExtInfo;
}
+ private void applyTransformers(AtlasEntityWithExtInfo entityWithExtInfo) {
+ if (entityWithExtInfo == null) {
+ return;
+ }
+
+ transform(entityWithExtInfo.getEntity());
+
+ if (MapUtils.isNotEmpty(entityWithExtInfo.getReferredEntities())) {
+ for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) {
+ transform(e);
+ }
+ }
+ }
+
+ private void transform(AtlasEntity e) {
+ for (BaseEntityHandler handler : entityHandlers) {
+ handler.transform(e);
+ }
+ }
+
private <T> T convertFromJson(Class<T> clazz, String jsonData) throws AtlasBaseException {
T t;
try {
[12/17] atlas git commit: ATLAS-2888: Change marker fix for server
name. Unit test fix.
Posted by am...@apache.org.
ATLAS-2888: Change marker fix for server name. Unit test fix.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f6acb086
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f6acb086
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f6acb086
Branch: refs/heads/master
Commit: f6acb086146976ea2a63d93648be8a39d7acbc53
Parents: 6f74720
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Mon Oct 1 22:08:30 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:28 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ExportServiceTest.java | 16 ++++++++++------
.../atlas/repository/impexp/ImportServiceTest.java | 10 +++++-----
.../impexp/ReplicationEntityAttributeTest.java | 4 ++--
.../ClassificationPropagationTest.java | 2 +-
4 files changed, 18 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 377bd67..7aa0b57 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
@@ -84,9 +85,11 @@ public class ExportServiceTest extends ExportImportTestBase {
@Inject
private ExportImportAuditService auditService;
+ @Inject
+ private AtlasEntityStoreV2 entityStore;
+
private DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class);;
private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
- private AtlasEntityStoreV2 entityStore;
@BeforeTest
public void setupTest() throws IOException, AtlasBaseException {
@@ -106,11 +109,12 @@ public class ExportServiceTest extends ExportImportTestBase {
typeDefStore.createTypesDef(typesToCreate);
}
- AtlasEntity.AtlasEntitiesWithExtInfo hrDept = TestUtilsV2.createDeptEg2();
-
- AtlasEntityStream entityStream = new AtlasEntityStream(hrDept);
- entityStore.createOrUpdate(entityStream, false);
- LOG.debug("==> setupSampleData: ", AtlasEntity.dumpObjects(hrDept.getEntities(), null).toString());
+ AtlasEntity.AtlasEntitiesWithExtInfo deptEg2 = TestUtilsV2.createDeptEg2();
+ AtlasEntityStream entityStream = new AtlasEntityStream(deptEg2);
+ EntityMutationResponse emr = entityStore.createOrUpdate(entityStream, false);
+ assertNotNull(emr);
+ assertNotNull(emr.getCreatedEntities());
+ assertTrue(emr.getCreatedEntities().size() > 0);
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index e0bbb11..a1d6cef 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -213,7 +213,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "stocks-legacy")
- public static Object[][] getDataFromLegacyStocks(ITestContext context) throws IOException {
+ public static Object[][] getDataFromLegacyStocks(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("stocks.zip");
}
@@ -254,7 +254,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "stocks-glossary")
- public static Object[][] getDataFromGlossary(ITestContext context) throws IOException {
+ public static Object[][] getDataFromGlossary(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("stocks-glossary.zip");
}
@@ -298,12 +298,12 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "relationshipLineage")
- public static Object[][] getImportWithRelationships(ITestContext context) throws IOException {
+ public static Object[][] getImportWithRelationships(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("rel-lineage.zip");
}
@DataProvider(name = "tag-prop-2")
- public static Object[][] getImportWithTagProp2(ITestContext context) throws IOException {
+ public static Object[][] getImportWithTagProp2(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("tag-prop-2.zip");
}
@@ -316,7 +316,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "relationship")
- public static Object[][] getImportWithRelationshipsWithLineage(ITestContext context) throws IOException {
+ public static Object[][] getImportWithRelationshipsWithLineage(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("stocks-rel-2.zip");
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
index 81b9106..1eccdbf 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -84,14 +84,14 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase {
@Inject
AtlasServerService atlasServerService;
- private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
+ @Inject
private AtlasEntityStoreV2 entityStore;
+
private ZipSource zipSource;
@BeforeClass
public void setup() throws IOException, AtlasBaseException {
basicSetup(typeDefStore, typeRegistry);
- entityStore = new AtlasEntityStoreV2(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
createEntities(entityStore, ENTITIES_SUB_DIR, new String[]{"db", "table-columns"});
AtlasType refType = typeRegistry.getType("Referenceable");
http://git-wip-us.apache.org/repos/asf/atlas/blob/f6acb086/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
index 8459963..d5f36bc 100644
--- a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
@@ -605,7 +605,7 @@ public class ClassificationPropagationTest {
}
}
- public static ZipSource getZipSource(String fileName) throws IOException {
+ public static ZipSource getZipSource(String fileName) throws IOException, AtlasBaseException {
FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream(fileName);
return new ZipSource(fs);
}
[13/17] atlas git commit: ATLAS-2897: Better handling of empty zip
files.
Posted by am...@apache.org.
ATLAS-2897: Better handling of empty zip files.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/016eaffa
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/016eaffa
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/016eaffa
Branch: refs/heads/master
Commit: 016eaffae5518b86bba7bfdda4fc720749362a93
Parents: 8a26c79
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Oct 4 14:38:21 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:29 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/AtlasServerService.java | 1 -
.../org/apache/atlas/repository/impexp/ZipSource.java | 12 ++++++++----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/016eaffa/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index a3489a8..13a8cd9 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -65,7 +65,6 @@ public class AtlasServerService {
try {
return dataAccess.load(server);
} catch (AtlasBaseException e) {
- LOG.error("dataAccess", e);
throw e;
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/016eaffa/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index be8c168..bf23f81 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -72,10 +72,14 @@ public class ZipSource implements EntityImportStream {
}
private boolean isZipFileEmpty() {
- return MapUtils.isEmpty(guidEntityJsonMap) ||
- (guidEntityJsonMap.containsKey(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) &&
- (guidEntityJsonMap.get(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) == null)
- );
+ if (MapUtils.isEmpty(guidEntityJsonMap)) {
+ return true;
+ }
+
+ String key = ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString();
+ return (guidEntityJsonMap.containsKey(key) &&
+ StringUtils.isNotEmpty(guidEntityJsonMap.get(key)) &&
+ guidEntityJsonMap.get(key).equals("[]"));
}
public ImportTransforms getImportTransform() { return this.importTransform; }
[09/17] atlas git commit: ATLAS-2895: Server full name processing
Posted by am...@apache.org.
ATLAS-2895: Server full name processing
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/84c6fb2b
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/84c6fb2b
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/84c6fb2b
Branch: refs/heads/master
Commit: 84c6fb2bc3c6ec68315ed522027a566bfed8f678
Parents: f4dac18
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Sep 26 14:24:13 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:27 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/model/impexp/AtlasExportRequest.java | 14 ++++++++------
.../apache/atlas/repository/impexp/AuditsWriter.java | 9 ++++++++-
.../impexp/ReplicationEntityAttributeTest.java | 14 +++++++++++++-
3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
index e78bb53..8fb7c68 100644
--- a/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
+++ b/intg/src/main/java/org/apache/atlas/model/impexp/AtlasExportRequest.java
@@ -81,7 +81,7 @@ public class AtlasExportRequest implements Serializable {
}
public String getFetchTypeOptionValue() {
- if(getOptions() == null || !getOptions().containsKey(OPTION_FETCH_TYPE)) {
+ if(MapUtils.isEmpty(getOptions()) || !getOptions().containsKey(OPTION_FETCH_TYPE)) {
return FETCH_TYPE_FULL;
}
@@ -94,7 +94,8 @@ public class AtlasExportRequest implements Serializable {
}
public boolean getSkipLineageOptionValue() {
- if(!getOptions().containsKey(AtlasExportRequest.OPTION_SKIP_LINEAGE)) {
+ if(MapUtils.isEmpty(getOptions()) ||
+ !getOptions().containsKey(AtlasExportRequest.OPTION_SKIP_LINEAGE)) {
return false;
}
@@ -123,12 +124,13 @@ public class AtlasExportRequest implements Serializable {
}
public long getChangeTokenFromOptions() {
- if(getFetchTypeOptionValue().equalsIgnoreCase(FETCH_TYPE_INCREMENTAL) &&
- getOptions().containsKey(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER)) {
- return Long.parseLong(getOptions().get(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER).toString());
+ if (MapUtils.isEmpty(getOptions()) ||
+ !getFetchTypeOptionValue().equalsIgnoreCase(FETCH_TYPE_INCREMENTAL) ||
+ !getOptions().containsKey(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER)) {
+ return 0L;
}
- return 0L;
+ return Long.parseLong(getOptions().get(AtlasExportRequest.FETCH_TYPE_INCREMENTAL_CHANGE_MARKER).toString());
}
public StringBuilder toString(StringBuilder sb) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index cc10660..7737197 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -127,7 +127,14 @@ public class AuditsWriter {
return fullName;
}
- return StringUtils.split(fullName, "$")[1];
+ String[] splits = StringUtils.split(fullName, DC_SERVER_NAME_SEPARATOR);
+ if (splits == null || splits.length < 1) {
+ return "";
+ } else if (splits.length >= 2) {
+ return splits[1];
+ } else {
+ return splits[0];
+ }
}
private void saveCurrentServer() throws AtlasBaseException {
http://git-wip-us.apache.org/repos/asf/atlas/blob/84c6fb2b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
index 94483f5..81b9106 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -123,7 +123,19 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase {
assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_TO);
}
- @Test(dependsOnMethods = "exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute", enabled = false)
+ @Test
+ public void fullServerName() {
+ final String expectedClusterName = "cl1";
+
+ assertEquals(AuditsWriter.getServerNameFromFullName(""), "");
+ assertEquals(AuditsWriter.getServerNameFromFullName(expectedClusterName), expectedClusterName);
+ assertEquals(AuditsWriter.getServerNameFromFullName("SFO$cl1"), expectedClusterName);
+ assertEquals(AuditsWriter.getServerNameFromFullName("cl1$"), expectedClusterName);
+ assertEquals(AuditsWriter.getServerNameFromFullName("$cl1"), expectedClusterName);
+ }
+
+
+ @Test(dependsOnMethods = "exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute")
public void importWithReplicationFromOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException, IOException {
AtlasImportRequest request = getImportRequestWithReplicationOption();
AtlasImportResult importResult = runImportWithParameters(importService, request, zipSource);
[05/17] atlas git commit: ATLAS-2886: Support for fully qualified
server name
Posted by am...@apache.org.
ATLAS-2886: Support for fully qualified server name
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8639ada6
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8639ada6
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8639ada6
Branch: refs/heads/master
Commit: 8639ada6a74cdaa32b0b493aaff1168733487eef
Parents: 31c3bea
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Sun Sep 23 08:50:06 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:25 2018 -0700
----------------------------------------------------------------------
addons/models/0010-base_model.json | 335 +++++++++++++++++++
.../apache/atlas/entitytransform/Condition.java | 2 -
.../atlas/repository/impexp/AuditsWriter.java | 44 ++-
.../impexp/ExportImportAuditServiceTest.java | 2 +-
.../IncrementalExportEntityProviderTest.java | 2 -
.../impexp/ReplicationEntityAttributeTest.java | 14 +-
.../stocksDB-Entities/export-replicatedTo.json | 2 +-
.../import-replicatedFrom.json | 2 +-
8 files changed, 377 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
new file mode 100644
index 0000000..1bfbf2f
--- /dev/null
+++ b/addons/models/0010-base_model.json
@@ -0,0 +1,335 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [
+ {
+ "name": "TaxonomyTerm",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "atlas.taxonomy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "entityDefs": [
+ {
+ "name": "Referenceable",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "qualifiedName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ }
+ ]
+ },
+ {
+ "name": "__internal",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Asset",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "owner",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "DataSet",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Infrastructure",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Process",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "AtlasServer",
+ "typeVersion": "1.0",
+ "superTypes": [
+ ],
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "displayName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "fullName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "urls",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "additionalInfo",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "__AtlasUserProfile",
+ "superTypes": [
+ "__internal"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "fullName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "savedSearches",
+ "typeName": "array<__AtlasUserSavedSearch>",
+ "cardinality": "LIST",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ }
+ ]
+ },
+ {
+ "name": "__AtlasUserSavedSearch",
+ "superTypes": [
+ "__internal"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "ownerName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "uniqueName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "searchType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "searchParameters",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "searchParameters",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "__ExportImportAuditEntry",
+ "typeVersion": "1.0",
+ "superTypes": [
+ "__internal"
+ ],
+ "attributeDefs": [
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operation",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "sourceServerName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "targetServerName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operationParams",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "operationStartTime",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operationEndTime",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "resultSummary",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index 3bf49f0..b834f46 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -17,7 +17,6 @@
*/
package org.apache.atlas.entitytransform;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
@@ -226,7 +225,6 @@ public abstract class Condition {
}
}
- @VisibleForTesting
void addObjectId(AtlasObjectId objId) {
this.objectIds.add(objId);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index 407b406..f72de56 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -44,6 +44,7 @@ import java.util.Map;
public class AuditsWriter {
private static final Logger LOG = LoggerFactory.getLogger(AuditsWriter.class);
private static final String CLUSTER_NAME_DEFAULT = "default";
+ private static final String DC_SERVER_NAME_SEPARATOR = "$";
private AtlasServerService atlasServerService;
private ExportImportAuditService auditService;
@@ -74,7 +75,7 @@ public class AuditsWriter {
}
private void updateReplicationAttribute(boolean isReplicationSet,
- String serverName,
+ String serverName, String serverFullName,
List<String> exportedGuids,
String attrNameReplicated,
long lastModifiedTimestamp) throws AtlasBaseException {
@@ -82,7 +83,7 @@ public class AuditsWriter {
return;
}
- AtlasServer server = saveServer(serverName, exportedGuids.get(0), lastModifiedTimestamp);
+ AtlasServer server = saveServer(serverName, serverFullName, exportedGuids.get(0), lastModifiedTimestamp);
atlasServerService.updateEntitiesWithServer(server, exportedGuids, attrNameReplicated);
}
@@ -92,15 +93,16 @@ public class AuditsWriter {
: StringUtils.EMPTY;
}
- private AtlasServer saveServer(String name) throws AtlasBaseException {
- return atlasServerService.save(new AtlasServer(name, name));
+ private AtlasServer saveServer(String name, String serverFullName) {
+ AtlasServer cluster = new AtlasServer(name, serverFullName);
+ return atlasServerService.save(cluster);
}
- private AtlasServer saveServer(String name,
+ private AtlasServer saveServer(String name, String serverFullName,
String entityGuid,
- long lastModifiedTimestamp) throws AtlasBaseException {
+ long lastModifiedTimestamp) {
- AtlasServer server = new AtlasServer(name, name);
+ AtlasServer server = new AtlasServer(name, serverFullName);
server.setAdditionalInfoRepl(entityGuid, lastModifiedTimestamp);
if (LOG.isDebugEnabled()) {
@@ -120,11 +122,20 @@ public class AuditsWriter {
return StringUtils.EMPTY;
}
+ static String getServerNameFromFullName(String fullName) {
+ if (StringUtils.isEmpty(fullName) || !fullName.contains(DC_SERVER_NAME_SEPARATOR)) {
+ return fullName;
+ }
+
+ return StringUtils.split(fullName, "$")[1];
+ }
+
private class ExportAudits {
private AtlasExportRequest request;
private String targetServerName;
private String optionKeyReplicatedTo;
private boolean replicationOptionState;
+ private String targetServerFullName;
public void add(String userName, AtlasExportResult result,
long startTime, long endTime,
@@ -143,16 +154,17 @@ public class AuditsWriter {
return;
}
- updateReplicationAttribute(replicationOptionState, targetServerName,
+ updateReplicationAttribute(replicationOptionState, targetServerName, targetServerFullName,
entityGuids, Constants.ATTR_NAME_REPLICATED_TO, result.getChangeMarker());
}
private void saveServers() throws AtlasBaseException {
- saveServer(getCurrentClusterName());
+ saveServer(getCurrentClusterName(), getCurrentClusterName());
- targetServerName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
+ targetServerFullName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
+ targetServerName = getServerNameFromFullName(targetServerFullName);
if(StringUtils.isNotEmpty(targetServerName)) {
- saveServer(targetServerName);
+ saveServer(targetServerName, targetServerFullName);
}
}
}
@@ -162,6 +174,7 @@ public class AuditsWriter {
private boolean replicationOptionState;
private String sourceServerName;
private String optionKeyReplicatedFrom;
+ private String sourceServerFullName;
public void add(String userName, AtlasImportResult result,
long startTime, long endTime,
@@ -181,16 +194,17 @@ public class AuditsWriter {
return;
}
- updateReplicationAttribute(replicationOptionState, this.sourceServerName, entityGuids,
+ updateReplicationAttribute(replicationOptionState, sourceServerName, sourceServerFullName, entityGuids,
Constants.ATTR_NAME_REPLICATED_FROM, result.getExportResult().getChangeMarker());
}
private void saveServers() throws AtlasBaseException {
- saveServer(getCurrentClusterName());
+ saveServer(getCurrentClusterName(), getCurrentClusterName());
- sourceServerName = getClusterNameFromOptionsState();
+ sourceServerFullName = getClusterNameFromOptionsState();
+ sourceServerName = getServerNameFromFullName(sourceServerFullName);
if(StringUtils.isNotEmpty(sourceServerName)) {
- saveServer(sourceServerName);
+ saveServer(sourceServerName, sourceServerFullName);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
index 16fd39d..ba7a8a0 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportAuditServiceTest.java
@@ -61,7 +61,7 @@ public class ExportImportAuditServiceTest extends ExportImportTestBase {
}
@Test
- public void saveLogEntry() throws AtlasBaseException, InterruptedException {
+ public void saveLogEntry() throws AtlasBaseException {
final String source1 = "clx";
final String target1 = "cly";
ExportImportAuditEntry entry = saveAndGet(source1, ExportImportAuditEntry.OPERATION_EXPORT, target1);
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
index de0a8f8..85ed5f9 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/IncrementalExportEntityProviderTest.java
@@ -23,7 +23,6 @@ import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStoreV2;
-import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.repository.util.UniqueList;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -64,7 +63,6 @@ public class IncrementalExportEntityProviderTest extends ExportImportTestBase {
verifyCreatedEntities(entityStore, entityGuids, 2);
gremlinScriptEngine = atlasGraph.getGremlinScriptEngine();
- EntityGraphRetriever entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry);
incrementalExportEntityProvider = new IncrementalExportEntityProvider(atlasGraph, gremlinScriptEngine);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
index 79a5e05..94483f5 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -116,7 +116,10 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase {
assertNotNull(zipSource.getCreationOrder());
assertEquals(zipSource.getCreationOrder().size(), expectedEntityCount);
- assertCluster(REPLICATED_TO_CLUSTER_NAME, null);
+ assertCluster(
+ AuditsWriter.getServerNameFromFullName(REPLICATED_TO_CLUSTER_NAME),
+ REPLICATED_TO_CLUSTER_NAME, null);
+
assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_TO);
}
@@ -125,7 +128,9 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase {
AtlasImportRequest request = getImportRequestWithReplicationOption();
AtlasImportResult importResult = runImportWithParameters(importService, request, zipSource);
- assertCluster(REPLICATED_FROM_CLUSTER_NAME, importResult);
+ assertCluster(
+ AuditsWriter.getServerNameFromFullName(REPLICATED_FROM_CLUSTER_NAME),
+ REPLICATED_FROM_CLUSTER_NAME, importResult);
assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_FROM);
}
@@ -141,11 +146,12 @@ public class ReplicationEntityAttributeTest extends ExportImportTestBase {
}
}
- private void assertCluster(String name, AtlasImportResult importResult) throws AtlasBaseException {
- AtlasServer actual = atlasServerService.get(new AtlasServer(name, name));
+ private void assertCluster(String name, String fullName, AtlasImportResult importResult) throws AtlasBaseException {
+ AtlasServer actual = atlasServerService.get(new AtlasServer(name, fullName));
assertNotNull(actual);
assertEquals(actual.getName(), name);
+ assertEquals(actual.getFullName(), fullName);
if(importResult != null) {
assertClusterAdditionalInfo(actual, importResult);
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
index a69fe9e..a6fec6c 100644
--- a/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
+++ b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
@@ -6,6 +6,6 @@
],
"options": {
"fetchType": "full",
- "replicatedTo": "clTarget"
+ "replicatedTo": "dc2$clTarget"
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/8639ada6/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
index 1ce00ad..29268ef 100644
--- a/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
+++ b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
@@ -1,5 +1,5 @@
{
"options": {
- "replicatedFrom": "clSource"
+ "replicatedFrom": "dc1$clSource"
}
}
[10/17] atlas git commit: ATLAS-2897: Elegant handling of empty zip
files. Unit test fix.
Posted by am...@apache.org.
ATLAS-2897: Elegant handling of empty zip files. Unit test fix.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/67871413
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/67871413
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/67871413
Branch: refs/heads/master
Commit: 678714134342f749465c61ec70a4c3ba672e7379
Parents: 5fe6d83
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Mon Oct 1 10:53:38 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:27 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/repository/impexp/ZipSourceTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/67871413/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index f0034fa..7436dc0 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -56,7 +56,7 @@ public class ZipSourceTest {
return getZipSource("sales-v1-full.zip");
}
- @Test
+ @Test(expectedExceptions = AtlasBaseException.class)
public void improperInit_ReturnsNullCreationOrder() throws IOException, AtlasBaseException {
byte bytes[] = new byte[10];
ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
[03/17] atlas git commit: ATLAS-2882: AddClassification transform for
new transforms
Posted by am...@apache.org.
ATLAS-2882: AddClassification transform for new transforms
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/9d4f9728
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/9d4f9728
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/9d4f9728
Branch: refs/heads/master
Commit: 9d4f972892cc4248e4e1b0cb5f2933806a4e0fbc
Parents: afa314c
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Sep 20 12:54:36 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 15:40:34 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/entitytransform/Action.java | 68 +++++++++++
.../entitytransform/AtlasEntityTransformer.java | 11 +-
.../entitytransform/BaseEntityHandler.java | 101 ++++++++++++++--
.../apache/atlas/entitytransform/Condition.java | 86 +++++++++++++
.../atlas/entitytransform/NeedsContext.java | 23 ++++
.../entitytransform/TransformerContext.java | 47 ++++++++
.../TransformationHandlerTest.java | 120 +++++++++++++------
.../atlas/repository/impexp/ImportService.java | 36 ++----
.../atlas/repository/impexp/ZipSource.java | 6 +-
9 files changed, 424 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index f01c6ce..fa18558 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -17,16 +17,26 @@
*/
package org.apache.atlas.entitytransform;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.commons.lang.StringUtils;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Collections;
+
public abstract class Action {
private static final Logger LOG = LoggerFactory.getLogger(Action.class);
+ private static final String ENTITY_KEY = "__entity";
private static final String ACTION_DELIMITER = ":";
+ private static final String ACTION_ADD_CLASSIFICATION = "ADDCLASSIFICATION";
private static final String ACTION_NAME_SET = "SET";
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
@@ -65,6 +75,10 @@ public abstract class Action {
value = StringUtils.trim(value);
switch (actionName.toUpperCase()) {
+ case ACTION_ADD_CLASSIFICATION:
+ ret = new AddClassificationAction(actionValue);
+ break;
+
case ACTION_NAME_REPLACE_PREFIX:
ret = new PrefixReplaceAction(key, actionValue);
break;
@@ -115,6 +129,60 @@ public abstract class Action {
}
}
+ public static class AddClassificationAction extends Action implements NeedsContext {
+
+ private final String classificationName;
+ private TransformerContext transformerContext;
+
+ public AddClassificationAction(String classificationName) {
+ super(ENTITY_KEY);
+
+ this.classificationName = classificationName;
+ }
+
+ @Override
+ public void apply(AtlasTransformableEntity transformableEntity) {
+ AtlasEntity entity = transformableEntity.entity;
+ if (entity.getClassifications() == null) {
+ entity.setClassifications(new ArrayList<AtlasClassification>());
+ }
+
+ for (AtlasClassification c : entity.getClassifications()) {
+ if (c.getTypeName().equals(classificationName)) {
+ return;
+ }
+ }
+
+ entity.getClassifications().add(new AtlasClassification(classificationName));
+ }
+
+ @Override
+ public void setContext(TransformerContext transformerContext) {
+ this.transformerContext = transformerContext;
+ getCreateTag(classificationName);
+ }
+
+ private void getCreateTag(String classificationName) {
+ if (transformerContext == null) {
+ return;
+ }
+
+ try {
+ AtlasClassificationDef classificationDef = transformerContext.getTypeRegistry().getClassificationDefByName(classificationName);
+ if (classificationDef != null) {
+ return;
+ }
+
+ classificationDef = new AtlasClassificationDef(classificationName);
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ typesDef.setClassificationDefs(Collections.singletonList(classificationDef));
+ transformerContext.getTypeDefStore().createTypesDef(typesDef);
+ LOG.info("created classification: {}", classificationName);
+ } catch (AtlasBaseException e) {
+ LOG.error("Error creating classification: {}", classificationName, e);
+ }
+ }
+ }
public static class PrefixReplaceAction extends Action {
private final String fromPrefix;
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java b/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
index c14f2fd..e9b2afd 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
@@ -19,14 +19,17 @@ package org.apache.atlas.entitytransform;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.apache.atlas.model.impexp.AttributeTransform;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.type.AtlasType;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
+import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
-
-
public class AtlasEntityTransformer {
private final List<Condition> conditions;
private final List<Action> actions;
@@ -35,6 +38,10 @@ public class AtlasEntityTransformer {
this(attributeTransform.getConditions(), attributeTransform.getAction());
}
+ public AtlasEntityTransformer(AtlasObjectId objectId, Map<String, String> actions) {
+ this(Collections.singletonMap("__entity", AtlasType.toJson(objectId)), actions);
+ }
+
public AtlasEntityTransformer(Map<String, String> conditions, Map<String, String> actions) {
this.conditions = createConditions(conditions);
this.actions = createActions(actions);
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index 9d44043..dd6c665 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -17,9 +17,14 @@
*/
package org.apache.atlas.entitytransform;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -33,6 +38,7 @@ public class BaseEntityHandler {
protected final List<AtlasEntityTransformer> transformers;
protected final boolean hasCustomAttributeTransformer;
+ private TransformerContext transformerContext;
public BaseEntityHandler(List<AtlasEntityTransformer> transformers) {
this(transformers, null);
@@ -48,26 +54,45 @@ public class BaseEntityHandler {
}
public AtlasEntity transform(AtlasEntity entity) {
- if (CollectionUtils.isNotEmpty(transformers)) {
- AtlasTransformableEntity transformableEntity = getTransformableEntity(entity);
+ if (!CollectionUtils.isNotEmpty(transformers)) {
+ return entity;
+ }
- if (transformableEntity != null) {
- for (AtlasEntityTransformer transformer : transformers) {
- transformer.transform(transformableEntity);
- }
+ AtlasTransformableEntity transformableEntity = getTransformableEntity(entity);
+ if (transformableEntity == null) {
+ return entity;
+ }
- transformableEntity.transformComplete();
- }
+ for (AtlasEntityTransformer transformer : transformers) {
+ transformer.transform(transformableEntity);
}
+ transformableEntity.transformComplete();
+
return entity;
}
+ private void setContextForActions(List<Action> actions) {
+ for(Action action : actions) {
+ if (action instanceof NeedsContext) {
+ ((NeedsContext) action).setContext(transformerContext);
+ }
+ }
+ }
+
+ private void setContextForConditions(List<Condition> conditions) {
+ for(Condition condition : conditions) {
+ if (condition instanceof NeedsContext) {
+ ((NeedsContext) condition).setContext(transformerContext);
+ }
+ }
+ }
+
public AtlasTransformableEntity getTransformableEntity(AtlasEntity entity) {
return new AtlasTransformableEntity(entity);
}
- public static List<BaseEntityHandler> createEntityHandlers(List<AttributeTransform> transforms) {
+ public static List<BaseEntityHandler> createEntityHandlers(List<AttributeTransform> transforms, TransformerContext context) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> BaseEntityHandler.createEntityHandlers(transforms={})", transforms);
}
@@ -92,10 +117,18 @@ public class BaseEntityHandler {
for (BaseEntityHandler handler : handlers) {
if (handler.hasCustomAttributeTransformer()) {
ret.add(handler);
+ handler.setContext(context);
}
}
if (CollectionUtils.isEmpty(ret)) {
+ BaseEntityHandler be = new BaseEntityHandler(transformers);
+ be.setContext(context);
+
+ ret.add(be);
+ }
+
+ if (CollectionUtils.isEmpty(ret)) {
ret.add(new BaseEntityHandler(transformers));
}
@@ -119,7 +152,20 @@ public class BaseEntityHandler {
return false;
}
+ public void setContext(AtlasTypeRegistry typeRegistry, AtlasTypeDefStore typeDefStore, AtlasExportRequest request) {
+ setContext(new TransformerContext(typeRegistry, typeDefStore, request));
+ }
+ public void setContext(TransformerContext context) {
+ this.transformerContext = context;
+
+ for (AtlasEntityTransformer transformer : transformers) {
+ if (transformerContext != null) {
+ setContextForActions(transformer.getActions());
+ setContextForConditions(transformer.getConditions());
+ }
+ }
+ }
public static class AtlasTransformableEntity {
protected final AtlasEntity entity;
@@ -170,4 +216,41 @@ public class BaseEntityHandler {
// implementations can override to set value of computed-attributes
}
}
+
+ public static List<BaseEntityHandler> fromJson(String transformersString, TransformerContext context) {
+ if (StringUtils.isEmpty(transformersString)) {
+ return null;
+ }
+
+ Object transformersObj = AtlasType.fromJson(transformersString, Object.class);
+ List transformers = (transformersObj != null && transformersObj instanceof List) ? (List) transformersObj : null;
+
+ List<AttributeTransform> attributeTransforms = new ArrayList<>();
+
+ if (CollectionUtils.isEmpty(transformers)) {
+ return null;
+ }
+
+ for (Object transformer : transformers) {
+ String transformerStr = AtlasType.toJson(transformer);
+ AttributeTransform attributeTransform = AtlasType.fromJson(transformerStr, AttributeTransform.class);
+
+ if (attributeTransform == null) {
+ continue;
+ }
+
+ attributeTransforms.add(attributeTransform);
+ }
+
+ if (CollectionUtils.isEmpty(attributeTransforms)) {
+ return null;
+ }
+
+ List<BaseEntityHandler> entityHandlers = createEntityHandlers(attributeTransforms, context);
+ if (CollectionUtils.isEmpty(entityHandlers)) {
+ return null;
+ }
+
+ return entityHandlers;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index bc63079..174b9b4 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -18,15 +18,25 @@
package org.apache.atlas.entitytransform;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
public abstract class Condition {
private static final Logger LOG = LoggerFactory.getLogger(Condition.class);
private static final String CONDITION_DELIMITER = ":";
+ private static final String CONDITION_ENTITY_OBJECT_ID = "OBJECTID";
+ private static final String CONDITION_ENTITY_TOP_LEVEL = "TOPLEVEL";
+ private static final String CONDITION_ENTITY_ALL = "ALL";
private static final String CONDITION_NAME_EQUALS = "EQUALS";
private static final String CONDITION_NAME_EQUALS_IGNORE_CASE = "EQUALS_IGNORE_CASE";
private static final String CONDITION_NAME_STARTS_WITH = "STARTS_WITH";
@@ -60,6 +70,18 @@ public abstract class Condition {
value = StringUtils.trim(value);
switch (conditionName.toUpperCase()) {
+ case CONDITION_ENTITY_ALL:
+ ret = new ObjectIdEquals(key, CONDITION_ENTITY_ALL);
+ break;
+
+ case CONDITION_ENTITY_TOP_LEVEL:
+ ret = new ObjectIdEquals(key, CONDITION_ENTITY_TOP_LEVEL);
+ break;
+
+ case CONDITION_ENTITY_OBJECT_ID:
+ ret = new ObjectIdEquals(key, conditionValue);
+ break;
+
case CONDITION_NAME_EQUALS:
ret = new EqualsCondition(key, conditionValue);
break;
@@ -164,6 +186,70 @@ public abstract class Condition {
}
}
+ static class ObjectIdEquals extends Condition implements NeedsContext {
+ private final List<AtlasObjectId> objectIds;
+ private String scope;
+ private TransformerContext transformerContext;
+
+ public ObjectIdEquals(String key, String conditionValue) {
+ super(key);
+
+ objectIds = new ArrayList<>();
+ this.scope = conditionValue;
+ }
+
+ @Override
+ public boolean matches(AtlasTransformableEntity entity) {
+ for (AtlasObjectId objectId : objectIds) {
+ return isMatch(objectId, entity.entity);
+ }
+
+ return objectIds.size() == 0;
+ }
+
+ public void add(AtlasObjectId objectId) {
+ this.objectIds.add(objectId);
+ }
+
+ private boolean isMatch(AtlasObjectId objectId, AtlasEntity entity) {
+ boolean ret = true;
+ if (!StringUtils.isEmpty(objectId.getGuid())) {
+ return Objects.equals(objectId.getGuid(), entity.getGuid());
+ }
+
+ ret = Objects.equals(objectId.getTypeName(), entity.getTypeName());
+ if (!ret) {
+ return ret;
+ }
+
+ for (Map.Entry<String, Object> entry : objectId.getUniqueAttributes().entrySet()) {
+ ret = ret && Objects.equals(entity.getAttribute(entry.getKey()), entry.getValue());
+ if (!ret) {
+ break;
+ }
+ }
+
+ return ret;
+ }
+
+ @Override
+ public void setContext(TransformerContext transformerContext) {
+ this.transformerContext = transformerContext;
+ if(StringUtils.isEmpty(scope) || scope.equals(CONDITION_ENTITY_ALL)) {
+ return;
+ }
+
+ addObjectIdsFromExportRequest();
+ }
+
+ private void addObjectIdsFromExportRequest() {
+ for(AtlasObjectId objectId : this.transformerContext.getExportRequest().getItemsToExport()) {
+ add(objectId);
+ }
+ }
+ }
+
+
public static class HasValueCondition extends Condition {
protected final String attributeValue;
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java b/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
new file mode 100644
index 0000000..5c16bcf
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.entitytransform;
+
+public interface NeedsContext {
+ void setContext(TransformerContext transformerContext);
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java b/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
new file mode 100644
index 0000000..a7a77b5
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.entitytransform;
+
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+
+public class TransformerContext {
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasTypeDefStore typeDefStore;
+ private final AtlasExportRequest exportRequest;
+
+ public TransformerContext(AtlasTypeRegistry typeRegistry, AtlasTypeDefStore typeDefStore, AtlasExportRequest exportRequest) {
+ this.typeRegistry = typeRegistry;
+ this.typeDefStore = typeDefStore;
+ this.exportRequest = exportRequest;
+ }
+
+ public AtlasTypeRegistry getTypeRegistry() {
+ return this.typeRegistry;
+ }
+
+ public AtlasTypeDefStore getTypeDefStore() {
+ return typeDefStore;
+ }
+
+ public AtlasExportRequest getExportRequest() {
+ return exportRequest;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
index a0ebe59..c76f959 100644
--- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
+++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
@@ -19,8 +19,9 @@ package org.apache.atlas.entitytransform;
import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
-import org.testng.Assert;
import org.testng.annotations.Test;
import java.util.ArrayList;
@@ -30,6 +31,10 @@ import java.util.List;
import java.util.Map;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE;
public class TransformationHandlerTest {
@@ -50,9 +55,9 @@ public class TransformationHandlerTest {
String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
if (endsWithCl1) {
- Assert.assertTrue(transformedValue.endsWith("@cl2"), transformedValue + ": expected to end with @cl2");
+ assertTrue(transformedValue.endsWith("@cl2"), transformedValue + ": expected to end with @cl2");
} else {
- Assert.assertEquals(qualifiedName, transformedValue, "not expected to change");
+ assertEquals(qualifiedName, transformedValue, "not expected to change");
}
}
}
@@ -76,9 +81,9 @@ public class TransformationHandlerTest {
String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
if (endsWithCl1) {
- Assert.assertTrue(transformedValue.endsWith("@CL1"), transformedValue + ": expected to end with @CL1");
+ assertTrue(transformedValue.endsWith("@CL1"), transformedValue + ": expected to end with @CL1");
} else {
- Assert.assertEquals(qualifiedName, transformedValue, "not expected to change");
+ assertEquals(qualifiedName, transformedValue, "not expected to change");
}
}
@@ -97,9 +102,9 @@ public class TransformationHandlerTest {
String transformedValue = (String) hdfsPath.getAttribute("qualifiedName");
if (endsWithCL1) {
- Assert.assertTrue(transformedValue.endsWith("@cl1"), transformedValue + ": expected to end with @cl1");
+ assertTrue(transformedValue.endsWith("@cl1"), transformedValue + ": expected to end with @cl1");
} else {
- Assert.assertEquals(qualifiedName, transformedValue, "not expected to change");
+ assertEquals(qualifiedName, transformedValue, "not expected to change");
}
}
}
@@ -118,7 +123,7 @@ public class TransformationHandlerTest {
String replicatedTo = (String) entity.getAttribute("replicatedTo");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ assertTrue(StringUtils.isNotEmpty(replicatedTo));
}
applyTransforms(entity, handlers);
@@ -126,7 +131,7 @@ public class TransformationHandlerTest {
String transformedValue = (String) entity.getAttribute("replicatedTo");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isEmpty(transformedValue));
+ assertTrue(StringUtils.isEmpty(transformedValue));
}
}
}
@@ -149,8 +154,8 @@ public class TransformationHandlerTest {
String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
- Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ assertTrue(StringUtils.isNotEmpty(replicatedFrom));
}
applyTransforms(entity, handlers);
@@ -159,8 +164,8 @@ public class TransformationHandlerTest {
replicatedFrom = (String) entity.getAttribute("replicatedFrom");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
- Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ assertTrue(StringUtils.isEmpty(replicatedTo));
+ assertTrue(StringUtils.isEmpty(replicatedFrom));
}
}
}
@@ -182,8 +187,8 @@ public class TransformationHandlerTest {
String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
- Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ assertTrue(StringUtils.isNotEmpty(replicatedFrom));
}
applyTransforms(entity, handlers);
@@ -192,8 +197,8 @@ public class TransformationHandlerTest {
replicatedFrom = (String) entity.getAttribute("replicatedFrom");
if (entity.getTypeName() == HIVE_TABLE) {
- Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
- Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ assertTrue(StringUtils.isEmpty(replicatedTo));
+ assertTrue(StringUtils.isEmpty(replicatedFrom));
}
}
}
@@ -215,9 +220,9 @@ public class TransformationHandlerTest {
String transformedValue = (String) hdfsPath.getAttribute("name");
if (startsWith_aa_bb_) {
- Assert.assertTrue(transformedValue.startsWith("/xx/yy/"), transformedValue + ": expected to start with /xx/yy/");
+ assertTrue(transformedValue.startsWith("/xx/yy/"), transformedValue + ": expected to start with /xx/yy/");
} else {
- Assert.assertEquals(name, transformedValue, "not expected to change");
+ assertEquals(name, transformedValue, "not expected to change");
}
}
}
@@ -241,11 +246,11 @@ public class TransformationHandlerTest {
String transformedValue = (String) entity.getAttribute("qualifiedName");
if (!isHdfsPath && endsWithCl1) {
- Assert.assertTrue(transformedValue.endsWith("@cl1_backup"), transformedValue + ": expected to end with @cl1_backup");
+ assertTrue(transformedValue.endsWith("@cl1_backup"), transformedValue + ": expected to end with @cl1_backup");
} else if (!isHdfsPath && containsCl1) {
- Assert.assertTrue(transformedValue.contains("@cl1_backup"), transformedValue + ": expected to contains @cl1_backup");
+ assertTrue(transformedValue.contains("@cl1_backup"), transformedValue + ": expected to contains @cl1_backup");
} else {
- Assert.assertEquals(qualifiedName, transformedValue, "not expected to change");
+ assertEquals(qualifiedName, transformedValue, "not expected to change");
}
}
}
@@ -266,11 +271,11 @@ public class TransformationHandlerTest {
applyTransforms(entity, handlers);
if (startsWithHrDot) {
- Assert.assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup."));
+ assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup."));
} else if (startsWithHrAt) {
- Assert.assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup@"));
+ assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr_backup@"));
} else {
- Assert.assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
}
}
}
@@ -293,11 +298,11 @@ public class TransformationHandlerTest {
applyTransforms(entity, handlers);
if (startsWithHrEmployeesDot) {
- Assert.assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup."));
+ assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup."));
} else if (startsWithHrEmployeesAt) {
- Assert.assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup@"));
+ assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees_backup@"));
} else {
- Assert.assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
}
}
}
@@ -320,15 +325,56 @@ public class TransformationHandlerTest {
applyTransforms(entity, handlers);
if (startsWithHrEmployeesAgeAt) {
- Assert.assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees.age_backup@"));
+ assertTrue(((String) entity.getAttribute("qualifiedName")).startsWith("hr.employees.age_backup@"));
} else {
- Assert.assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ assertEquals(qualifiedName, (String) entity.getAttribute("qualifiedName"), "not expected to change");
+ }
+ }
+ }
+
+ @Test
+ public void verifyAddClassification() {
+ AtlasEntityTransformer entityTransformer = new AtlasEntityTransformer(
+ Collections.singletonMap("hdfs_path.qualifiedName", "EQUALS: hr@cl1"),
+ Collections.singletonMap("__entity", "addClassification: replicated")
+ );
+
+ List<BaseEntityHandler> handlers = new ArrayList<>();
+ handlers.add(new BaseEntityHandler(Collections.singletonList(entityTransformer)));
+ assertApplyTransform(handlers);
+ }
+
+ @Test
+ public void verifyAddClassificationUsingScope() {
+ AtlasObjectId objectId = new AtlasObjectId("hive_db", Collections.singletonMap("qualifiedName", "hr@cl1"));
+ AtlasEntityTransformer entityTransformer = new AtlasEntityTransformer(
+ Collections.singletonMap("__entity", "topLevel: "),
+ Collections.singletonMap("__entity", "addClassification: replicated")
+ );
+
+ List<BaseEntityHandler> handlers = new ArrayList<>();
+ handlers.add(new BaseEntityHandler(Collections.singletonList(entityTransformer)));
+ Condition condition = handlers.get(0).transformers.get(0).getConditions().get(0);
+ Condition.ObjectIdEquals objectIdEquals = (Condition.ObjectIdEquals) condition;
+ objectIdEquals.add(objectId);
+
+ assertApplyTransform(handlers);
+ }
+
+ private void assertApplyTransform(List<BaseEntityHandler> handlers) {
+ for (AtlasEntity entity : getAllEntities()) {
+ applyTransforms(entity, handlers);
+
+ if(entity.getAttribute("qualifiedName").equals("hr@cl1")) {
+ assertNotNull(entity.getClassifications());
+ } else{
+ assertNull(entity.getClassifications());
}
}
}
private List<BaseEntityHandler> initializeHandlers(List<AttributeTransform> params) {
- return BaseEntityHandler.createEntityHandlers(params);
+ return BaseEntityHandler.createEntityHandlers(params, null);
}
private void applyTransforms(AtlasEntity entity, List<BaseEntityHandler> handlers) {
@@ -425,10 +471,12 @@ public class TransformationHandlerTest {
}
private AtlasEntity getHiveTableEntity(String clusterName, String dbName, String tableName) {
+ String qualifiedName = dbName + "." + tableName + "@" + clusterName;
+
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_TABLE);
entity.setAttribute("name", tableName);
- entity.setAttribute("qualifiedName", dbName + "." + tableName + "@" + clusterName);
+ entity.setAttribute("qualifiedName", qualifiedName);
entity.setAttribute("owner", "hive");
entity.setAttribute("temporary", false);
entity.setAttribute("lastAccessTime", "1535656355000");
@@ -442,11 +490,13 @@ public class TransformationHandlerTest {
}
private AtlasEntity getHiveStorageDescriptorEntity(String clusterName, String dbName, String tableName) {
+ String qualifiedName = "hdfs://localhost.localdomain:8020/warehouse/tablespace/managed/hive/" + dbName + ".db" + "/" + tableName;
+
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_STORAGE_DESCRIPTOR);
entity.setAttribute("qualifiedName", dbName + "." + tableName + "@" + clusterName + "_storage");
entity.setAttribute("storedAsSubDirectories", false);
- entity.setAttribute("location", "hdfs://localhost.localdomain:8020/warehouse/tablespace/managed/hive/" + dbName + ".db" + "/" + tableName);
+ entity.setAttribute("location", qualifiedName);
entity.setAttribute("compressed", false);
entity.setAttribute("inputFormat", "org.apache.hadoop.mapred.TextInputFormat");
entity.setAttribute("outputFormat", "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat");
@@ -456,10 +506,12 @@ public class TransformationHandlerTest {
}
private AtlasEntity getHiveColumnEntity(String clusterName, String dbName, String tableName, String columnName) {
+ String qualifiedName = dbName + "." + tableName + "." + columnName + "@" + clusterName;
+
AtlasEntity entity = new AtlasEntity(TransformationConstants.HIVE_COLUMN);
entity.setAttribute("owner", "hive");
- entity.setAttribute("qualifiedName", dbName + "." + tableName + "." + columnName +"@" + clusterName);
+ entity.setAttribute("qualifiedName", qualifiedName);
entity.setAttribute("name", columnName);
entity.setAttribute("position", 1);
entity.setAttribute("type", "string");
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index a09385e..b5d8b7c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -20,10 +20,10 @@ package org.apache.atlas.repository.impexp;
import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.entitytransform.BaseEntityHandler;
+import org.apache.atlas.entitytransform.TransformerContext;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
-import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.graph.BulkImporter;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -42,7 +42,6 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import static org.apache.atlas.model.impexp.AtlasImportRequest.TRANSFORMERS_KEY;
@@ -131,36 +130,19 @@ public class ImportService {
}
- private void setEntityTransformerHandlers(ZipSource source, String transformersString) {
- if (StringUtils.isEmpty(transformersString)) {
+ @VisibleForTesting
+ void setEntityTransformerHandlers(ZipSource source, String transformersJson) throws AtlasBaseException {
+ if (StringUtils.isEmpty(transformersJson)) {
return;
}
- Object transformersObj = AtlasType.fromJson(transformersString, Object.class);
- List transformers = (transformersObj != null && transformersObj instanceof List) ? (List) transformersObj : null;
-
- List<AttributeTransform> attributeTransforms = new ArrayList<>();
-
- if (CollectionUtils.isNotEmpty(transformers)) {
- for (Object transformer : transformers) {
- String transformerStr = AtlasType.toJson(transformer);
- AttributeTransform attributeTransform = AtlasType.fromJson(transformerStr, AttributeTransform.class);
-
- if (attributeTransform == null) {
- continue;
- }
-
- attributeTransforms.add(attributeTransform);
- }
+ TransformerContext context = new TransformerContext(typeRegistry, typeDefStore, source.getExportResult().getRequest());
+ List<BaseEntityHandler> entityHandlers = BaseEntityHandler.fromJson(transformersJson, context);
+ if (CollectionUtils.isEmpty(entityHandlers)) {
+ return;
}
- if (CollectionUtils.isNotEmpty(attributeTransforms)) {
- List<BaseEntityHandler> entityHandlers = BaseEntityHandler.createEntityHandlers(attributeTransforms);
-
- if (CollectionUtils.isNotEmpty(entityHandlers)) {
- source.setEntityHandlers(entityHandlers);
- }
- }
+ source.setEntityHandlers(entityHandlers);
}
private void debugLog(String s, Object... params) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/9d4f9728/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index 1f436ce..a292b96 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -138,10 +138,12 @@ public class ZipSource implements EntityImportStream {
String s = getFromCache(guid);
AtlasEntityWithExtInfo entityWithExtInfo = convertFromJson(AtlasEntityWithExtInfo.class, s);
+ if (importTransform != null) {
+ entityWithExtInfo = importTransform.apply(entityWithExtInfo);
+ }
+
if (entityHandlers != null) {
applyTransformers(entityWithExtInfo);
- } else if (importTransform != null) {
- entityWithExtInfo = importTransform.apply(entityWithExtInfo);
}
return entityWithExtInfo;
[07/17] atlas git commit: ATLAS-2892: Delete by name REST endpoint.
Posted by am...@apache.org.
ATLAS-2892: Delete by name REST endpoint.
Change-Id: I9b0a40b42bc945f51aa098e1b15f28f7d2c9fee2
Signed-off-by: Ashutosh Mestry <am...@hortonworks.com>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/f4dac184
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/f4dac184
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/f4dac184
Branch: refs/heads/master
Commit: f4dac18468b94f56fbe482853982d064fc561cd2
Parents: b04c5bd
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Sep 26 13:02:53 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:26 2018 -0700
----------------------------------------------------------------------
dashboardv2/public/js/models/VTag.js | 6 +--
.../public/js/views/tag/TagLayoutView.js | 2 +-
.../apache/atlas/store/AtlasTypeDefStore.java | 3 ++
.../store/graph/AtlasTypeDefGraphStore.java | 26 +++++++++
.../store/graph/AtlasTypeDefGraphStoreTest.java | 21 ++++++++
.../src/test/resources/json/hiveDBv2.json | 56 ++++++++++++++++++++
.../org/apache/atlas/web/rest/TypesREST.java | 26 +++++++++
7 files changed, 134 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/dashboardv2/public/js/models/VTag.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/models/VTag.js b/dashboardv2/public/js/models/VTag.js
index 384bcc3..d436bb5 100644
--- a/dashboardv2/public/js/models/VTag.js
+++ b/dashboardv2/public/js/models/VTag.js
@@ -49,11 +49,7 @@ define(['require',
return this.constructor.nonCrudOperation.call(this, url, 'DELETE', options);
},
deleteTag: function(options) {
- var url = UrlLinks.classificationDefApiUrl();
- options = _.extend({
- contentType: 'application/json',
- dataType: 'json'
- }, options);
+ var url = UrlLinks.classificationDefApiUrl(options.typeName);
return this.constructor.nonCrudOperation.call(this, url, 'DELETE', options);
},
saveTagAttribute: function(options) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/dashboardv2/public/js/views/tag/TagLayoutView.js
----------------------------------------------------------------------
diff --git a/dashboardv2/public/js/views/tag/TagLayoutView.js b/dashboardv2/public/js/views/tag/TagLayoutView.js
index 7381e64..a5df515 100644
--- a/dashboardv2/public/js/views/tag/TagLayoutView.js
+++ b/dashboardv2/public/js/views/tag/TagLayoutView.js
@@ -578,7 +578,7 @@ define(['require',
structDefs: []
};
deleteTagData.deleteTag({
- data: JSON.stringify(deleteJson),
+ typeName: that.tag,
success: function() {
Utils.notifySuccess({
content: "Classification " + that.tag + Messages.deleteSuccessMessage
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index 025f89a..9a74627 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -99,8 +99,11 @@ public interface AtlasTypeDefStore {
AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException;
+
/* Generic operation */
AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
+
+ void deleteTypeByName(String typeName) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index bd82eb5..b142179 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -598,6 +599,31 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
}
}
+
+ @Override
+ @GraphTransaction
+ public void deleteTypeByName(String typeName) throws AtlasBaseException {
+ AtlasType atlasType = typeRegistry.getType(typeName);
+ if (atlasType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS.TYPE_NAME_NOT_FOUND, typeName);
+ }
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+ AtlasBaseTypeDef baseTypeDef = getByName(typeName);
+
+ if (baseTypeDef instanceof AtlasClassificationDef) {
+ typesDef.setClassificationDefs(Collections.singletonList((AtlasClassificationDef) baseTypeDef));
+ } else if (baseTypeDef instanceof AtlasEntityDef) {
+ typesDef.setEntityDefs(Collections.singletonList((AtlasEntityDef) baseTypeDef));
+ } else if (baseTypeDef instanceof AtlasEnumDef) {
+ typesDef.setEnumDefs(Collections.singletonList((AtlasEnumDef) baseTypeDef));
+ } else if (baseTypeDef instanceof AtlasStructDef) {
+ typesDef.setStructDefs(Collections.singletonList((AtlasStructDef) baseTypeDef));
+ }
+
+ deleteTypesDef(typesDef);
+ }
+
@Override
public AtlasTypesDef searchTypesDef(SearchFilter searchFilter) throws AtlasBaseException {
final AtlasTypesDef typesDef = new AtlasTypesDef();
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
index f1b7736..493ad13 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
@@ -24,10 +24,16 @@ import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.*;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.runner.LocalSolrRunner;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.utils.TestResourceFileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -40,6 +46,8 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.HashSet;
+
+import java.io.IOException;
import java.util.Arrays;
import java.util.Date;
@@ -319,6 +327,19 @@ public class AtlasTypeDefGraphStoreTest {
}
}
+ @Test
+ public void deleteTypeByName() throws IOException {
+ try {
+ final String HIVEDB_v2_JSON = "hiveDBv2";
+ final String hiveDB2 = "hive_db_v2";
+ AtlasTypesDef typesDef = TestResourceFileUtils.readObjectFromJson(".", HIVEDB_v2_JSON, AtlasTypesDef.class);
+ typeDefStore.createTypesDef(typesDef);
+ typeDefStore.deleteTypeByName(hiveDB2);
+ } catch (AtlasBaseException e) {
+ fail("Deletion should've succeeded");
+ }
+ }
+
@Test(dependsOnMethods = "testGet")
public void testCreateWithValidAttributes(){
AtlasTypesDef hiveTypes = TestUtilsV2.defineHiveTypes();
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/repository/src/test/resources/json/hiveDBv2.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/hiveDBv2.json b/repository/src/test/resources/json/hiveDBv2.json
new file mode 100644
index 0000000..f46d5f7
--- /dev/null
+++ b/repository/src/test/resources/json/hiveDBv2.json
@@ -0,0 +1,56 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [{
+ "category": "ENTITY",
+ "name": "hive_db_v2",
+ "typeVersion": "1.0",
+ "attributeDefs": [{
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ }, {
+ "name": "description",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }, {
+ "name": "locationUri",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }, {
+ "name": "owner",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }, {
+ "name": "createTime",
+ "typeName": "int",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }]
+ }]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/f4dac184/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
index 13e6512..2091fdf 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/TypesREST.java
@@ -41,6 +41,7 @@ import javax.inject.Singleton;
import javax.servlet.http.HttpServletRequest;
import javax.ws.rs.*;
import javax.ws.rs.core.Context;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
@@ -410,6 +411,31 @@ public class TypesREST {
}
/**
+ * Delete API for type identified by its name.
+ * @param typeName Name of the type to be deleted.
+ * @throws AtlasBaseException
+ * @HTTP 204 On successful deletion of the requested type definitions
+ * @HTTP 400 On validation failure for any type definitions
+ */
+ @DELETE
+ @Path("/typedef/name/{typeName}")
+ @Consumes(Servlets.JSON_MEDIA_TYPE)
+ @Produces(Servlets.JSON_MEDIA_TYPE)
+ public void deleteAtlasTypeByName(@PathParam("typeName") final String typeName) throws AtlasBaseException {
+ AtlasPerfTracer perf = null;
+
+ try {
+ if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+ perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesREST.deleteAtlasTypeByName(" + typeName + ")");
+ }
+
+ typeDefStore.deleteTypeByName(typeName);
+ } finally {
+ AtlasPerfTracer.log(perf);
+ }
+ }
+
+ /**
* Populate a SearchFilter on the basis of the Query Parameters
* @return
*/
[04/17] atlas git commit: ATLAS-2888: Export & Import Process: Change
Marker Removed Inadvertently
Posted by am...@apache.org.
ATLAS-2888: Export & Import Process: Change Marker Removed Inadvertently
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/b04c5bd2
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/b04c5bd2
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/b04c5bd2
Branch: refs/heads/master
Commit: b04c5bd28c9a1685d42bd2294478ab0f350ac608
Parents: 8639ada
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Sun Sep 23 22:51:08 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:25 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/AuditsWriter.java | 38 ++++++--------------
1 file changed, 10 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/b04c5bd2/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index f72de56..cc10660 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -130,6 +130,10 @@ public class AuditsWriter {
return StringUtils.split(fullName, "$")[1];
}
+ private void saveCurrentServer() throws AtlasBaseException {
+ saveServer(getCurrentClusterName(), getCurrentClusterName());
+ }
+
private class ExportAudits {
private AtlasExportRequest request;
private String targetServerName;
@@ -144,8 +148,10 @@ public class AuditsWriter {
request = result.getRequest();
replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedTo);
- saveServers();
+ saveCurrentServer();
+ targetServerFullName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
+ targetServerName = getServerNameFromFullName(targetServerFullName);
auditService.add(userName, getCurrentClusterName(), targetServerName,
ExportImportAuditEntry.OPERATION_EXPORT,
AtlasType.toJson(result), startTime, endTime, !entityGuids.isEmpty());
@@ -157,16 +163,6 @@ public class AuditsWriter {
updateReplicationAttribute(replicationOptionState, targetServerName, targetServerFullName,
entityGuids, Constants.ATTR_NAME_REPLICATED_TO, result.getChangeMarker());
}
-
- private void saveServers() throws AtlasBaseException {
- saveServer(getCurrentClusterName(), getCurrentClusterName());
-
- targetServerFullName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
- targetServerName = getServerNameFromFullName(targetServerFullName);
- if(StringUtils.isNotEmpty(targetServerName)) {
- saveServer(targetServerName, targetServerFullName);
- }
- }
}
private class ImportAudits {
@@ -183,8 +179,10 @@ public class AuditsWriter {
request = result.getRequest();
replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedFrom);
- saveServers();
+ saveCurrentServer();
+ sourceServerFullName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom);
+ sourceServerName = getServerNameFromFullName(sourceServerFullName);
auditService.add(userName,
sourceServerName, getCurrentClusterName(),
ExportImportAuditEntry.OPERATION_IMPORT,
@@ -197,21 +195,5 @@ public class AuditsWriter {
updateReplicationAttribute(replicationOptionState, sourceServerName, sourceServerFullName, entityGuids,
Constants.ATTR_NAME_REPLICATED_FROM, result.getExportResult().getChangeMarker());
}
-
- private void saveServers() throws AtlasBaseException {
- saveServer(getCurrentClusterName(), getCurrentClusterName());
-
- sourceServerFullName = getClusterNameFromOptionsState();
- sourceServerName = getServerNameFromFullName(sourceServerFullName);
- if(StringUtils.isNotEmpty(sourceServerName)) {
- saveServer(sourceServerName, sourceServerFullName);
- }
- }
-
- private String getClusterNameFromOptionsState() {
- return replicationOptionState
- ? getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom)
- : StringUtils.EMPTY;
- }
}
}
[11/17] atlas git commit: ATLAS-2888: Change marker fix for server
name
Posted by am...@apache.org.
ATLAS-2888: Change marker fix for server name
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6f747203
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6f747203
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6f747203
Branch: refs/heads/master
Commit: 6f74720332d1dfb98c2b3da79516a17a37b5db1c
Parents: 6787141
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Mon Oct 1 16:07:06 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:28 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/AtlasServerService.java | 4 ++--
.../apache/atlas/repository/impexp/AuditsWriter.java | 14 ++++----------
2 files changed, 6 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/6f747203/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 0761441..a3489a8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -70,8 +70,8 @@ public class AtlasServerService {
}
}
- public AtlasServer getCreateAtlasServer(String name, String fullName) throws AtlasBaseException {
- AtlasServer defaultServer = new AtlasServer(name, fullName);
+ public AtlasServer getCreateAtlasServer(String clusterName, String serverFullName) throws AtlasBaseException {
+ AtlasServer defaultServer = new AtlasServer(clusterName, serverFullName);
AtlasServer server = getAtlasServer(defaultServer);
if (server == null) {
return save(defaultServer);
http://git-wip-us.apache.org/repos/asf/atlas/blob/6f747203/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
index 7737197..3612c45 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -93,18 +93,12 @@ public class AuditsWriter {
: StringUtils.EMPTY;
}
- private AtlasServer saveServer(String name, String serverFullName) {
- AtlasServer cluster = new AtlasServer(name, serverFullName);
- return atlasServerService.save(cluster);
- }
-
- private AtlasServer saveServer(String name, String serverFullName,
+ private AtlasServer saveServer(String clusterName, String serverFullName,
String entityGuid,
- long lastModifiedTimestamp) {
+ long lastModifiedTimestamp) throws AtlasBaseException {
- AtlasServer server = new AtlasServer(name, serverFullName);
+ AtlasServer server = atlasServerService.getCreateAtlasServer(clusterName, serverFullName);
server.setAdditionalInfoRepl(entityGuid, lastModifiedTimestamp);
-
if (LOG.isDebugEnabled()) {
LOG.debug("saveServer: {}", server);
}
@@ -138,7 +132,7 @@ public class AuditsWriter {
}
private void saveCurrentServer() throws AtlasBaseException {
- saveServer(getCurrentClusterName(), getCurrentClusterName());
+ atlasServerService.getCreateAtlasServer(getCurrentClusterName(), getCurrentClusterName());
}
private class ExportAudits {
[06/17] atlas git commit: ATLAS-2882: refactored import transformer
to set context in constructor; fixed incorrect objId match
Posted by am...@apache.org.
ATLAS-2882: refactored import transformer to set context in constructor; fixed incorrect objId match
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/31c3bea1
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/31c3bea1
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/31c3bea1
Branch: refs/heads/master
Commit: 31c3bea1316d49fe233a061965cef248a97f1168
Parents: 9d4f972
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Thu Sep 20 17:27:28 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:25 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/entitytransform/Action.java | 150 +++++++++--------
.../entitytransform/AtlasEntityTransformer.java | 27 ++--
.../entitytransform/BaseEntityHandler.java | 138 +++++-----------
.../apache/atlas/entitytransform/Condition.java | 141 ++++++++--------
.../atlas/entitytransform/EntityAttribute.java | 69 ++++++++
.../entitytransform/HdfsPathEntityHandler.java | 28 ++--
.../HiveColumnEntityHandler.java | 28 ++--
.../HiveDatabaseEntityHandler.java | 24 +--
.../HiveStorageDescriptorEntityHandler.java | 26 +--
.../entitytransform/HiveTableEntityHandler.java | 26 +--
.../atlas/entitytransform/NeedsContext.java | 23 ---
.../entitytransform/TransformerContext.java | 8 +-
.../TransformationHandlerTest.java | 160 +++++++++++++++----
13 files changed, 470 insertions(+), 378 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index fa18558..0c6102d 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -22,6 +22,8 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.slf4j.Logger;
@@ -31,35 +33,35 @@ import java.util.ArrayList;
import java.util.Collections;
+
public abstract class Action {
private static final Logger LOG = LoggerFactory.getLogger(Action.class);
- private static final String ENTITY_KEY = "__entity";
private static final String ACTION_DELIMITER = ":";
- private static final String ACTION_ADD_CLASSIFICATION = "ADDCLASSIFICATION";
+ private static final String ACTION_ADD_CLASSIFICATION = "ADD_CLASSIFICATION";
private static final String ACTION_NAME_SET = "SET";
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
private static final String ACTION_NAME_CLEAR = "CLEAR";
- protected final String attributeName;
+ protected final EntityAttribute attribute;
- protected Action(String attributeName) {
- this.attributeName = attributeName;
+ protected Action(EntityAttribute attribute) {
+ this.attribute = attribute;
}
- public String getAttributeName() { return attributeName; }
+ public EntityAttribute getAttribute() { return attribute; }
public boolean isValid() {
- return StringUtils.isNotEmpty(attributeName);
+ return true;
}
public abstract void apply(AtlasTransformableEntity entity);
- public static Action createAction(String key, String value) {
+ public static Action createAction(String key, String value, TransformerContext context) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> Action.createAction(key={}, value={})", key, value);
}
@@ -74,33 +76,35 @@ public abstract class Action {
actionValue = StringUtils.trim(actionValue);
value = StringUtils.trim(value);
+ EntityAttribute attribute = new EntityAttribute(StringUtils.trim(key), context);
+
switch (actionName.toUpperCase()) {
case ACTION_ADD_CLASSIFICATION:
- ret = new AddClassificationAction(actionValue);
- break;
+ ret = new AddClassificationAction(attribute, actionValue, context);
+ break;
case ACTION_NAME_REPLACE_PREFIX:
- ret = new PrefixReplaceAction(key, actionValue);
+ ret = new PrefixReplaceAction(attribute, actionValue);
break;
case ACTION_NAME_TO_LOWER:
- ret = new ToLowerCaseAction(key);
+ ret = new ToLowerCaseAction(attribute);
break;
case ACTION_NAME_TO_UPPER:
- ret = new ToUpperCaseAction(key);
+ ret = new ToUpperCaseAction(attribute);
break;
case ACTION_NAME_SET:
- ret = new SetAction(key, actionValue);
+ ret = new SetAction(attribute, actionValue);
break;
case ACTION_NAME_CLEAR:
- ret = new ClearAction(key);
- break;
+ ret = new ClearAction(attribute);
+ break;
default:
- ret = new SetAction(key, value); // treat unspecified/unknown action as 'SET'
+ ret = new SetAction(attribute, value); // treat unspecified/unknown action as 'SET'
break;
}
@@ -115,71 +119,79 @@ public abstract class Action {
public static class SetAction extends Action {
private final String attributeValue;
- public SetAction(String attributeName, String attributeValue) {
- super(attributeName);
+ public SetAction(EntityAttribute attribute, String attributeValue) {
+ super(attribute);
this.attributeValue = attributeValue;
}
@Override
public void apply(AtlasTransformableEntity entity) {
- if (isValid()) {
- entity.setAttribute(attributeName, attributeValue);
- }
+ entity.setAttribute(attribute, attributeValue);
}
}
- public static class AddClassificationAction extends Action implements NeedsContext {
-
+ public static class AddClassificationAction extends Action {
private final String classificationName;
- private TransformerContext transformerContext;
- public AddClassificationAction(String classificationName) {
- super(ENTITY_KEY);
+ public AddClassificationAction(EntityAttribute attribute, String classificationName, TransformerContext context) {
+ super(attribute);
this.classificationName = classificationName;
+
+ createClassificationDefIfNotExists(classificationName, context);
}
@Override
public void apply(AtlasTransformableEntity transformableEntity) {
- AtlasEntity entity = transformableEntity.entity;
+ AtlasEntity entity = transformableEntity.getEntity();
+
if (entity.getClassifications() == null) {
- entity.setClassifications(new ArrayList<AtlasClassification>());
+ entity.setClassifications(new ArrayList<>());
}
+ boolean hasClassification = false;
+
for (AtlasClassification c : entity.getClassifications()) {
- if (c.getTypeName().equals(classificationName)) {
- return;
+ hasClassification = c.getTypeName().equals(classificationName);
+
+ if (hasClassification) {
+ break;
}
}
- entity.getClassifications().add(new AtlasClassification(classificationName));
+ if (!hasClassification) {
+ entity.getClassifications().add(new AtlasClassification(classificationName));
+ }
}
- @Override
- public void setContext(TransformerContext transformerContext) {
- this.transformerContext = transformerContext;
- getCreateTag(classificationName);
- }
+ private void createClassificationDefIfNotExists(String classificationName, TransformerContext context) {
+ AtlasTypeRegistry typeRegistry = context != null ? context.getTypeRegistry() : null;
- private void getCreateTag(String classificationName) {
- if (transformerContext == null) {
- return;
- }
+ if (typeRegistry != null) {
+ try {
+ AtlasClassificationDef classificationDef = typeRegistry.getClassificationDefByName(classificationName);
- try {
- AtlasClassificationDef classificationDef = transformerContext.getTypeRegistry().getClassificationDefByName(classificationName);
- if (classificationDef != null) {
- return;
- }
+ if (classificationDef == null) {
+ AtlasTypeDefStore typeDefStore = context.getTypeDefStore();
+
+ if (typeDefStore != null) {
+ classificationDef = new AtlasClassificationDef(classificationName);
- classificationDef = new AtlasClassificationDef(classificationName);
- AtlasTypesDef typesDef = new AtlasTypesDef();
- typesDef.setClassificationDefs(Collections.singletonList(classificationDef));
- transformerContext.getTypeDefStore().createTypesDef(typesDef);
- LOG.info("created classification: {}", classificationName);
- } catch (AtlasBaseException e) {
- LOG.error("Error creating classification: {}", classificationName, e);
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+
+ typesDef.setClassificationDefs(Collections.singletonList(classificationDef));
+
+ typeDefStore.createTypesDef(typesDef);
+
+ LOG.info("created classification: {}", classificationName);
+ } else {
+ LOG.warn("skipped creation of classification {}. typeDefStore is null", classificationName);
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.error("Error creating classification: {}", classificationName, e);
+ }
}
}
}
@@ -188,8 +200,8 @@ public abstract class Action {
private final String fromPrefix;
private final String toPrefix;
- public PrefixReplaceAction(String attributeName, String actionValue) {
- super(attributeName);
+ public PrefixReplaceAction(EntityAttribute attribute, String actionValue) {
+ super(attribute);
// actionValue => =:prefixToReplace=replacedValue
if (actionValue != null) {
@@ -224,61 +236,61 @@ public abstract class Action {
@Override
public void apply(AtlasTransformableEntity entity) {
if (isValid()) {
- Object currValue = entity.getAttribute(attributeName);
+ Object currValue = entity.getAttribute(attribute);
String strValue = currValue != null ? currValue.toString() : null;
if (strValue != null && strValue.startsWith(fromPrefix)) {
- entity.setAttribute(attributeName, StringUtils.replace(strValue, fromPrefix, toPrefix, 1));
+ entity.setAttribute(attribute, StringUtils.replace(strValue, fromPrefix, toPrefix, 1));
}
}
}
}
public static class ToLowerCaseAction extends Action {
- public ToLowerCaseAction(String attributeName) {
- super(attributeName);
+ public ToLowerCaseAction(EntityAttribute attribute) {
+ super(attribute);
}
@Override
public void apply(AtlasTransformableEntity entity) {
if (isValid()) {
- Object currValue = entity.getAttribute(attributeName);
+ Object currValue = entity.getAttribute(attribute);
String strValue = currValue instanceof String ? (String) currValue : null;
if (strValue != null) {
- entity.setAttribute(attributeName, strValue.toLowerCase());
+ entity.setAttribute(attribute, strValue.toLowerCase());
}
}
}
}
public static class ToUpperCaseAction extends Action {
- public ToUpperCaseAction(String attributeName) {
- super(attributeName);
+ public ToUpperCaseAction(EntityAttribute attribute) {
+ super(attribute);
}
@Override
public void apply(AtlasTransformableEntity entity) {
if (isValid()) {
- Object currValue = entity.getAttribute(attributeName);
+ Object currValue = entity.getAttribute(attribute);
String strValue = currValue instanceof String ? (String) currValue : null;
if (strValue != null) {
- entity.setAttribute(attributeName, strValue.toUpperCase());
+ entity.setAttribute(attribute, strValue.toUpperCase());
}
}
}
}
public static class ClearAction extends Action {
- public ClearAction(String attributeName) {
- super(attributeName);
+ public ClearAction(EntityAttribute attribute) {
+ super(attribute);
}
@Override
public void apply(AtlasTransformableEntity entity) {
- if (isValid() && entity.hasAttribute(attributeName)) {
- entity.setAttribute(attributeName, null);
+ if (isValid() && entity.hasAttribute(attribute)) {
+ entity.setAttribute(attribute, null);
}
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java b/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
index e9b2afd..27a57a6 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/AtlasEntityTransformer.java
@@ -19,32 +19,25 @@ package org.apache.atlas.entitytransform;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
import org.apache.atlas.model.impexp.AttributeTransform;
-import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.type.AtlasType;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
-import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
+
public class AtlasEntityTransformer {
private final List<Condition> conditions;
private final List<Action> actions;
- public AtlasEntityTransformer(AttributeTransform attributeTransform) {
- this(attributeTransform.getConditions(), attributeTransform.getAction());
- }
- public AtlasEntityTransformer(AtlasObjectId objectId, Map<String, String> actions) {
- this(Collections.singletonMap("__entity", AtlasType.toJson(objectId)), actions);
+ public AtlasEntityTransformer(AttributeTransform attributeTransform, TransformerContext context) {
+ this(attributeTransform.getConditions(), attributeTransform.getAction(), context);
}
- public AtlasEntityTransformer(Map<String, String> conditions, Map<String, String> actions) {
- this.conditions = createConditions(conditions);
- this.actions = createActions(actions);
+ public AtlasEntityTransformer(Map<String, String> conditions, Map<String, String> actions, TransformerContext context) {
+ this.conditions = createConditions(conditions, context);
+ this.actions = createActions(actions, context);
}
public List<Condition> getConditions() {
@@ -71,12 +64,12 @@ public class AtlasEntityTransformer {
}
}
- private List<Condition> createConditions(Map<String, String> conditions) {
+ private List<Condition> createConditions(Map<String, String> conditions, TransformerContext context) {
List<Condition> ret = new ArrayList<>();
if (MapUtils.isNotEmpty(conditions)) {
for (Map.Entry<String, String> entry : conditions.entrySet()) {
- Condition condition = Condition.createCondition(entry.getKey(), entry.getValue());
+ Condition condition = Condition.createCondition(entry.getKey(), entry.getValue(), context);
ret.add(condition);
}
@@ -85,12 +78,12 @@ public class AtlasEntityTransformer {
return ret;
}
- private List<Action> createActions(Map<String, String> actions) {
+ private List<Action> createActions(Map<String, String> actions, TransformerContext context) {
List<Action> ret = new ArrayList<>();
if (MapUtils.isNotEmpty(actions)) {
for (Map.Entry<String, String> entry : actions.entrySet()) {
- Action action = Action.createAction(entry.getKey(), entry.getValue());
+ Action action = Action.createAction(entry.getKey(), entry.getValue(), context);
ret.add(action);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index dd6c665..975e4dd 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -17,12 +17,9 @@
*/
package org.apache.atlas.entitytransform;
-import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
-import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -31,34 +28,24 @@ import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
-import static org.apache.atlas.entitytransform.TransformationConstants.TYPE_NAME_ATTRIBUTE_NAME_SEP;
public class BaseEntityHandler {
private static final Logger LOG = LoggerFactory.getLogger(BaseEntityHandler.class);
protected final List<AtlasEntityTransformer> transformers;
- protected final boolean hasCustomAttributeTransformer;
- private TransformerContext transformerContext;
- public BaseEntityHandler(List<AtlasEntityTransformer> transformers) {
- this(transformers, null);
- }
- public BaseEntityHandler(List<AtlasEntityTransformer> transformers, List<String> customTransformAttributes) {
- this.transformers = transformers;
- this.hasCustomAttributeTransformer = hasTransformerForAnyAttribute(customTransformAttributes);
- }
-
- public boolean hasCustomAttributeTransformer() {
- return hasCustomAttributeTransformer;
+ public BaseEntityHandler(List<AtlasEntityTransformer> transformers) {
+ this.transformers = transformers;
}
public AtlasEntity transform(AtlasEntity entity) {
- if (!CollectionUtils.isNotEmpty(transformers)) {
+ if (CollectionUtils.isEmpty(transformers)) {
return entity;
}
AtlasTransformableEntity transformableEntity = getTransformableEntity(entity);
+
if (transformableEntity == null) {
return entity;
}
@@ -72,22 +59,6 @@ public class BaseEntityHandler {
return entity;
}
- private void setContextForActions(List<Action> actions) {
- for(Action action : actions) {
- if (action instanceof NeedsContext) {
- ((NeedsContext) action).setContext(transformerContext);
- }
- }
- }
-
- private void setContextForConditions(List<Condition> conditions) {
- for(Condition condition : conditions) {
- if (condition instanceof NeedsContext) {
- ((NeedsContext) condition).setContext(transformerContext);
- }
- }
- }
-
public AtlasTransformableEntity getTransformableEntity(AtlasEntity entity) {
return new AtlasTransformableEntity(entity);
}
@@ -97,39 +68,38 @@ public class BaseEntityHandler {
LOG.debug("==> BaseEntityHandler.createEntityHandlers(transforms={})", transforms);
}
- List<AtlasEntityTransformer> transformers = new ArrayList<>();
+ List<BaseEntityHandler> ret = new ArrayList<>();
- for (AttributeTransform transform : transforms) {
- transformers.add(new AtlasEntityTransformer(transform));
- }
+ if (CollectionUtils.isNotEmpty(transforms)) {
+ List<AtlasEntityTransformer> transformers = new ArrayList<>();
- BaseEntityHandler[] handlers = new BaseEntityHandler[] {
- new HdfsPathEntityHandler(transformers),
- new HiveDatabaseEntityHandler(transformers),
- new HiveTableEntityHandler(transformers),
- new HiveColumnEntityHandler(transformers),
- new HiveStorageDescriptorEntityHandler(transformers)
- };
+ for (AttributeTransform transform : transforms) {
+ transformers.add(new AtlasEntityTransformer(transform, context));
+ }
- List<BaseEntityHandler> ret = new ArrayList<>();
+ if (hasTransformerForAnyAttribute(transformers, HdfsPathEntityHandler.CUSTOM_TRANSFORM_ATTRIBUTES)) {
+ ret.add(new HdfsPathEntityHandler(transformers));
+ }
- // include customer handlers, only if its customer attribute is transformed
- for (BaseEntityHandler handler : handlers) {
- if (handler.hasCustomAttributeTransformer()) {
- ret.add(handler);
- handler.setContext(context);
+ if (hasTransformerForAnyAttribute(transformers, HiveDatabaseEntityHandler.CUSTOM_TRANSFORM_ATTRIBUTES)) {
+ ret.add(new HiveDatabaseEntityHandler(transformers));
}
- }
- if (CollectionUtils.isEmpty(ret)) {
- BaseEntityHandler be = new BaseEntityHandler(transformers);
- be.setContext(context);
+ if (hasTransformerForAnyAttribute(transformers, HiveTableEntityHandler.CUSTOM_TRANSFORM_ATTRIBUTES)) {
+ ret.add(new HiveTableEntityHandler(transformers));
+ }
- ret.add(be);
- }
+ if (hasTransformerForAnyAttribute(transformers, HiveColumnEntityHandler.CUSTOM_TRANSFORM_ATTRIBUTES)) {
+ ret.add(new HiveColumnEntityHandler(transformers));
+ }
+
+ if (hasTransformerForAnyAttribute(transformers, HiveStorageDescriptorEntityHandler.CUSTOM_TRANSFORM_ATTRIBUTES)) {
+ ret.add(new HiveStorageDescriptorEntityHandler(transformers));
+ }
- if (CollectionUtils.isEmpty(ret)) {
- ret.add(new BaseEntityHandler(transformers));
+ if (CollectionUtils.isEmpty(ret)) {
+ ret.add(new BaseEntityHandler(transformers));
+ }
}
if (LOG.isDebugEnabled()) {
@@ -139,11 +109,11 @@ public class BaseEntityHandler {
return ret;
}
- private boolean hasTransformerForAnyAttribute(List<String> attributes) {
+ private static boolean hasTransformerForAnyAttribute(List<AtlasEntityTransformer> transformers, List<String> attributes) {
if (CollectionUtils.isNotEmpty(transformers) && CollectionUtils.isNotEmpty(attributes)) {
for (AtlasEntityTransformer transformer : transformers) {
for (Action action : transformer.getActions()) {
- if (attributes.contains(action.getAttributeName())) {
+ if (attributes.contains(action.getAttribute().getAttributeKey())) {
return true;
}
}
@@ -152,20 +122,6 @@ public class BaseEntityHandler {
return false;
}
- public void setContext(AtlasTypeRegistry typeRegistry, AtlasTypeDefStore typeDefStore, AtlasExportRequest request) {
- setContext(new TransformerContext(typeRegistry, typeDefStore, request));
- }
-
- public void setContext(TransformerContext context) {
- this.transformerContext = context;
-
- for (AtlasEntityTransformer transformer : transformers) {
- if (transformerContext != null) {
- setContextForActions(transformer.getActions());
- setContextForConditions(transformer.getConditions());
- }
- }
- }
public static class AtlasTransformableEntity {
protected final AtlasEntity entity;
@@ -178,38 +134,26 @@ public class BaseEntityHandler {
return entity;
}
- public Object getAttribute(String attributeName) {
- Object ret = null;
-
- if (entity != null && attributeName != null) {
- ret = entity.getAttribute(attributeName);
+ public Object getAttribute(EntityAttribute attribute) {
+ final Object ret;
- if (ret == null) { // try after dropping typeName prefix, if attributeName contains it
- int idxSep = attributeName.indexOf(TYPE_NAME_ATTRIBUTE_NAME_SEP);
-
- if (idxSep != -1) {
- ret = entity.getAttribute(attributeName.substring(idxSep + 1));
- }
- }
+ if (attribute.appliesToEntityType(entity.getTypeName())) {
+ ret = entity.getAttribute(attribute.getAttributeName());
+ } else {
+ ret = null;
}
return ret;
}
- public void setAttribute(String attributeName, String attributeValue) {
- if (entity != null && attributeName != null) {
- int idxSep = attributeName.indexOf(TYPE_NAME_ATTRIBUTE_NAME_SEP); // drop typeName prefix, if attributeName contains it
-
- if (idxSep != -1) {
- entity.setAttribute(attributeName.substring(idxSep + 1), attributeValue);
- } else {
- entity.setAttribute(attributeName, attributeValue);
- }
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ if (attribute.appliesToEntityType(entity.getTypeName())) {
+ entity.setAttribute(attribute.getAttributeName(), attributeValue);
}
}
- public boolean hasAttribute(String attributeName) {
- return getAttribute(attributeName) != null;
+ public boolean hasAttribute(EntityAttribute attribute) {
+ return getAttribute(attribute) != null;
}
public void transformComplete() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index 174b9b4..3bf49f0 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -17,7 +17,9 @@
*/
package org.apache.atlas.entitytransform;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.entitytransform.BaseEntityHandler.AtlasTransformableEntity;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.commons.lang.StringUtils;
@@ -30,6 +32,7 @@ import java.util.Map;
import java.util.Objects;
+
public abstract class Condition {
private static final Logger LOG = LoggerFactory.getLogger(Condition.class);
@@ -43,18 +46,19 @@ public abstract class Condition {
private static final String CONDITION_NAME_STARTS_WITH_IGNORE_CASE = "STARTS_WITH_IGNORE_CASE";
private static final String CONDITION_NAME_HAS_VALUE = "HAS_VALUE";
- protected final String attributeName;
+ protected final EntityAttribute attribute;
+
- protected Condition(String attributeName) {
- this.attributeName = attributeName;
+ protected Condition(EntityAttribute attribute) {
+ this.attribute = attribute;
}
- public String getAttributeName() { return attributeName; }
+ public EntityAttribute getAttribute() { return attribute; }
public abstract boolean matches(AtlasTransformableEntity entity);
- public static Condition createCondition(String key, String value) {
+ public static Condition createCondition(String key, String value, TransformerContext context) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> Condition.createCondition(key={}, value={})", key, value);
}
@@ -69,41 +73,43 @@ public abstract class Condition {
conditionValue = StringUtils.trim(conditionValue);
value = StringUtils.trim(value);
+ EntityAttribute attribute = new EntityAttribute(StringUtils.trim(key), context);
+
switch (conditionName.toUpperCase()) {
case CONDITION_ENTITY_ALL:
- ret = new ObjectIdEquals(key, CONDITION_ENTITY_ALL);
+ ret = new ObjectIdEquals(attribute, CONDITION_ENTITY_ALL, context);
break;
case CONDITION_ENTITY_TOP_LEVEL:
- ret = new ObjectIdEquals(key, CONDITION_ENTITY_TOP_LEVEL);
+ ret = new ObjectIdEquals(attribute, CONDITION_ENTITY_TOP_LEVEL, context);
break;
case CONDITION_ENTITY_OBJECT_ID:
- ret = new ObjectIdEquals(key, conditionValue);
+ ret = new ObjectIdEquals(attribute, conditionValue, context);
break;
case CONDITION_NAME_EQUALS:
- ret = new EqualsCondition(key, conditionValue);
+ ret = new EqualsCondition(attribute, conditionValue);
break;
case CONDITION_NAME_EQUALS_IGNORE_CASE:
- ret = new EqualsIgnoreCaseCondition(key, conditionValue);
+ ret = new EqualsIgnoreCaseCondition(attribute, conditionValue);
break;
case CONDITION_NAME_STARTS_WITH:
- ret = new StartsWithCondition(key, conditionValue);
+ ret = new StartsWithCondition(attribute, conditionValue);
break;
case CONDITION_NAME_STARTS_WITH_IGNORE_CASE:
- ret = new StartsWithIgnoreCaseCondition(key, conditionValue);
+ ret = new StartsWithIgnoreCaseCondition(attribute, conditionValue);
break;
case CONDITION_NAME_HAS_VALUE:
- ret = new HasValueCondition(key, conditionValue);
+ ret = new HasValueCondition(attribute);
break;
default:
- ret = new EqualsCondition(key, value); // treat unspecified/unknown condition as 'EQUALS'
+ ret = new EqualsCondition(attribute, value); // treat unspecified/unknown condition as 'EQUALS'
break;
}
@@ -118,15 +124,15 @@ public abstract class Condition {
public static class EqualsCondition extends Condition {
protected final String attributeValue;
- public EqualsCondition(String attributeName, String attributeValue) {
- super(attributeName);
+ public EqualsCondition(EntityAttribute attribute, String attributeValue) {
+ super(attribute);
this.attributeValue = attributeValue;
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+ Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
return attributeValue != null && StringUtils.equals(attributeValue.toString(), this.attributeValue);
}
@@ -136,15 +142,15 @@ public abstract class Condition {
public static class EqualsIgnoreCaseCondition extends Condition {
protected final String attributeValue;
- public EqualsIgnoreCaseCondition(String attributeName, String attributeValue) {
- super(attributeName);
+ public EqualsIgnoreCaseCondition(EntityAttribute attribute, String attributeValue) {
+ super(attribute);
this.attributeValue = attributeValue;
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+ Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
return attributeValue != null && StringUtils.equalsIgnoreCase(attributeValue.toString(), this.attributeValue);
}
@@ -154,15 +160,15 @@ public abstract class Condition {
public static class StartsWithCondition extends Condition {
protected final String prefix;
- public StartsWithCondition(String attributeName, String prefix) {
- super(attributeName);
+ public StartsWithCondition(EntityAttribute attribute, String prefix) {
+ super(attribute);
this.prefix = prefix;
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+ Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
return attributeValue != null && StringUtils.startsWith(attributeValue.toString(), this.prefix);
}
@@ -172,96 +178,89 @@ public abstract class Condition {
public static class StartsWithIgnoreCaseCondition extends Condition {
protected final String prefix;
- public StartsWithIgnoreCaseCondition(String attributeName, String prefix) {
- super(attributeName);
+ public StartsWithIgnoreCaseCondition(EntityAttribute attribute, String prefix) {
+ super(attribute);
this.prefix = prefix;
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+ Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
return attributeValue != null && StringUtils.startsWithIgnoreCase(attributeValue.toString(), this.prefix);
}
}
- static class ObjectIdEquals extends Condition implements NeedsContext {
+ static class ObjectIdEquals extends Condition {
+ private final boolean isMatchAll;
private final List<AtlasObjectId> objectIds;
- private String scope;
- private TransformerContext transformerContext;
- public ObjectIdEquals(String key, String conditionValue) {
- super(key);
+ public ObjectIdEquals(EntityAttribute attribute, String scope, TransformerContext context) {
+ super(attribute);
- objectIds = new ArrayList<>();
- this.scope = conditionValue;
+ this.isMatchAll = StringUtils.equals(scope, CONDITION_ENTITY_ALL);
+ this.objectIds = new ArrayList<>();
+
+ if (!isMatchAll && context != null && context.getExportRequest() != null) {
+ AtlasExportRequest request = context.getExportRequest();
+
+ for(AtlasObjectId objectId : request.getItemsToExport()) {
+ addObjectId(objectId);
+ }
+ }
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- for (AtlasObjectId objectId : objectIds) {
- return isMatch(objectId, entity.entity);
- }
+ if (isMatchAll) {
+ return true;
+ } else {
+ for (AtlasObjectId objectId : objectIds) {
+ if (isMatch(objectId, entity.getEntity())) {
+ return true;
+ }
+ }
- return objectIds.size() == 0;
+ return false;
+ }
}
- public void add(AtlasObjectId objectId) {
- this.objectIds.add(objectId);
+ @VisibleForTesting
+ void addObjectId(AtlasObjectId objId) {
+ this.objectIds.add(objId);
}
private boolean isMatch(AtlasObjectId objectId, AtlasEntity entity) {
- boolean ret = true;
if (!StringUtils.isEmpty(objectId.getGuid())) {
return Objects.equals(objectId.getGuid(), entity.getGuid());
}
- ret = Objects.equals(objectId.getTypeName(), entity.getTypeName());
- if (!ret) {
- return ret;
- }
+ boolean ret = Objects.equals(objectId.getTypeName(), entity.getTypeName());
+
+ if (ret) {
+ for (Map.Entry<String, Object> entry : objectId.getUniqueAttributes().entrySet()) {
+ ret = ret && Objects.equals(entity.getAttribute(entry.getKey()), entry.getValue());
- for (Map.Entry<String, Object> entry : objectId.getUniqueAttributes().entrySet()) {
- ret = ret && Objects.equals(entity.getAttribute(entry.getKey()), entry.getValue());
- if (!ret) {
- break;
+ if (!ret) {
+ break;
+ }
}
}
return ret;
}
-
- @Override
- public void setContext(TransformerContext transformerContext) {
- this.transformerContext = transformerContext;
- if(StringUtils.isEmpty(scope) || scope.equals(CONDITION_ENTITY_ALL)) {
- return;
- }
-
- addObjectIdsFromExportRequest();
- }
-
- private void addObjectIdsFromExportRequest() {
- for(AtlasObjectId objectId : this.transformerContext.getExportRequest().getItemsToExport()) {
- add(objectId);
- }
- }
}
public static class HasValueCondition extends Condition {
- protected final String attributeValue;
-
- public HasValueCondition(String attributeName, String attributeValue) {
- super(attributeName);
-
- this.attributeValue = attributeValue;
+ public HasValueCondition(EntityAttribute attribute) {
+ super(attribute);
}
@Override
public boolean matches(AtlasTransformableEntity entity) {
- Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+ Object attributeValue = entity != null ? entity.getAttribute(attribute) : null;
return attributeValue != null ? StringUtils.isNotEmpty(attributeValue.toString()) : false;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/EntityAttribute.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/EntityAttribute.java b/intg/src/main/java/org/apache/atlas/entitytransform/EntityAttribute.java
new file mode 100644
index 0000000..040c7cb
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/EntityAttribute.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.entitytransform;
+
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+
+import static org.apache.atlas.entitytransform.TransformationConstants.TYPE_NAME_ATTRIBUTE_NAME_SEP;
+
+public class EntityAttribute {
+ private final String attributeKey;
+ private final AtlasEntityType entityType;
+ private final String attributeName;
+
+ public EntityAttribute(String attributeKey, TransformerContext context) {
+ this.attributeKey = attributeKey;
+
+ int idx = attributeKey != null ? attributeKey.indexOf(TYPE_NAME_ATTRIBUTE_NAME_SEP) : -1;
+
+ if (idx != -1) {
+ this.attributeName = StringUtils.trim(attributeKey.substring(idx + 1));
+
+ AtlasTypeRegistry typeRegistry = context != null ? context.getTypeRegistry() : null;
+
+ if (typeRegistry != null) {
+ String typeName = StringUtils.trim(attributeKey.substring(0, idx));
+
+ this.entityType = typeRegistry.getEntityTypeByName(typeName);
+ } else {
+ this.entityType = null;
+ }
+ } else {
+ this.entityType = null;
+ this.attributeName = attributeKey;
+ }
+ }
+
+ public String getAttributeKey() {
+ return attributeKey;
+ }
+
+ public AtlasEntityType getEntityType() {
+ return entityType;
+ }
+
+ public String getAttributeName() {
+ return attributeName;
+ }
+
+ public boolean appliesToEntityType(String typeName) {
+ return entityType == null || StringUtils.isEmpty(typeName) || entityType.getTypeAndAllSubTypes().contains(typeName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
index 1a398ea..2df98a6 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HdfsPathEntityHandler.java
@@ -35,10 +35,10 @@ import static org.apache.atlas.entitytransform.TransformationConstants.QUALIFIED
public class HdfsPathEntityHandler extends BaseEntityHandler {
- private static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HDFS_PATH_NAME_ATTRIBUTE, HDFS_PATH_PATH_ATTRIBUTE, HDFS_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HDFS_PATH_NAME_ATTRIBUTE, HDFS_PATH_PATH_ATTRIBUTE, HDFS_CLUSTER_NAME_ATTRIBUTE);
public HdfsPathEntityHandler(List<AtlasEntityTransformer> transformers) {
- super(transformers, CUSTOM_TRANSFORM_ATTRIBUTES);
+ super(transformers);
}
@Override
@@ -56,8 +56,8 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
private String path;
private String name;
private String pathPrefix;
- private boolean isPathUpdated = false;
- private boolean isCustomerAttributeUpdated = false;
+ private boolean isPathUpdated = false;
+ private boolean isCustomAttributeUpdated = false;
public HdfsPathEntity(AtlasEntity entity) {
@@ -93,8 +93,8 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
}
@Override
- public Object getAttribute(String attributeName) {
- switch (attributeName) {
+ public Object getAttribute(EntityAttribute attribute) {
+ switch (attribute.getAttributeKey()) {
case HDFS_CLUSTER_NAME_ATTRIBUTE:
return clusterName;
@@ -105,40 +105,40 @@ public class HdfsPathEntityHandler extends BaseEntityHandler {
return path;
}
- return super.getAttribute(attributeName);
+ return super.getAttribute(attribute);
}
@Override
- public void setAttribute(String attributeName, String attributeValue) {
- switch (attributeName) {
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ switch (attribute.getAttributeKey()) {
case HDFS_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HDFS_PATH_NAME_ATTRIBUTE:
name = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HDFS_PATH_PATH_ATTRIBUTE:
path = attributeValue;
isPathUpdated = true;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
default:
- super.setAttribute(attributeName, attributeValue);
+ super.setAttribute(attribute, attributeValue);
break;
}
}
@Override
public void transformComplete() {
- if (isCustomerAttributeUpdated) {
+ if (isCustomAttributeUpdated) {
entity.setAttribute(CLUSTER_NAME_ATTRIBUTE, clusterName);
entity.setAttribute(NAME_ATTRIBUTE, name);
entity.setAttribute(PATH_ATTRIBUTE, toPath());
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
index fca94b6..686e11c 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveColumnEntityHandler.java
@@ -27,10 +27,10 @@ import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveColumnEntityHandler extends BaseEntityHandler {
- private static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_COLUMN_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_COLUMN_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
public HiveColumnEntityHandler(List<AtlasEntityTransformer> transformers) {
- super(transformers, CUSTOM_TRANSFORM_ATTRIBUTES);
+ super(transformers);
}
@Override
@@ -48,7 +48,7 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
private String tableName;
private String columnName;
private String clusterName;
- private boolean isCustomerAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated = false;
public HiveColumnEntity(AtlasEntity entity) {
super(entity);
@@ -73,8 +73,8 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
}
@Override
- public Object getAttribute(String attributeName) {
- switch (attributeName) {
+ public Object getAttribute(EntityAttribute attribute) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
return databaseName;
@@ -88,45 +88,45 @@ public class HiveColumnEntityHandler extends BaseEntityHandler {
return clusterName;
}
- return super.getAttribute(attributeName);
+ return super.getAttribute(attribute);
}
@Override
- public void setAttribute(String attributeName, String attributeValue) {
- switch (attributeName) {
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
databaseName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_TABLE_NAME_ATTRIBUTE:
tableName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_COLUMN_NAME_ATTRIBUTE:
columnName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
default:
- super.setAttribute(attributeName, attributeValue);
+ super.setAttribute(attribute, attributeValue);
break;
}
}
@Override
public void transformComplete() {
- if (isCustomerAttributeUpdated) {
+ if (isCustomAttributeUpdated) {
entity.setAttribute(NAME_ATTRIBUTE, columnName);
entity.setAttribute(QUALIFIED_NAME_ATTRIBUTE, toQualifiedName());
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
index 8a2e813..b8032aa 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveDatabaseEntityHandler.java
@@ -26,10 +26,10 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveDatabaseEntityHandler extends BaseEntityHandler {
- private static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
public HiveDatabaseEntityHandler(List<AtlasEntityTransformer> transformers) {
- super(transformers, CUSTOM_TRANSFORM_ATTRIBUTES);
+ super(transformers);
}
@Override
@@ -45,7 +45,7 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
private static class HiveDatabaseEntity extends AtlasTransformableEntity {
private String databaseName;
private String clusterName;
- private boolean isCustomerAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated = false;
public HiveDatabaseEntity(AtlasEntity entity) {
super(entity);
@@ -64,8 +64,8 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
}
@Override
- public Object getAttribute(String attributeName) {
- switch (attributeName) {
+ public Object getAttribute(EntityAttribute attribute) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
return databaseName;
@@ -73,33 +73,33 @@ public class HiveDatabaseEntityHandler extends BaseEntityHandler {
return clusterName;
}
- return super.getAttribute(attributeName);
+ return super.getAttribute(attribute);
}
@Override
- public void setAttribute(String attributeName, String attributeValue) {
- switch (attributeName) {
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
databaseName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
default:
- super.setAttribute(attributeName, attributeValue);
+ super.setAttribute(attribute, attributeValue);
break;
}
}
@Override
public void transformComplete() {
- if (isCustomerAttributeUpdated) {
+ if (isCustomAttributeUpdated) {
entity.setAttribute(NAME_ATTRIBUTE, databaseName);
entity.setAttribute(CLUSTER_NAME_ATTRIBUTE, clusterName);
entity.setAttribute(QUALIFIED_NAME_ATTRIBUTE, toQualifiedName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
index 6a7b17b..dc4edfb 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
@@ -26,11 +26,11 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
- private static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
public HiveStorageDescriptorEntityHandler(List<AtlasEntityTransformer> transformers) {
- super(transformers, CUSTOM_TRANSFORM_ATTRIBUTES);
+ super(transformers);
}
@Override
@@ -47,7 +47,7 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
private String tableName;
private String clusterName;
private String location;
- private boolean isCustomerAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated = false;
public HiveStorageDescriptorEntity(AtlasEntity entity) {
@@ -80,8 +80,8 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
}
@Override
- public Object getAttribute(String attributeName) {
- switch (attributeName) {
+ public Object getAttribute(EntityAttribute attribute) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
return databaseName;
@@ -92,39 +92,39 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
return clusterName;
}
- return super.getAttribute(attributeName);
+ return super.getAttribute(attribute);
}
@Override
- public void setAttribute(String attributeName, String attributeValue) {
- switch (attributeName) {
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ switch (attribute.getAttributeKey()) {
case HIVE_DB_NAME_ATTRIBUTE:
databaseName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_TABLE_NAME_ATTRIBUTE:
tableName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
default:
- super.setAttribute(attributeName, attributeValue);
+ super.setAttribute(attribute, attributeValue);
break;
}
}
@Override
public void transformComplete() {
- if (isCustomerAttributeUpdated) {
+ if (isCustomAttributeUpdated) {
entity.setAttribute(LOCATION_ATTRIBUTE, toLocation());
entity.setAttribute(QUALIFIED_NAME_ATTRIBUTE, toQualifiedName());
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
index b008e6c..9eb44d7 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveTableEntityHandler.java
@@ -26,11 +26,11 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveTableEntityHandler extends BaseEntityHandler {
- private static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
public HiveTableEntityHandler(List<AtlasEntityTransformer> transformers) {
- super(transformers, CUSTOM_TRANSFORM_ATTRIBUTES);
+ super(transformers);
}
@Override
@@ -46,7 +46,7 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
private String databaseName;
private String tableName;
private String clusterName;
- private boolean isCustomerAttributeUpdated = false;
+ private boolean isCustomAttributeUpdated = false;
public HiveTableEntity(AtlasEntity entity) {
@@ -69,8 +69,8 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
}
@Override
- public Object getAttribute(String attributeName) {
- switch (attributeName) {
+ public Object getAttribute(EntityAttribute attribute) {
+ switch (attribute.getAttributeKey()) {
case HIVE_TABLE_NAME_ATTRIBUTE:
return tableName;
@@ -81,39 +81,39 @@ public class HiveTableEntityHandler extends BaseEntityHandler {
return clusterName;
}
- return super.getAttribute(attributeName);
+ return super.getAttribute(attribute);
}
@Override
- public void setAttribute(String attributeName, String attributeValue) {
- switch (attributeName) {
+ public void setAttribute(EntityAttribute attribute, String attributeValue) {
+ switch (attribute.getAttributeKey()) {
case HIVE_TABLE_NAME_ATTRIBUTE:
tableName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_DB_NAME_ATTRIBUTE:
databaseName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
clusterName = attributeValue;
- isCustomerAttributeUpdated = true;
+ isCustomAttributeUpdated = true;
break;
default:
- super.setAttribute(attributeName, attributeValue);
+ super.setAttribute(attribute, attributeValue);
break;
}
}
@Override
public void transformComplete() {
- if (isCustomerAttributeUpdated) {
+ if (isCustomAttributeUpdated) {
entity.setAttribute(NAME_ATTRIBUTE, tableName);
entity.setAttribute(QUALIFIED_NAME_ATTRIBUTE, toQualifiedName());
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java b/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
deleted file mode 100644
index 5c16bcf..0000000
--- a/intg/src/main/java/org/apache/atlas/entitytransform/NeedsContext.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.entitytransform;
-
-public interface NeedsContext {
- void setContext(TransformerContext transformerContext);
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java b/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
index a7a77b5..4b2ece6 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/TransformerContext.java
@@ -23,13 +23,13 @@ import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
public class TransformerContext {
- private final AtlasTypeRegistry typeRegistry;
- private final AtlasTypeDefStore typeDefStore;
+ private final AtlasTypeRegistry typeRegistry;
+ private final AtlasTypeDefStore typeDefStore;
private final AtlasExportRequest exportRequest;
public TransformerContext(AtlasTypeRegistry typeRegistry, AtlasTypeDefStore typeDefStore, AtlasExportRequest exportRequest) {
- this.typeRegistry = typeRegistry;
- this.typeDefStore = typeDefStore;
+ this.typeRegistry = typeRegistry;
+ this.typeDefStore = typeDefStore;
this.exportRequest = exportRequest;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/31c3bea1/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
index c76f959..d6b0ede 100644
--- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
+++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
@@ -17,20 +17,31 @@
*/
package org.apache.atlas.entitytransform;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
import org.apache.atlas.model.impexp.AttributeTransform;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasObjectId;
-import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.testng.annotations.Test;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_COLUMN;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_DATABASE;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_STORAGE_DESCRIPTOR;
import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
@@ -38,6 +49,17 @@ import static org.testng.Assert.assertTrue;
import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE;
public class TransformationHandlerTest {
+ private static final Logger LOG = LoggerFactory.getLogger(TransformationHandlerTest.class);
+
+ private static final String TYPENAME_REFERENCEABLE = "Referenceable";
+ private static final String TYPENAME_ASSET = "Asset";
+ private static final String TYPENAME_NON_ASSET = "non_asset";
+
+ private static final String[] CLUSTER_NAMES = new String[] { "cl1", "prod" };
+ private static final String[] DATABASE_NAMES = new String[] { "hr", "sales", "engg" };
+ private static final String[] TABLE_NAMES = new String[] { "employees", "products", "invoice" };
+ private static final String[] COLUMN_NAMES = new String[] { "name", "age", "dob" };
+
@Test
public void testHdfsClusterRenameHandler() {
// Rename clusterName from cl1 to cl2
@@ -139,8 +161,8 @@ public class TransformationHandlerTest {
@Test
public void testEntityClearAttributesActionWithNoCondition() {
// clear replicatedFrom attribute for hive_table entities without any condition
- Map<String, String> actions = new HashMap<String, String>() {{ put("__entity.replicatedTo", "CLEAR:");
- put("__entity.replicatedFrom", "CLEAR:"); }};
+ Map<String, String> actions = new HashMap<String, String>() {{ put("Referenceable.replicatedTo", "CLEAR:");
+ put("Referenceable.replicatedFrom", "CLEAR:"); }};
AttributeTransform transform = new AttributeTransform(null, actions);
@@ -334,38 +356,75 @@ public class TransformationHandlerTest {
@Test
public void verifyAddClassification() {
- AtlasEntityTransformer entityTransformer = new AtlasEntityTransformer(
- Collections.singletonMap("hdfs_path.qualifiedName", "EQUALS: hr@cl1"),
- Collections.singletonMap("__entity", "addClassification: replicated")
- );
+ AtlasEntityTransformer transformer = new AtlasEntityTransformer(Collections.singletonMap("hive_db.qualifiedName", "EQUALS: hr@cl1"),
+ Collections.singletonMap("Referenceable.", "ADD_CLASSIFICATION: replicated"),
+ getTransformerContext());
+
+ List<BaseEntityHandler> handlers = Collections.singletonList(new BaseEntityHandler(Collections.singletonList(transformer)));
- List<BaseEntityHandler> handlers = new ArrayList<>();
- handlers.add(new BaseEntityHandler(Collections.singletonList(entityTransformer)));
assertApplyTransform(handlers);
}
@Test
public void verifyAddClassificationUsingScope() {
- AtlasObjectId objectId = new AtlasObjectId("hive_db", Collections.singletonMap("qualifiedName", "hr@cl1"));
- AtlasEntityTransformer entityTransformer = new AtlasEntityTransformer(
- Collections.singletonMap("__entity", "topLevel: "),
- Collections.singletonMap("__entity", "addClassification: replicated")
- );
-
- List<BaseEntityHandler> handlers = new ArrayList<>();
- handlers.add(new BaseEntityHandler(Collections.singletonList(entityTransformer)));
- Condition condition = handlers.get(0).transformers.get(0).getConditions().get(0);
- Condition.ObjectIdEquals objectIdEquals = (Condition.ObjectIdEquals) condition;
- objectIdEquals.add(objectId);
+ AtlasExportRequest exportRequest = new AtlasExportRequest();
+
+ exportRequest.setItemsToExport(Collections.singletonList(new AtlasObjectId("hive_db", Collections.singletonMap("qualifiedName", "hr@cl1"))));
+
+ AtlasEntityTransformer transformer = new AtlasEntityTransformer(Collections.singletonMap("Referenceable.", "topLevel: "),
+ Collections.singletonMap("Referenceable", "ADD_CLASSIFICATION: replicated"),
+ new TransformerContext(getTypeRegistry(), null, exportRequest));
+
+ List<BaseEntityHandler> handlers = Collections.singletonList(new BaseEntityHandler(Collections.singletonList(transformer)));
assertApplyTransform(handlers);
}
+ @Test
+ public void verifyEntityTypeInAttributeName() {
+ AttributeTransform p = new AttributeTransform();
+ p.addAction("Asset.name", "SET: renamed");
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p));
+
+ AtlasEntity assetEntity = new AtlasEntity(TYPENAME_ASSET, "name", "originalName");
+ AtlasEntity assetSubEntity = new AtlasEntity(HIVE_DATABASE, "name", "originalName");
+ AtlasEntity nonAssetEntity = new AtlasEntity(TYPENAME_NON_ASSET, "name", "originalName");
+
+ applyTransforms(assetEntity, handlers);
+ applyTransforms(assetSubEntity, handlers);
+ applyTransforms(nonAssetEntity, handlers);
+
+ assertEquals((String) assetEntity.getAttribute("name"), "renamed", "Asset.name expected to be updated for Asset entity");
+ assertEquals((String) assetSubEntity.getAttribute("name"), "renamed", "Asset.name expected to be updated for Asset sub-type entity");
+ assertEquals((String) nonAssetEntity.getAttribute("name"), "originalName", "Asset.name expected to be not updated for non-Asset type entity");
+ }
+
+ @Test
+ public void verifyNoEntityTypeInAttributeName() {
+ AttributeTransform p = new AttributeTransform();
+ p.addAction("name", "SET: renamed");
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p));
+
+ AtlasEntity assetEntity = new AtlasEntity(TYPENAME_ASSET, "name", "originalName");
+ AtlasEntity assetSubEntity = new AtlasEntity(HIVE_DATABASE, "name", "originalName");
+ AtlasEntity nonAssetEntity = new AtlasEntity(TYPENAME_NON_ASSET, "name", "originalName");
+
+ applyTransforms(assetEntity, handlers);
+ applyTransforms(assetSubEntity, handlers);
+ applyTransforms(nonAssetEntity, handlers);
+
+ assertEquals((String) assetEntity.getAttribute("name"), "renamed", "name expected to be updated for Asset entity");
+ assertEquals((String) assetSubEntity.getAttribute("name"), "renamed", "name expected to be updated for Asset sub-type entity");
+ assertEquals((String) nonAssetEntity.getAttribute("name"), "renamed", "name expected to be not updated for non-Asset type entity");
+ }
+
private void assertApplyTransform(List<BaseEntityHandler> handlers) {
for (AtlasEntity entity : getAllEntities()) {
applyTransforms(entity, handlers);
- if(entity.getAttribute("qualifiedName").equals("hr@cl1")) {
+ if(entity.getTypeName().equals("hive_db") && entity.getAttribute("qualifiedName").equals("hr@cl1")) {
assertNotNull(entity.getClassifications());
} else{
assertNull(entity.getClassifications());
@@ -374,7 +433,7 @@ public class TransformationHandlerTest {
}
private List<BaseEntityHandler> initializeHandlers(List<AttributeTransform> params) {
- return BaseEntityHandler.createEntityHandlers(params, null);
+ return BaseEntityHandler.createEntityHandlers(params, getTransformerContext());
}
private void applyTransforms(AtlasEntity entity, List<BaseEntityHandler> handlers) {
@@ -383,15 +442,50 @@ public class TransformationHandlerTest {
}
}
- final String[] clusterNames = new String[] { "cl1", "prod" };
- final String[] databaseNames = new String[] { "hr", "sales", "engg" };
- final String[] tableNames = new String[] { "employees", "products", "invoice" };
- final String[] columnNames = new String[] { "name", "age", "dob" };
+ private TransformerContext getTransformerContext() {
+ return new TransformerContext(getTypeRegistry(), null, null);
+ }
+
+ private AtlasTypeRegistry getTypeRegistry() {
+ AtlasTypeRegistry ret = new AtlasTypeRegistry();
+
+ AtlasEntityDef defReferenceable = new AtlasEntityDef(TYPENAME_REFERENCEABLE);
+ AtlasEntityDef defAsset = new AtlasEntityDef(TYPENAME_ASSET);
+ AtlasEntityDef defHdfsPath = new AtlasEntityDef(HDFS_PATH);
+ AtlasEntityDef defHiveDb = new AtlasEntityDef(HIVE_DATABASE);
+ AtlasEntityDef defHiveTable = new AtlasEntityDef(HIVE_TABLE);
+ AtlasEntityDef defHiveColumn = new AtlasEntityDef(HIVE_COLUMN);
+ AtlasEntityDef defHiveStorDesc = new AtlasEntityDef(HIVE_STORAGE_DESCRIPTOR);
+ AtlasEntityDef defNonAsset = new AtlasEntityDef(TYPENAME_NON_ASSET);
+
+ defAsset.addSuperType(TYPENAME_REFERENCEABLE);
+ defHdfsPath.addSuperType(TYPENAME_ASSET);
+ defHiveDb.addSuperType(TYPENAME_ASSET);
+ defHiveTable.addSuperType(TYPENAME_ASSET);
+ defHiveColumn.addSuperType(TYPENAME_ASSET);
+ defNonAsset.addSuperType(TYPENAME_REFERENCEABLE);
+
+ AtlasTypesDef typesDef = new AtlasTypesDef();
+
+ typesDef.setEntityDefs(Arrays.asList(defReferenceable, defAsset, defHdfsPath, defHiveDb, defHiveTable, defHiveColumn, defHiveStorDesc, defNonAsset));
+
+ try {
+ AtlasTypeRegistry.AtlasTransientTypeRegistry ttr = ret.lockTypeRegistryForUpdate();
+
+ ttr.addTypes(typesDef);
+
+ ret.releaseTypeRegistryForUpdate(ttr, true);
+ } catch (AtlasBaseException excp) {
+ LOG.warn("failed to initialize type-registry", excp);
+ }
+
+ return ret;
+ }
private List<AtlasEntity> getHdfsPathEntities() {
List<AtlasEntity> ret = new ArrayList<>();
- for (String clusterName : clusterNames) {
+ for (String clusterName : CLUSTER_NAMES) {
ret.add(getHdfsPathEntity1(clusterName));
ret.add(getHdfsPathEntity2(clusterName));
}
@@ -402,18 +496,18 @@ public class TransformationHandlerTest {
private List<AtlasEntity> getAllEntities() {
List<AtlasEntity> ret = new ArrayList<>();
- for (String clusterName : clusterNames) {
+ for (String clusterName : CLUSTER_NAMES) {
ret.add(getHdfsPathEntity1(clusterName));
ret.add(getHdfsPathEntity2(clusterName));
- for (String databaseName : databaseNames) {
+ for (String databaseName : DATABASE_NAMES) {
ret.add(getHiveDbEntity(clusterName, databaseName));
- for (String tableName : tableNames) {
+ for (String tableName : TABLE_NAMES) {
ret.add(getHiveTableEntity(clusterName, databaseName, tableName));
ret.add(getHiveStorageDescriptorEntity(clusterName, databaseName, tableName));
- for (String columnName : columnNames) {
+ for (String columnName : COLUMN_NAMES) {
ret.add(getHiveColumnEntity(clusterName, databaseName, tableName, columnName));
}
}
@@ -518,4 +612,8 @@ public class TransformationHandlerTest {
return entity;
}
+
+ private AtlasEntity getNonAssetEntity() {
+ return new AtlasEntity(TYPENAME_NON_ASSET);
+ }
}
\ No newline at end of file
[14/17] atlas git commit: ATLAS-2897: Elegant handling of empty zip
files. Part 2.
Posted by am...@apache.org.
ATLAS-2897: Elegant handling of empty zip files. Part 2.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7763fd0d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7763fd0d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7763fd0d
Branch: refs/heads/master
Commit: 7763fd0d329e5995ff9dc4c1f7bf73099eb85c27
Parents: f6acb08
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Oct 3 09:01:07 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:29 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ZipSource.java | 19 +++++++++++--------
.../repository/impexp/ExportServiceTest.java | 1 -
.../impexp/ZipFileResourceTestUtils.java | 20 ++++++++------------
.../atlas/web/resources/AdminResource.java | 19 ++++++++++++++++++-
4 files changed, 37 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index bfa0441..be8c168 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -64,13 +64,20 @@ public class ZipSource implements EntityImportStream {
this.importTransform = importTransform;
updateGuidZipEntryMap();
- if (MapUtils.isEmpty(guidEntityJsonMap)) {
+ if (isZipFileEmpty()) {
throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP, "Attempting to import empty ZIP.");
}
setCreationOrder();
}
+ private boolean isZipFileEmpty() {
+ return MapUtils.isEmpty(guidEntityJsonMap) ||
+ (guidEntityJsonMap.containsKey(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) &&
+ (guidEntityJsonMap.get(ZipExportFileNames.ATLAS_EXPORT_ORDER_NAME.toString()) == null)
+ );
+ }
+
public ImportTransforms getImportTransform() { return this.importTransform; }
public void setImportTransform(ImportTransforms importTransform) {
@@ -136,7 +143,7 @@ public class ZipSource implements EntityImportStream {
zipInputStream.close();
}
- public List<String> getCreationOrder() throws AtlasBaseException {
+ public List<String> getCreationOrder() {
return this.creationOrder;
}
@@ -234,12 +241,8 @@ public class ZipSource implements EntityImportStream {
@Override
public void reset() {
- try {
- getCreationOrder();
- this.iterator = this.creationOrder.iterator();
- } catch (AtlasBaseException e) {
- LOG.error("reset", e);
- }
+ getCreationOrder();
+ this.iterator = this.creationOrder.iterator();
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 7aa0b57..7886a64 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -314,7 +314,6 @@ public class ExportServiceTest extends ExportImportTestBase {
assertNotNull(zipSource.getCreationOrder());
assertEquals(zipSource.getCreationOrder().size(), 0);
- assertEquals(AtlasExportResult.OperationStatus.FAIL, zipSource.getExportResult().getOperationStatus());
}
@Test
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index fe473b8..5e287d8 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -255,19 +255,15 @@ public class ZipFileResourceTestUtils {
public static AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource source, int expectedCount) {
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo();
- try {
- int count = 0;
- for (String s : source.getCreationOrder()) {
- AtlasEntity entity = source.getByGuid(s);
- entityWithExtInfo.addReferredEntity(s, entity);
- count++;
- }
-
- assertEquals(count, expectedCount);
- return entityWithExtInfo;
- } catch (AtlasBaseException e) {
- throw new SkipException("getEntities: failed!");
+ int count = 0;
+ for (String s : source.getCreationOrder()) {
+ AtlasEntity entity = source.getByGuid(s);
+ entityWithExtInfo.addReferredEntity(s, entity);
+ count++;
}
+
+ assertEquals(count, expectedCount);
+ return entityWithExtInfo;
}
public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
http://git-wip-us.apache.org/repos/asf/atlas/blob/7763fd0d/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index 55e8b9e..d9b1a41 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -391,7 +391,7 @@ public class AdminResource {
AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_IMPORT), "importData");
acquireExportImportLock("import");
- AtlasImportResult result;
+ AtlasImportResult result = null;
try {
AtlasImportRequest request = AtlasType.fromJson(jsonData, AtlasImportRequest.class);
@@ -400,6 +400,15 @@ public class AdminResource {
result = importService.run(zipSource, request, Servlets.getUserName(httpServletRequest),
Servlets.getHostName(httpServletRequest),
AtlasAuthorizationUtils.getRequestIpAddress(httpServletRequest));
+ } catch (AtlasBaseException excp) {
+ if (excp.getAtlasErrorCode().equals(AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP)) {
+ LOG.info(excp.getMessage());
+ } else {
+ LOG.error("importData(binary) failed", excp);
+ }
+
+ throw excp;
+
} catch (Exception excp) {
LOG.error("importData(binary) failed", excp);
@@ -434,6 +443,14 @@ public class AdminResource {
result = importService.run(request, Servlets.getUserName(httpServletRequest),
Servlets.getHostName(httpServletRequest),
AtlasAuthorizationUtils.getRequestIpAddress(httpServletRequest));
+ } catch (AtlasBaseException excp) {
+ if (excp.getAtlasErrorCode().getErrorCode().equals(AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP)) {
+ LOG.info(excp.getMessage());
+ } else {
+ LOG.error("importData(binary) failed", excp);
+ }
+
+ throw excp;
} catch (Exception excp) {
LOG.error("importFile() failed", excp);
[16/17] atlas git commit: ATLAS-2897: Better handling of empty zip
files. Unit test fix.
Posted by am...@apache.org.
ATLAS-2897: Better handling of empty zip files. Unit test fix.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/7ffbec1a
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/7ffbec1a
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/7ffbec1a
Branch: refs/heads/master
Commit: 7ffbec1a2a16f5288fff27d5bb81254d3280a900
Parents: 016eaff
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Oct 4 15:30:13 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:30 2018 -0700
----------------------------------------------------------------------
.../atlas/repository/impexp/ExportServiceTest.java | 15 ++++-----------
1 file changed, 4 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/7ffbec1a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 7886a64..9f72f1b 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -218,7 +218,7 @@ public class ExportServiceTest extends ExportImportTestBase {
assertNotNull(result.getSourceClusterName());
}
- @Test
+ @Test(expectedExceptions = AtlasBaseException.class)
public void requestingEntityNotFound_NoData() throws AtlasBaseException, IOException {
String requestingIP = "1.0.0.0";
String hostName = "root";
@@ -231,11 +231,7 @@ public class ExportServiceTest extends ExportImportTestBase {
Assert.assertNull(result.getData());
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
- ZipSource zipSource = new ZipSource(bais);
-
- assertNotNull(exportService);
- assertNotNull(zipSource.getCreationOrder());
- Assert.assertFalse(zipSource.hasNext());
+ new ZipSource(bais);
}
@Test
@@ -306,14 +302,11 @@ public class ExportServiceTest extends ExportImportTestBase {
AtlasExportResult.OperationStatus.FAIL));
}
- @Test
+ @Test(expectedExceptions = AtlasBaseException.class)
public void requestingExportOfNonExistentEntity_ReturnsFailure() throws Exception {
AtlasExportRequest request = getRequestForEmployee();
tamperEmployeeRequest(request);
- ZipSource zipSource = runExportWithParameters(request);
-
- assertNotNull(zipSource.getCreationOrder());
- assertEquals(zipSource.getCreationOrder().size(), 0);
+ runExportWithParameters(request);
}
@Test
[08/17] atlas git commit: ATLAS-2897: Elegant handling of empty zip
files.
Posted by am...@apache.org.
ATLAS-2897: Elegant handling of empty zip files.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5fe6d830
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5fe6d830
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5fe6d830
Branch: refs/heads/master
Commit: 5fe6d8306da60bc20dbd8c6e51a9040ea32e9ebb
Parents: 84c6fb2
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Sep 27 09:27:30 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:27 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../atlas/repository/impexp/ZipSource.java | 21 +++++++++++-----
.../repository/impexp/ImportServiceTest.java | 24 +++++++++++++------
.../impexp/ZipFileResourceTestUtils.java | 6 ++---
.../atlas/repository/impexp/ZipSourceTest.java | 8 +++----
repository/src/test/resources/empty.zip | Bin 0 -> 22 bytes
6 files changed, 40 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 2fe389c..e847014 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -153,6 +153,7 @@ public enum AtlasErrorCode {
INVALID_TIMEBOUNDRY_END_TIME(400, "ATLAS-400-00-87C", "Invalid endTime {0}"),
INVALID_TIMEBOUNDRY_DATERANGE(400, "ATLAS-400-00-87D", "Invalid dateRange: startTime {0} must be before endTime {1}"),
PROPAGATED_CLASSIFICATION_REMOVAL_NOT_SUPPORTED(400, "ATLAS-400-00-87E", "Removal of classification {0}, which is propagated from entity {1}, is not supported"),
+ IMPORT_ATTEMPTING_EMPTY_ZIP(400, "ATLAS-400-00-87F", "Attempting to import empty ZIP file."),
UNAUTHORIZED_ACCESS(403, "ATLAS-403-00-001", "{0} is not authorized to perform {1}"),
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
index a292b96..bfa0441 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ZipSource.java
@@ -40,6 +40,8 @@ import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import static org.apache.atlas.AtlasErrorCode.IMPORT_ATTEMPTING_EMPTY_ZIP;
+
public class ZipSource implements EntityImportStream {
private static final Logger LOG = LoggerFactory.getLogger(ZipSource.class);
@@ -52,16 +54,20 @@ public class ZipSource implements EntityImportStream {
private List<BaseEntityHandler> entityHandlers;
private int currentPosition;
- public ZipSource(InputStream inputStream) throws IOException {
+ public ZipSource(InputStream inputStream) throws IOException, AtlasBaseException {
this(inputStream, null);
}
- public ZipSource(InputStream inputStream, ImportTransforms importTransform) throws IOException {
+ public ZipSource(InputStream inputStream, ImportTransforms importTransform) throws IOException, AtlasBaseException {
this.inputStream = inputStream;
this.guidEntityJsonMap = new HashMap<>();
this.importTransform = importTransform;
updateGuidZipEntryMap();
+ if (MapUtils.isEmpty(guidEntityJsonMap)) {
+ throw new AtlasBaseException(IMPORT_ATTEMPTING_EMPTY_ZIP, "Attempting to import empty ZIP.");
+ }
+
setCreationOrder();
}
@@ -82,7 +88,7 @@ public class ZipSource implements EntityImportStream {
public AtlasTypesDef getTypesDef() throws AtlasBaseException {
final String fileName = ZipExportFileNames.ATLAS_TYPESDEF_NAME.toString();
- String s = (String) getFromCache(fileName);
+ String s = getFromCache(fileName);
return convertFromJson(AtlasTypesDef.class, s);
}
@@ -185,7 +191,12 @@ public class ZipSource implements EntityImportStream {
}
private String getFromCache(String entryName) {
- return guidEntityJsonMap.get(entryName);
+ String s = guidEntityJsonMap.get(entryName);
+ if (StringUtils.isEmpty(s)) {
+ LOG.warn("Could not fetch requested contents of file: {}", entryName);
+ }
+
+ return s;
}
public void close() {
@@ -288,6 +299,4 @@ public class ZipSource implements EntityImportStream {
public int getPosition() {
return currentPosition;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index a13df66..e0bbb11 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -113,7 +113,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "sales")
- public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException {
+ public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("sales-v1-full.zip");
}
@@ -130,7 +130,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "reporting")
- public static Object[][] getDataFromReporting() throws IOException {
+ public static Object[][] getDataFromReporting() throws IOException, AtlasBaseException {
return getZipSource("reporting-v1-full.zip");
}
@@ -141,7 +141,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "logging")
- public static Object[][] getDataFromLogging(ITestContext context) throws IOException {
+ public static Object[][] getDataFromLogging(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("logging-v1-full.zip");
}
@@ -152,7 +152,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "salesNewTypeAttrs")
- public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException {
+ public static Object[][] getDataFromSalesNewTypeAttrs(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("salesNewTypeAttrs.zip");
}
@@ -163,7 +163,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "salesNewTypeAttrs-next")
- public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException {
+ public static Object[][] getDataFromSalesNewTypeAttrsNext(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("salesNewTypeAttrs-next.zip");
}
@@ -200,7 +200,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "ctas")
- public static Object[][] getDataFromCtas(ITestContext context) throws IOException {
+ public static Object[][] getDataFromCtas(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("ctas.zip");
}
@@ -276,7 +276,7 @@ public class ImportServiceTest extends ExportImportTestBase {
}
@DataProvider(name = "hdfs_path1")
- public static Object[][] getDataFromHdfsPath1(ITestContext context) throws IOException {
+ public static Object[][] getDataFromHdfsPath1(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("hdfs_path1.zip");
}
@@ -421,4 +421,14 @@ public class ImportServiceTest extends ExportImportTestBase {
assertTrue(importTransforms.getTransforms().containsKey("hive_column"));
assertEquals(importTransforms.getTransforms().get("hive_table").get("qualifiedName").size(), 2);
}
+
+ @Test(dataProvider = "empty-zip", expectedExceptions = AtlasBaseException.class)
+ public void importEmptyZip(ZipSource zipSource) {
+
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class)
+ public void importEmptyZip() throws IOException, AtlasBaseException {
+ getZipSource("empty.zip");
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index faf68fe..fe473b8 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -150,17 +150,17 @@ public class ZipFileResourceTestUtils {
return s;
}
- public static Object[][] getZipSource(String fileName) throws IOException {
+ public static Object[][] getZipSource(String fileName) throws IOException, AtlasBaseException {
return new Object[][]{{getZipSourceFrom(fileName)}};
}
- public static ZipSource getZipSourceFrom(String fileName) throws IOException {
+ public static ZipSource getZipSourceFrom(String fileName) throws IOException, AtlasBaseException {
FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream(fileName);
return new ZipSource(fs);
}
- private static ZipSource getZipSourceFrom(ByteArrayOutputStream baos) throws IOException {
+ private static ZipSource getZipSourceFrom(ByteArrayOutputStream baos) throws IOException, AtlasBaseException {
ByteArrayInputStream bis = new ByteArrayInputStream(baos.toByteArray());
ZipSource zipSource = new ZipSource(bis);
return zipSource;
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
index 1c1c68f..f0034fa 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipSourceTest.java
@@ -38,21 +38,21 @@ import static org.testng.AssertJUnit.assertTrue;
public class ZipSourceTest {
@DataProvider(name = "zipFileStocks")
- public static Object[][] getDataFromZipFile() throws IOException {
+ public static Object[][] getDataFromZipFile() throws IOException, AtlasBaseException {
FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks.zip");
return new Object[][] {{ new ZipSource(fs) }};
}
@DataProvider(name = "zipFileStocksFloat")
- public static Object[][] getDataFromZipFileWithLongFloats() throws IOException {
+ public static Object[][] getDataFromZipFileWithLongFloats() throws IOException, AtlasBaseException {
FileInputStream fs = ZipFileResourceTestUtils.getFileInputStream("stocks-float.zip");
return new Object[][] {{ new ZipSource(fs) }};
}
@DataProvider(name = "sales")
- public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException {
+ public static Object[][] getDataFromQuickStart_v1_Sales(ITestContext context) throws IOException, AtlasBaseException {
return getZipSource("sales-v1-full.zip");
}
@@ -66,7 +66,7 @@ public class ZipSourceTest {
}
@Test(dataProvider = "zipFileStocks")
- public void examineContents_BehavesAsExpected(ZipSource zipSource) throws IOException, AtlasBaseException {
+ public void examineContents_BehavesAsExpected(ZipSource zipSource) throws AtlasBaseException {
List<String> creationOrder = zipSource.getCreationOrder();
assertNotNull(creationOrder);
http://git-wip-us.apache.org/repos/asf/atlas/blob/5fe6d830/repository/src/test/resources/empty.zip
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/empty.zip b/repository/src/test/resources/empty.zip
new file mode 100644
index 0000000..15cb0ec
Binary files /dev/null and b/repository/src/test/resources/empty.zip differ
[17/17] atlas git commit: ATLAS-2909: ChangeMarker updated during
initialization.
Posted by am...@apache.org.
ATLAS-2909: ChangeMarker updated during initialization.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/1eb99543
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/1eb99543
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/1eb99543
Branch: refs/heads/master
Commit: 1eb995434cb00d228eb403f47799fadc8dd58c08
Parents: 7ffbec1
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Sun Oct 7 22:59:00 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:31 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/repository/impexp/AtlasServerService.java | 9 ++-------
.../repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java | 2 +-
.../apache/atlas/repository/impexp/ExportServiceTest.java | 4 +---
.../src/main/java/org/apache/atlas/RequestContext.java | 5 +++++
4 files changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
index 13a8cd9..8859a9d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AtlasServerService.java
@@ -88,13 +88,8 @@ public class AtlasServerService {
}
@GraphTransaction
- public AtlasServer save(AtlasServer server) {
-
- try {
- return dataAccess.save(server);
- } catch (AtlasBaseException e) {
- return server;
- }
+ public AtlasServer save(AtlasServer server) throws AtlasBaseException {
+ return dataAccess.save(server);
}
@GraphTransaction
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
index bcdc7a8..3421331 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
@@ -524,7 +524,7 @@ public class AtlasTypeDefGraphStoreV2 extends AtlasTypeDefGraphStore {
}
private String getCurrentUser() {
- String ret = RequestContext.get().getUser();
+ String ret = RequestContext.getCurrentUser();
if (StringUtils.isBlank(ret)) {
ret = System.getProperty("user.name");
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
index 9f72f1b..18e7c41 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportServiceTest.java
@@ -95,13 +95,11 @@ public class ExportServiceTest extends ExportImportTestBase {
public void setupTest() throws IOException, AtlasBaseException {
RequestContext.clear();
RequestContext.get().setUser(TestUtilsV2.TEST_USER, null);
- ZipFileResourceTestUtils.loadBaseModel(typeDefStore, typeRegistry);
+ basicSetup(typeDefStore, typeRegistry);
}
@BeforeClass
public void setupSampleData() throws AtlasBaseException {
- entityStore = new AtlasEntityStoreV2(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);;
-
AtlasTypesDef sampleTypes = TestUtilsV2.defineDeptEmployeeTypes();
AtlasTypesDef typesToCreate = AtlasTypeDefStoreInitializer.getTypesToCreate(sampleTypes, typeRegistry);
http://git-wip-us.apache.org/repos/asf/atlas/blob/1eb99543/server-api/src/main/java/org/apache/atlas/RequestContext.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 25a35ce..9a9bba6 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -93,6 +93,11 @@ public class RequestContext {
CURRENT_CONTEXT.remove();
}
+ public static String getCurrentUser() {
+ RequestContext context = CURRENT_CONTEXT.get();
+ return context != null ? context.getUser() : null;
+ }
+
public String getUser() {
return user;
}
[02/17] atlas git commit: ATLAS-2875: Implement clear attribute value
transformer for Atlas Entity Transformer
Posted by am...@apache.org.
ATLAS-2875: Implement clear attribute value transformer for Atlas Entity Transformer
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/afa314cb
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/afa314cb
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/afa314cb
Branch: refs/heads/master
Commit: afa314cb520774e6417339f3180d292a6b0c13d6
Parents: 4b3c078
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Sep 20 11:20:39 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 15:40:21 2018 -0700
----------------------------------------------------------------------
.../apache/atlas/entitytransform/Action.java | 18 ++++
.../entitytransform/BaseEntityHandler.java | 8 ++
.../apache/atlas/entitytransform/Condition.java | 22 +++++
.../TransformationHandlerTest.java | 99 ++++++++++++++++++++
4 files changed, 147 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
index ca5f3a8..f01c6ce 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Action.java
@@ -31,6 +31,7 @@ public abstract class Action {
private static final String ACTION_NAME_REPLACE_PREFIX = "REPLACE_PREFIX";
private static final String ACTION_NAME_TO_LOWER = "TO_LOWER";
private static final String ACTION_NAME_TO_UPPER = "TO_UPPER";
+ private static final String ACTION_NAME_CLEAR = "CLEAR";
protected final String attributeName;
@@ -80,6 +81,10 @@ public abstract class Action {
ret = new SetAction(key, actionValue);
break;
+ case ACTION_NAME_CLEAR:
+ ret = new ClearAction(key);
+ break;
+
default:
ret = new SetAction(key, value); // treat unspecified/unknown action as 'SET'
break;
@@ -196,4 +201,17 @@ public abstract class Action {
}
}
}
+
+ public static class ClearAction extends Action {
+ public ClearAction(String attributeName) {
+ super(attributeName);
+ }
+
+ @Override
+ public void apply(AtlasTransformableEntity entity) {
+ if (isValid() && entity.hasAttribute(attributeName)) {
+ entity.setAttribute(attributeName, null);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
index c1f2869..9d44043 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/BaseEntityHandler.java
@@ -95,6 +95,10 @@ public class BaseEntityHandler {
}
}
+ if (CollectionUtils.isEmpty(ret)) {
+ ret.add(new BaseEntityHandler(transformers));
+ }
+
if (LOG.isDebugEnabled()) {
LOG.debug("<== BaseEntityHandler.createEntityHandlers(transforms={}): ret.size={}", transforms, ret.size());
}
@@ -158,6 +162,10 @@ public class BaseEntityHandler {
}
}
+ public boolean hasAttribute(String attributeName) {
+ return getAttribute(attributeName) != null;
+ }
+
public void transformComplete() {
// implementations can override to set value of computed-attributes
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
index d44f575..bc63079 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/Condition.java
@@ -31,6 +31,7 @@ public abstract class Condition {
private static final String CONDITION_NAME_EQUALS_IGNORE_CASE = "EQUALS_IGNORE_CASE";
private static final String CONDITION_NAME_STARTS_WITH = "STARTS_WITH";
private static final String CONDITION_NAME_STARTS_WITH_IGNORE_CASE = "STARTS_WITH_IGNORE_CASE";
+ private static final String CONDITION_NAME_HAS_VALUE = "HAS_VALUE";
protected final String attributeName;
@@ -75,6 +76,10 @@ public abstract class Condition {
ret = new StartsWithIgnoreCaseCondition(key, conditionValue);
break;
+ case CONDITION_NAME_HAS_VALUE:
+ ret = new HasValueCondition(key, conditionValue);
+ break;
+
default:
ret = new EqualsCondition(key, value); // treat unspecified/unknown condition as 'EQUALS'
break;
@@ -158,4 +163,21 @@ public abstract class Condition {
return attributeValue != null && StringUtils.startsWithIgnoreCase(attributeValue.toString(), this.prefix);
}
}
+
+ public static class HasValueCondition extends Condition {
+ protected final String attributeValue;
+
+ public HasValueCondition(String attributeName, String attributeValue) {
+ super(attributeName);
+
+ this.attributeValue = attributeValue;
+ }
+
+ @Override
+ public boolean matches(AtlasTransformableEntity entity) {
+ Object attributeValue = entity != null ? entity.getAttribute(attributeName) : null;
+
+ return attributeValue != null ? StringUtils.isNotEmpty(attributeValue.toString()) : false;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/afa314cb/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
index 69fba1e..a0ebe59 100644
--- a/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
+++ b/intg/src/test/java/org/apache/atlas/entitytransform/TransformationHandlerTest.java
@@ -25,9 +25,12 @@ import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.apache.atlas.entitytransform.TransformationConstants.HDFS_PATH;
+import static org.apache.atlas.entitytransform.TransformationConstants.HIVE_TABLE;
public class TransformationHandlerTest {
@Test
@@ -102,6 +105,100 @@ public class TransformationHandlerTest {
}
@Test
+ public void testHiveTableClearAttributeHandler() {
+ // clear replicatedTo attribute for hive_table entities
+ AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hive_table.replicatedTo", "HAS_VALUE:"),
+ Collections.singletonMap("hive_table.replicatedTo", "CLEAR:"));
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(p1));
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ }
+
+ applyTransforms(entity, handlers);
+
+ String transformedValue = (String) entity.getAttribute("replicatedTo");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(transformedValue));
+ }
+ }
+ }
+
+ @Test
+ public void testEntityClearAttributesActionWithNoCondition() {
+ // clear replicatedFrom attribute for hive_table entities without any condition
+ Map<String, String> actions = new HashMap<String, String>() {{ put("__entity.replicatedTo", "CLEAR:");
+ put("__entity.replicatedFrom", "CLEAR:"); }};
+
+ AttributeTransform transform = new AttributeTransform(null, actions);
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform));
+
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+ String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ }
+
+ applyTransforms(entity, handlers);
+
+ replicatedTo = (String) entity.getAttribute("replicatedTo");
+ replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ }
+ }
+ }
+
+ @Test
+ public void testEntityClearAttributesActionWithNoTypeNameAndNoCondition() {
+ // clear replicatedFrom attribute for hive_table entities without any condition
+ Map<String, String> actions = new HashMap<String, String>() {{ put("replicatedTo", "CLEAR:");
+ put("replicatedFrom", "CLEAR:"); }};
+
+ AttributeTransform transform = new AttributeTransform(null, actions);
+
+ List<BaseEntityHandler> handlers = initializeHandlers(Collections.singletonList(transform));
+
+ List<AtlasEntity> entities = getAllEntities();
+
+ for (AtlasEntity entity : entities) {
+ String replicatedTo = (String) entity.getAttribute("replicatedTo");
+ String replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isNotEmpty(replicatedFrom));
+ }
+
+ applyTransforms(entity, handlers);
+
+ replicatedTo = (String) entity.getAttribute("replicatedTo");
+ replicatedFrom = (String) entity.getAttribute("replicatedFrom");
+
+ if (entity.getTypeName() == HIVE_TABLE) {
+ Assert.assertTrue(StringUtils.isEmpty(replicatedTo));
+ Assert.assertTrue(StringUtils.isEmpty(replicatedFrom));
+ }
+ }
+ }
+
+ @Test
public void testHdfsPathNameReplacePrefixHandler() {
// Prefix replace hdfs_path name from /aa/bb/ to /xx/yy/
AttributeTransform p1 = new AttributeTransform(Collections.singletonMap("hdfs_path.name", "STARTS_WITH: /aa/bb/"),
@@ -338,6 +435,8 @@ public class TransformationHandlerTest {
entity.setAttribute("tableType", "EXTERNAL_TABLE");
entity.setAttribute("createTime", "1535656355000");
entity.setAttribute("retention", 0);
+ entity.setAttribute("replicatedTo", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]");
+ entity.setAttribute("replicatedFrom", "[{\"guid\":\"f378cfa5-c4aa-4699-a733-8f11d2f089cd\",\"typeName\":\"AtlasServer\"},{\"guid\":\"58e42789-ea3e-4eaa-a0c4-d38d8632e548\",\"typeName\":\"AtlasServer\"}]");
return entity;
}
[15/17] atlas git commit: ATLAS-2906: Allow transforms to be applied
when entity-level transforms are present.
Posted by am...@apache.org.
ATLAS-2906: Allow transforms to be applied when entity-level transforms are present.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/8a26c79e
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/8a26c79e
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/8a26c79e
Branch: refs/heads/master
Commit: 8a26c79e2f9ecfc14052c0954cb6e66308af09c6
Parents: 7763fd0
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Wed Oct 3 18:08:51 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Oct 11 17:21:29 2018 -0700
----------------------------------------------------------------------
.../entitytransform/HiveStorageDescriptorEntityHandler.java | 9 ++++++++-
.../atlas/entitytransform/TransformationConstants.java | 1 +
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/8a26c79e/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
index dc4edfb..38de206 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/HiveStorageDescriptorEntityHandler.java
@@ -26,7 +26,7 @@ import java.util.List;
import static org.apache.atlas.entitytransform.TransformationConstants.*;
public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
- static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE);
+ static final List<String> CUSTOM_TRANSFORM_ATTRIBUTES = Arrays.asList(HIVE_DB_NAME_ATTRIBUTE, HIVE_TABLE_NAME_ATTRIBUTE, HIVE_DB_CLUSTER_NAME_ATTRIBUTE, HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE);
public HiveStorageDescriptorEntityHandler(List<AtlasEntityTransformer> transformers) {
@@ -90,6 +90,9 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
case HIVE_DB_CLUSTER_NAME_ATTRIBUTE:
return clusterName;
+
+ case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+ return location;
}
return super.getAttribute(attribute);
@@ -116,6 +119,10 @@ public class HiveStorageDescriptorEntityHandler extends BaseEntityHandler {
isCustomAttributeUpdated = true;
break;
+ case HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE:
+ location = attributeValue;
+ break;
+
default:
super.setAttribute(attribute, attributeValue);
break;
http://git-wip-us.apache.org/repos/asf/atlas/blob/8a26c79e/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
index 51c3ace..247de73 100644
--- a/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
+++ b/intg/src/main/java/org/apache/atlas/entitytransform/TransformationConstants.java
@@ -37,6 +37,7 @@ public final class TransformationConstants {
public static final String HDFS_PATH_NAME_ATTRIBUTE = "hdfs_path.name";
public static final String HDFS_PATH_PATH_ATTRIBUTE = "hdfs_path.path";
public static final String HDFS_CLUSTER_NAME_ATTRIBUTE = "hdfs_path.clusterName";
+ public static final String HIVE_STORAGE_DESC_LOCATION_ATTRIBUTE = "hive_storagedesc.location";
public static final char TYPE_NAME_ATTRIBUTE_NAME_SEP = '.';
public static final char CLUSTER_DELIMITER = '@';