You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2020/12/16 18:08:11 UTC
[atlas] branch master updated (de87800 -> c1c28c1)
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from de87800 ATLAS-4058: UI: Fix Entity detail page Hive Column Position with 0 is set to NA.
new 2e40de9 ATLAS-4076: Propagated-tag-addition entries were appearing multiple times on derived table audit
new c1c28c1 ATLAS-4067: Updates in Custom Attributes and Business Attributes must be captured separately in entity audits.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dashboardv2/public/js/utils/Enums.js | 1 +
dashboardv3/public/js/utils/Enums.js | 1 +
.../apache/atlas/exception/AtlasBaseException.java | 1 -
.../atlas/model/audit/EntityAuditEventV2.java | 4 ++-
.../repository/audit/EntityAuditListenerV2.java | 13 ++++++++-
.../store/graph/v2/AtlasEntityChangeNotifier.java | 2 ++
.../store/graph/v2/AtlasEntityStoreV2.java | 18 ++++++++----
.../main/java/org/apache/atlas/RequestContext.java | 32 ++++++++++++++++++++++
8 files changed, 64 insertions(+), 8 deletions(-)
[atlas] 02/02: ATLAS-4067: Updates in Custom Attributes and
Business Attributes must be captured separately in entity audits.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit c1c28c12add6bf4b19a8218037b52409970d9857
Author: Deep Singh <de...@gmail.com>
AuthorDate: Mon Dec 14 12:40:08 2020 -0600
ATLAS-4067: Updates in Custom Attributes and Business Attributes must be captured separately in entity audits.
Signed-off-by: Ashutosh Mestry <am...@cloudera.com>
---
dashboardv2/public/js/utils/Enums.js | 1 +
dashboardv3/public/js/utils/Enums.js | 1 +
.../apache/atlas/exception/AtlasBaseException.java | 1 -
.../atlas/model/audit/EntityAuditEventV2.java | 4 +++-
.../repository/audit/EntityAuditListenerV2.java | 13 +++++++++++-
.../store/graph/v2/AtlasEntityStoreV2.java | 18 +++++++++++-----
.../main/java/org/apache/atlas/RequestContext.java | 24 ++++++++++++++++++++++
7 files changed, 54 insertions(+), 8 deletions(-)
diff --git a/dashboardv2/public/js/utils/Enums.js b/dashboardv2/public/js/utils/Enums.js
index ebfceba..d2e476d 100644
--- a/dashboardv2/public/js/utils/Enums.js
+++ b/dashboardv2/public/js/utils/Enums.js
@@ -42,6 +42,7 @@ define(["require", "backbone"], function(require) {
BUSINESS_ATTRIBUTE_ADD: "Business Attribute(s) Added",
BUSINESS_ATTRIBUTE_UPDATE: "Business Attribute(s) Updated",
BUSINESS_ATTRIBUTE_DELETE: "Business Attribute(s) Deleted",
+ CUSTOM_ATTRIBUTE_UPDATE: "User-defined Attribute(s) Updated",
TYPE_DEF_UPDATE: "Type Updated",
TYPE_DEF_CREATE: "Type Created",
TYPE_DEF_DELETE: "Type Deleted",
diff --git a/dashboardv3/public/js/utils/Enums.js b/dashboardv3/public/js/utils/Enums.js
index ebfceba..d2e476d 100644
--- a/dashboardv3/public/js/utils/Enums.js
+++ b/dashboardv3/public/js/utils/Enums.js
@@ -42,6 +42,7 @@ define(["require", "backbone"], function(require) {
BUSINESS_ATTRIBUTE_ADD: "Business Attribute(s) Added",
BUSINESS_ATTRIBUTE_UPDATE: "Business Attribute(s) Updated",
BUSINESS_ATTRIBUTE_DELETE: "Business Attribute(s) Deleted",
+ CUSTOM_ATTRIBUTE_UPDATE: "User-defined Attribute(s) Updated",
TYPE_DEF_UPDATE: "Type Updated",
TYPE_DEF_CREATE: "Type Created",
TYPE_DEF_DELETE: "Type Deleted",
diff --git a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
index 5538702..88a26f4 100644
--- a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
+++ b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
@@ -19,7 +19,6 @@ package org.apache.atlas.exception;
import org.apache.atlas.AtlasErrorCode;
-import javax.ws.rs.core.Response;
import java.util.List;
/**
diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
index c37e282..083acac 100644
--- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
+++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
@@ -54,7 +54,7 @@ public class EntityAuditEventV2 implements Serializable, Clearable {
CLASSIFICATION_ADD, CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE,
PROPAGATED_CLASSIFICATION_ADD, PROPAGATED_CLASSIFICATION_DELETE, PROPAGATED_CLASSIFICATION_UPDATE,
TERM_ADD, TERM_DELETE, LABEL_ADD, LABEL_DELETE, ENTITY_PURGE,
- BUSINESS_ATTRIBUTE_UPDATE;
+ BUSINESS_ATTRIBUTE_UPDATE, CUSTOM_ATTRIBUTE_UPDATE;
public static EntityAuditActionV2 fromString(String strValue) {
switch (strValue) {
@@ -97,6 +97,8 @@ public class EntityAuditEventV2 implements Serializable, Clearable {
return LABEL_DELETE;
case "BUSINESS_ATTRIBUTE_UPDATE":
return BUSINESS_ATTRIBUTE_UPDATE;
+ case "CUSTOM_ATTRIBUTE_UPDATE":
+ return CUSTOM_ATTRIBUTE_UPDATE;
}
throw new IllegalArgumentException("No enum constant " + EntityAuditActionV2.class.getCanonicalName() + "." + strValue);
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
index ca6f373..0043f1c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
@@ -63,6 +63,7 @@ import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_UPDATE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CUSTOM_ATTRIBUTE_UPDATE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_ADD;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
@@ -109,7 +110,11 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
FixedBufferList<EntityAuditEventV2> updatedEvents = getAuditEventsList();
for (AtlasEntity entity : entities) {
- createEvent(updatedEvents.next(), entity, isImport ? ENTITY_IMPORT_UPDATE : ENTITY_UPDATE);
+ EntityAuditActionV2 action = isImport ? ENTITY_IMPORT_UPDATE :
+ RequestContext.get().checkIfEntityIsForCustomAttributeUpdate(entity.getGuid()) ? CUSTOM_ATTRIBUTE_UPDATE :
+ RequestContext.get().checkIfEntityIsForBusinessAttributeUpdate(entity.getGuid()) ? BUSINESS_ATTRIBUTE_UPDATE :
+ ENTITY_UPDATE;
+ createEvent(updatedEvents.next(), entity, action);
}
auditRepository.putEventsV2(updatedEvents.toList());
@@ -579,6 +584,12 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
case ENTITY_UPDATE:
ret = "Updated: ";
break;
+ case CUSTOM_ATTRIBUTE_UPDATE:
+ ret = "Updated custom attribute: ";
+ break;
+ case BUSINESS_ATTRIBUTE_UPDATE:
+ ret = "Updated business attribute: ";
+ break;
case ENTITY_DELETE:
ret = "Deleted: ";
break;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index b80e42e..2440722 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -1149,7 +1149,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
String guid = entity.getGuid();
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
- boolean hasUpdates = false;
+ boolean hasUpdates = false, hasUpdatesInCustAttr = false, hasUpdatesInBusAttr = false;
if (!hasUpdates) {
hasUpdates = entity.getStatus() == AtlasEntity.Status.DELETED; // entity status could be updated during import
@@ -1199,21 +1199,21 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
}
- if (!hasUpdates && entity.getCustomAttributes() != null) {
+ if (entity.getCustomAttributes() != null) {
Map<String, String> currCustomAttributes = getCustomAttributes(vertex);
Map<String, String> newCustomAttributes = entity.getCustomAttributes();
if (!Objects.equals(currCustomAttributes, newCustomAttributes)) {
- hasUpdates = true;
+ hasUpdatesInCustAttr = true;
}
}
- if (!hasUpdates && replaceBusinessAttributes) {
+ if (replaceBusinessAttributes) {
Map<String, Map<String, Object>> currBusinessMetadata = entityRetriever.getBusinessMetadata(vertex);
Map<String, Map<String, Object>> newBusinessMetadata = entity.getBusinessAttributes();
if (!Objects.equals(currBusinessMetadata, newBusinessMetadata)) {
- hasUpdates = true;
+ hasUpdatesInBusAttr = true;
}
}
@@ -1231,6 +1231,14 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
}
+ if(!hasUpdates && (hasUpdatesInCustAttr || hasUpdatesInBusAttr)) {
+ hasUpdates = true;
+ if (hasUpdatesInCustAttr ^ hasUpdatesInBusAttr) {
+ if(hasUpdatesInCustAttr) RequestContext.get().recordEntityWithCustomAttributeUpdate(entity.getGuid());
+ if(hasUpdatesInBusAttr) RequestContext.get().recordEntityWithBusinessAttributeUpdate(entity.getGuid());
+ }
+ }
+
if (!hasUpdates) {
if (entitiesToSkipUpdate == null) {
entitiesToSkipUpdate = new ArrayList<>();
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 7f0cfe5..216ba08 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -58,6 +58,8 @@ public class RequestContext {
private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null;
private List<EntityGuidPair> entityGuidInRequest = null;
private final Set<String> entitiesToSkipUpdate = new HashSet<>();
+ private final Set<String> onlyCAUpdateEntities = new HashSet<>();
+ private final Set<String> onlyBAUpdateEntities = new HashSet<>();
private String user;
private Set<String> userGroups;
@@ -115,6 +117,8 @@ public class RequestContext {
this.addedPropagations.clear();
this.removedPropagations.clear();
this.entitiesToSkipUpdate.clear();
+ this.onlyCAUpdateEntities.clear();
+ this.onlyBAUpdateEntities.clear();
if (metrics != null && !metrics.isEmpty()) {
METRICS.debug(metrics.toString());
@@ -243,6 +247,26 @@ public class RequestContext {
}
}
+ public void recordEntityWithCustomAttributeUpdate(String guid) {
+ if(! StringUtils.isEmpty(guid)) {
+ onlyCAUpdateEntities.add(guid);
+ }
+ }
+
+ public void recordEntityWithBusinessAttributeUpdate(String guid) {
+ if(! StringUtils.isEmpty(guid)) {
+ onlyBAUpdateEntities.add(guid);
+ }
+ }
+
+ public boolean checkIfEntityIsForCustomAttributeUpdate(String guid) {
+ return StringUtils.isNotEmpty(guid) && onlyCAUpdateEntities.contains(guid);
+ }
+
+ public boolean checkIfEntityIsForBusinessAttributeUpdate(String guid) {
+ return StringUtils.isNotEmpty(guid) && onlyBAUpdateEntities.contains(guid);
+ }
+
public void recordEntityDelete(AtlasEntityHeader entity) {
if (entity != null && entity.getGuid() != null) {
deletedEntities.put(entity.getGuid(), entity);
[atlas] 01/02: ATLAS-4076: Propagated-tag-addition entries were
appearing multiple times on derived table audit
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 2e40de917083c7c5583a61537b6447ab3af41918
Author: Deep Singh <de...@gmail.com>
AuthorDate: Fri Dec 11 12:45:37 2020 -0600
ATLAS-4076: Propagated-tag-addition entries were appearing multiple times on derived table audit
Signed-off-by: Ashutosh Mestry <am...@cloudera.com>
---
.../repository/store/graph/v2/AtlasEntityChangeNotifier.java | 2 ++
server-api/src/main/java/org/apache/atlas/RequestContext.java | 8 ++++++++
2 files changed, 10 insertions(+)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index 32ad65e..99be169 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -359,7 +359,9 @@ public class AtlasEntityChangeNotifier implements IAtlasEntityChangeNotifier {
Map<String, List<AtlasClassification>> removedPropagations = context.getRemovedPropagations();
notifyPropagatedEntities(addedPropagations, PROPAGATED_CLASSIFICATION_ADD);
+ context.clearAddedPropagations();
notifyPropagatedEntities(removedPropagations, PROPAGATED_CLASSIFICATION_DELETE);
+ context.clearRemovePropagations();
}
@Override
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 32ffddd..7f0cfe5 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -329,6 +329,14 @@ public class RequestContext {
return deletedEntities.values();
}
+ public void clearRemovePropagations() {
+ removedPropagations.clear();
+ }
+
+ public void clearAddedPropagations() {
+ addedPropagations.clear();
+ }
+
/**
* Checks if an instance with the given guid is in the cache for this request. Either returns the instance
* or null if it is not in the cache.