You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2020/12/16 18:55:07 UTC
[atlas] branch branch-2.0 updated (5aec0b8 -> 2a42e78)
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from 5aec0b8 ATLAS-4058: UI: Fix Entity detail page Hive Column Position with 0 is set to NA.
new 9f20077 ATLAS-4076: Propagated-tag-addition entries were appearing multiple times on derived table audit
new 2a42e78 ATLAS-4067: Updates in Custom Attributes and Business Attributes must be captured separately in entity audits.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
dashboardv2/public/js/utils/Enums.js | 1 +
dashboardv3/public/js/utils/Enums.js | 1 +
.../apache/atlas/exception/AtlasBaseException.java | 1 -
.../atlas/model/audit/EntityAuditEventV2.java | 4 ++-
.../repository/audit/EntityAuditListenerV2.java | 13 ++++++++-
.../store/graph/v2/AtlasEntityChangeNotifier.java | 2 ++
.../store/graph/v2/AtlasEntityStoreV2.java | 18 ++++++++----
.../main/java/org/apache/atlas/RequestContext.java | 32 ++++++++++++++++++++++
8 files changed, 64 insertions(+), 8 deletions(-)
[atlas] 02/02: ATLAS-4067: Updates in Custom Attributes and
Business Attributes must be captured separately in entity audits.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 2a42e7826e715f8b05d27c7d65d9b2c753e12b57
Author: Deep Singh <de...@gmail.com>
AuthorDate: Mon Dec 14 12:40:08 2020 -0600
ATLAS-4067: Updates in Custom Attributes and Business Attributes must be captured separately in entity audits.
Signed-off-by: Ashutosh Mestry <am...@cloudera.com>
---
dashboardv2/public/js/utils/Enums.js | 1 +
dashboardv3/public/js/utils/Enums.js | 1 +
.../apache/atlas/exception/AtlasBaseException.java | 1 -
.../atlas/model/audit/EntityAuditEventV2.java | 4 +++-
.../repository/audit/EntityAuditListenerV2.java | 13 +++++++++++-
.../store/graph/v2/AtlasEntityStoreV2.java | 18 +++++++++++-----
.../main/java/org/apache/atlas/RequestContext.java | 24 ++++++++++++++++++++++
7 files changed, 54 insertions(+), 8 deletions(-)
diff --git a/dashboardv2/public/js/utils/Enums.js b/dashboardv2/public/js/utils/Enums.js
index ebfceba..d2e476d 100644
--- a/dashboardv2/public/js/utils/Enums.js
+++ b/dashboardv2/public/js/utils/Enums.js
@@ -42,6 +42,7 @@ define(["require", "backbone"], function(require) {
BUSINESS_ATTRIBUTE_ADD: "Business Attribute(s) Added",
BUSINESS_ATTRIBUTE_UPDATE: "Business Attribute(s) Updated",
BUSINESS_ATTRIBUTE_DELETE: "Business Attribute(s) Deleted",
+ CUSTOM_ATTRIBUTE_UPDATE: "User-defined Attribute(s) Updated",
TYPE_DEF_UPDATE: "Type Updated",
TYPE_DEF_CREATE: "Type Created",
TYPE_DEF_DELETE: "Type Deleted",
diff --git a/dashboardv3/public/js/utils/Enums.js b/dashboardv3/public/js/utils/Enums.js
index ebfceba..d2e476d 100644
--- a/dashboardv3/public/js/utils/Enums.js
+++ b/dashboardv3/public/js/utils/Enums.js
@@ -42,6 +42,7 @@ define(["require", "backbone"], function(require) {
BUSINESS_ATTRIBUTE_ADD: "Business Attribute(s) Added",
BUSINESS_ATTRIBUTE_UPDATE: "Business Attribute(s) Updated",
BUSINESS_ATTRIBUTE_DELETE: "Business Attribute(s) Deleted",
+ CUSTOM_ATTRIBUTE_UPDATE: "User-defined Attribute(s) Updated",
TYPE_DEF_UPDATE: "Type Updated",
TYPE_DEF_CREATE: "Type Created",
TYPE_DEF_DELETE: "Type Deleted",
diff --git a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
index 5538702..88a26f4 100644
--- a/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
+++ b/intg/src/main/java/org/apache/atlas/exception/AtlasBaseException.java
@@ -19,7 +19,6 @@ package org.apache.atlas.exception;
import org.apache.atlas.AtlasErrorCode;
-import javax.ws.rs.core.Response;
import java.util.List;
/**
diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
index c37e282..083acac 100644
--- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
+++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
@@ -54,7 +54,7 @@ public class EntityAuditEventV2 implements Serializable, Clearable {
CLASSIFICATION_ADD, CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE,
PROPAGATED_CLASSIFICATION_ADD, PROPAGATED_CLASSIFICATION_DELETE, PROPAGATED_CLASSIFICATION_UPDATE,
TERM_ADD, TERM_DELETE, LABEL_ADD, LABEL_DELETE, ENTITY_PURGE,
- BUSINESS_ATTRIBUTE_UPDATE;
+ BUSINESS_ATTRIBUTE_UPDATE, CUSTOM_ATTRIBUTE_UPDATE;
public static EntityAuditActionV2 fromString(String strValue) {
switch (strValue) {
@@ -97,6 +97,8 @@ public class EntityAuditEventV2 implements Serializable, Clearable {
return LABEL_DELETE;
case "BUSINESS_ATTRIBUTE_UPDATE":
return BUSINESS_ATTRIBUTE_UPDATE;
+ case "CUSTOM_ATTRIBUTE_UPDATE":
+ return CUSTOM_ATTRIBUTE_UPDATE;
}
throw new IllegalArgumentException("No enum constant " + EntityAuditActionV2.class.getCanonicalName() + "." + strValue);
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
index ca6f373..0043f1c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
@@ -63,6 +63,7 @@ import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_UPDATE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CUSTOM_ATTRIBUTE_UPDATE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_ADD;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
@@ -109,7 +110,11 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
FixedBufferList<EntityAuditEventV2> updatedEvents = getAuditEventsList();
for (AtlasEntity entity : entities) {
- createEvent(updatedEvents.next(), entity, isImport ? ENTITY_IMPORT_UPDATE : ENTITY_UPDATE);
+ EntityAuditActionV2 action = isImport ? ENTITY_IMPORT_UPDATE :
+ RequestContext.get().checkIfEntityIsForCustomAttributeUpdate(entity.getGuid()) ? CUSTOM_ATTRIBUTE_UPDATE :
+ RequestContext.get().checkIfEntityIsForBusinessAttributeUpdate(entity.getGuid()) ? BUSINESS_ATTRIBUTE_UPDATE :
+ ENTITY_UPDATE;
+ createEvent(updatedEvents.next(), entity, action);
}
auditRepository.putEventsV2(updatedEvents.toList());
@@ -579,6 +584,12 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
case ENTITY_UPDATE:
ret = "Updated: ";
break;
+ case CUSTOM_ATTRIBUTE_UPDATE:
+ ret = "Updated custom attribute: ";
+ break;
+ case BUSINESS_ATTRIBUTE_UPDATE:
+ ret = "Updated business attribute: ";
+ break;
case ENTITY_DELETE:
ret = "Deleted: ";
break;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index b80e42e..2440722 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -1149,7 +1149,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
String guid = entity.getGuid();
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
- boolean hasUpdates = false;
+ boolean hasUpdates = false, hasUpdatesInCustAttr = false, hasUpdatesInBusAttr = false;
if (!hasUpdates) {
hasUpdates = entity.getStatus() == AtlasEntity.Status.DELETED; // entity status could be updated during import
@@ -1199,21 +1199,21 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
}
- if (!hasUpdates && entity.getCustomAttributes() != null) {
+ if (entity.getCustomAttributes() != null) {
Map<String, String> currCustomAttributes = getCustomAttributes(vertex);
Map<String, String> newCustomAttributes = entity.getCustomAttributes();
if (!Objects.equals(currCustomAttributes, newCustomAttributes)) {
- hasUpdates = true;
+ hasUpdatesInCustAttr = true;
}
}
- if (!hasUpdates && replaceBusinessAttributes) {
+ if (replaceBusinessAttributes) {
Map<String, Map<String, Object>> currBusinessMetadata = entityRetriever.getBusinessMetadata(vertex);
Map<String, Map<String, Object>> newBusinessMetadata = entity.getBusinessAttributes();
if (!Objects.equals(currBusinessMetadata, newBusinessMetadata)) {
- hasUpdates = true;
+ hasUpdatesInBusAttr = true;
}
}
@@ -1231,6 +1231,14 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
}
}
+ if(!hasUpdates && (hasUpdatesInCustAttr || hasUpdatesInBusAttr)) {
+ hasUpdates = true;
+ if (hasUpdatesInCustAttr ^ hasUpdatesInBusAttr) {
+ if(hasUpdatesInCustAttr) RequestContext.get().recordEntityWithCustomAttributeUpdate(entity.getGuid());
+ if(hasUpdatesInBusAttr) RequestContext.get().recordEntityWithBusinessAttributeUpdate(entity.getGuid());
+ }
+ }
+
if (!hasUpdates) {
if (entitiesToSkipUpdate == null) {
entitiesToSkipUpdate = new ArrayList<>();
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 7f0cfe5..216ba08 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -58,6 +58,8 @@ public class RequestContext {
private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null;
private List<EntityGuidPair> entityGuidInRequest = null;
private final Set<String> entitiesToSkipUpdate = new HashSet<>();
+ private final Set<String> onlyCAUpdateEntities = new HashSet<>();
+ private final Set<String> onlyBAUpdateEntities = new HashSet<>();
private String user;
private Set<String> userGroups;
@@ -115,6 +117,8 @@ public class RequestContext {
this.addedPropagations.clear();
this.removedPropagations.clear();
this.entitiesToSkipUpdate.clear();
+ this.onlyCAUpdateEntities.clear();
+ this.onlyBAUpdateEntities.clear();
if (metrics != null && !metrics.isEmpty()) {
METRICS.debug(metrics.toString());
@@ -243,6 +247,26 @@ public class RequestContext {
}
}
+ public void recordEntityWithCustomAttributeUpdate(String guid) {
+ if(! StringUtils.isEmpty(guid)) {
+ onlyCAUpdateEntities.add(guid);
+ }
+ }
+
+ public void recordEntityWithBusinessAttributeUpdate(String guid) {
+ if(! StringUtils.isEmpty(guid)) {
+ onlyBAUpdateEntities.add(guid);
+ }
+ }
+
+ public boolean checkIfEntityIsForCustomAttributeUpdate(String guid) {
+ return StringUtils.isNotEmpty(guid) && onlyCAUpdateEntities.contains(guid);
+ }
+
+ public boolean checkIfEntityIsForBusinessAttributeUpdate(String guid) {
+ return StringUtils.isNotEmpty(guid) && onlyBAUpdateEntities.contains(guid);
+ }
+
public void recordEntityDelete(AtlasEntityHeader entity) {
if (entity != null && entity.getGuid() != null) {
deletedEntities.put(entity.getGuid(), entity);
[atlas] 01/02: ATLAS-4076: Propagated-tag-addition entries were
appearing multiple times on derived table audit
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 9f20077ec9fc6ea672803b11f037fea1bad168e9
Author: Deep Singh <de...@gmail.com>
AuthorDate: Fri Dec 11 12:45:37 2020 -0600
ATLAS-4076: Propagated-tag-addition entries were appearing multiple times on derived table audit
Signed-off-by: Ashutosh Mestry <am...@cloudera.com>
---
.../repository/store/graph/v2/AtlasEntityChangeNotifier.java | 2 ++
server-api/src/main/java/org/apache/atlas/RequestContext.java | 8 ++++++++
2 files changed, 10 insertions(+)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index 32ad65e..99be169 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -359,7 +359,9 @@ public class AtlasEntityChangeNotifier implements IAtlasEntityChangeNotifier {
Map<String, List<AtlasClassification>> removedPropagations = context.getRemovedPropagations();
notifyPropagatedEntities(addedPropagations, PROPAGATED_CLASSIFICATION_ADD);
+ context.clearAddedPropagations();
notifyPropagatedEntities(removedPropagations, PROPAGATED_CLASSIFICATION_DELETE);
+ context.clearRemovePropagations();
}
@Override
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 32ffddd..7f0cfe5 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -329,6 +329,14 @@ public class RequestContext {
return deletedEntities.values();
}
+ public void clearRemovePropagations() {
+ removedPropagations.clear();
+ }
+
+ public void clearAddedPropagations() {
+ addedPropagations.clear();
+ }
+
/**
* Checks if an instance with the given guid is in the cache for this request. Either returns the instance
* or null if it is not in the cache.