You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/11/06 13:25:29 UTC
[atlas] branch branch-2.0 updated: ATLAS-3497 Add audit entry for
adding/removing labels
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 3adeb54 ATLAS-3497 Add audit entry for adding/removing labels
3adeb54 is described below
commit 3adeb54829e27f7c13224703fda742c04d90cf77
Author: Le Ma <lm...@cloudera.com>
AuthorDate: Thu Oct 31 14:56:49 2019 -0700
ATLAS-3497 Add audit entry for adding/removing labels
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 6050a8e6f7d572174d36946b9bee6968777e2850)
---
.../java/org/apache/atlas/EntityAuditEvent.java | 6 ++-
dashboardv2/public/js/utils/Enums.js | 4 +-
dashboardv3/public/js/utils/Enums.js | 4 +-
.../atlas/listener/EntityChangeListenerV2.java | 19 ++++++++
.../atlas/model/audit/EntityAuditEventV2.java | 6 ++-
.../repository/audit/EntityAuditListenerV2.java | 37 +++++++++++++++
.../converters/AtlasInstanceConverter.java | 4 ++
.../apache/atlas/repository/graph/GraphHelper.java | 10 +---
.../store/graph/v2/AtlasEntityChangeNotifier.java | 13 ++++++
.../store/graph/v2/EntityGraphMapper.java | 53 +++++++++++++++++-----
.../store/graph/v2/AtlasEntityStoreV2Test.java | 12 ++---
.../notification/EntityNotificationListenerV2.java | 10 ++++
12 files changed, 148 insertions(+), 30 deletions(-)
diff --git a/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java b/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java
index fcd6a62..1b452a9 100644
--- a/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java
+++ b/client/client-v1/src/main/java/org/apache/atlas/EntityAuditEvent.java
@@ -49,7 +49,7 @@ public class EntityAuditEvent implements Serializable {
ENTITY_CREATE, ENTITY_UPDATE, ENTITY_DELETE, TAG_ADD, TAG_DELETE, TAG_UPDATE,
PROPAGATED_TAG_ADD, PROPAGATED_TAG_DELETE, PROPAGATED_TAG_UPDATE,
ENTITY_IMPORT_CREATE, ENTITY_IMPORT_UPDATE, ENTITY_IMPORT_DELETE,
- TERM_ADD, TERM_DELETE;
+ TERM_ADD, TERM_DELETE, LABEL_ADD, LABEL_DELETE;
public static EntityAuditAction fromString(String strValue) {
switch (strValue) {
@@ -84,6 +84,10 @@ public class EntityAuditEvent implements Serializable {
return TERM_ADD;
case "TERM_DELETE":
return TERM_DELETE;
+ case "LABEL_ADD":
+ return LABEL_ADD;
+ case "LABEL_DELETE":
+ return LABEL_DELETE;
}
throw new IllegalArgumentException("No enum constant " + EntityAuditAction.class.getCanonicalName() + "." + strValue);
diff --git a/dashboardv2/public/js/utils/Enums.js b/dashboardv2/public/js/utils/Enums.js
index c7316d5..74a08d9 100644
--- a/dashboardv2/public/js/utils/Enums.js
+++ b/dashboardv2/public/js/utils/Enums.js
@@ -35,7 +35,9 @@ define(['require'], function(require) {
ENTITY_IMPORT_UPDATE: "Entity Updated by import",
ENTITY_IMPORT_DELETE: "Entity Deleted by import",
TERM_ADD: "Term Added",
- TERM_DELETE: "Term Deleted"
+ TERM_DELETE: "Term Deleted",
+ LABEL_ADD: "Label Added",
+ LABEL_DELETE: "Label Deleted"
}
Enums.entityStateReadOnly = {
diff --git a/dashboardv3/public/js/utils/Enums.js b/dashboardv3/public/js/utils/Enums.js
index c7316d5..74a08d9 100644
--- a/dashboardv3/public/js/utils/Enums.js
+++ b/dashboardv3/public/js/utils/Enums.js
@@ -35,7 +35,9 @@ define(['require'], function(require) {
ENTITY_IMPORT_UPDATE: "Entity Updated by import",
ENTITY_IMPORT_DELETE: "Entity Deleted by import",
TERM_ADD: "Term Added",
- TERM_DELETE: "Term Deleted"
+ TERM_DELETE: "Term Deleted",
+ LABEL_ADD: "Label Added",
+ LABEL_DELETE: "Label Deleted"
}
Enums.entityStateReadOnly = {
diff --git a/intg/src/main/java/org/apache/atlas/listener/EntityChangeListenerV2.java b/intg/src/main/java/org/apache/atlas/listener/EntityChangeListenerV2.java
index 106c797..444167e 100644
--- a/intg/src/main/java/org/apache/atlas/listener/EntityChangeListenerV2.java
+++ b/intg/src/main/java/org/apache/atlas/listener/EntityChangeListenerV2.java
@@ -26,6 +26,7 @@ import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasRelationship;
import java.util.List;
+import java.util.Set;
/**
* Entity change notification listener V2.
@@ -121,4 +122,22 @@ public interface EntityChangeListenerV2 {
* @param isImport
*/
void onRelationshipsDeleted(List<AtlasRelationship> relationships, boolean isImport) throws AtlasBaseException;
+
+ /**
+ * This is upon add new labels to an entity.
+ *
+ * @param entity the entity
+ * @param labels labels that needs to be added to an entity
+ * @throws AtlasBaseException if the listener notification fails
+ */
+ void onLabelsAdded(AtlasEntity entity, Set<String> labels) throws AtlasBaseException;
+
+ /**
+ * This is upon deleting labels from an entity.
+ *
+ * @param entity the entity
+ * @param labels labels that needs to be deleted for an entity
+ * @throws AtlasBaseException if the listener notification fails
+ */
+ void onLabelsDeleted(AtlasEntity entity, Set<String> labels) throws AtlasBaseException;
}
\ No newline at end of file
diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
index 649f11f..e9cc7cd 100644
--- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
+++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
@@ -50,7 +50,7 @@ public class EntityAuditEventV2 implements Serializable {
ENTITY_IMPORT_CREATE, ENTITY_IMPORT_UPDATE, ENTITY_IMPORT_DELETE,
CLASSIFICATION_ADD, CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE,
PROPAGATED_CLASSIFICATION_ADD, PROPAGATED_CLASSIFICATION_DELETE, PROPAGATED_CLASSIFICATION_UPDATE,
- TERM_ADD, TERM_DELETE;
+ TERM_ADD, TERM_DELETE, LABEL_ADD, LABEL_DELETE;
public static EntityAuditActionV2 fromString(String strValue) {
switch (strValue) {
@@ -85,6 +85,10 @@ public class EntityAuditEventV2 implements Serializable {
return TERM_ADD;
case "TERM_DELETE":
return TERM_DELETE;
+ case "LABEL_ADD":
+ return LABEL_ADD;
+ case "LABEL_DELETE":
+ return LABEL_DELETE;
}
throw new IllegalArgumentException("No enum constant " + EntityAuditActionV2.class.getCanonicalName() + "." + strValue);
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
index 20624da..43a9b84 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
@@ -47,6 +47,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_ADD;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_DELETE;
@@ -57,6 +58,8 @@ import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_UPDATE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_ADD;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.LABEL_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_DELETE;
import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_UPDATE;
@@ -241,6 +244,40 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
}
}
+ @Override
+ public void onLabelsAdded(AtlasEntity entity, Set<String> labels) throws AtlasBaseException {
+ if (CollectionUtils.isNotEmpty(labels)) {
+ MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit");
+
+ List<EntityAuditEventV2> events = new ArrayList<>();
+
+ String addedLabels = StringUtils.join(labels, " ");
+
+ events.add(createEvent(entity, LABEL_ADD, "Added labels: " + addedLabels));
+
+ auditRepository.putEventsV2(events);
+
+ RequestContext.get().endMetricRecord(metric);
+ }
+ }
+
+ @Override
+ public void onLabelsDeleted(AtlasEntity entity, Set<String> labels) throws AtlasBaseException {
+ if (CollectionUtils.isNotEmpty(labels)) {
+ MetricRecorder metric = RequestContext.get().startMetricRecord("entityAudit");
+
+ List<EntityAuditEventV2> events = new ArrayList<>();
+
+ String deletedLabels = StringUtils.join(labels, " ");
+
+ events.add(createEvent(entity, LABEL_DELETE, "Deleted labels: " + deletedLabels));
+
+ auditRepository.putEventsV2(events);
+
+ RequestContext.get().endMetricRecord(metric);
+ }
+ }
+
private EntityAuditEventV2 createEvent(AtlasEntity entity, EntityAuditActionV2 action, String details) {
return new EntityAuditEventV2(entity.getGuid(), RequestContext.get().getRequestTime(),
RequestContext.get().getUser(), action, details, entity);
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
index 1f0cc86..01e339d 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
@@ -394,6 +394,10 @@ public class AtlasInstanceConverter {
return EntityAuditEvent.EntityAuditAction.PROPAGATED_TAG_DELETE;
case PROPAGATED_CLASSIFICATION_UPDATE:
return EntityAuditEvent.EntityAuditAction.PROPAGATED_TAG_UPDATE;
+ case LABEL_ADD:
+ return EntityAuditEvent.EntityAuditAction.LABEL_ADD;
+ case LABEL_DELETE:
+ return EntityAuditEvent.EntityAuditAction.LABEL_DELETE;
}
return null;
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 173165a..1e7acf1 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -1851,16 +1851,10 @@ public final class GraphHelper {
}
private static Set<String> parseLabelsString(String labels) {
- Set<String> ret = null;
+ Set<String> ret = new HashSet<>();
if (StringUtils.isNotEmpty(labels)) {
- ret = new HashSet<>();
-
- for (String label : labels.split("\\" + LABEL_NAME_DELIMITER)) {
- if (StringUtils.isNotEmpty(label)) {
- ret.add(label);
- }
- }
+ ret.addAll(Arrays.asList(StringUtils.split(labels, "\\" + LABEL_NAME_DELIMITER)));
}
return ret;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index c910d9e..3389d24 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -256,6 +256,19 @@ public class AtlasEntityChangeNotifier {
}
}
+ public void onLabelsUpdatedFromEntity(String entityGuid, Set<String> addedLabels, Set<String> deletedLabels) throws AtlasBaseException {
+ doFullTextMapping(entityGuid);
+
+ if (isV2EntityNotificationEnabled) {
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
+
+ for (EntityChangeListenerV2 listener : entityChangeListenersV2) {
+ listener.onLabelsDeleted(entity, deletedLabels);
+ listener.onLabelsAdded(entity, addedLabels);
+ }
+ }
+ }
+
public void notifyPropagatedEntities() throws AtlasBaseException {
RequestContext context = RequestContext.get();
Map<String, List<AtlasClassification>> addedPropagations = context.getAddedPropagations();
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 2c2fc59..a114d25 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -92,6 +92,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdg
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
import static org.apache.atlas.repository.graph.GraphHelper.getDefaultRemovePropagations;
import static org.apache.atlas.repository.graph.GraphHelper.getDelimitedClassificationNames;
+import static org.apache.atlas.repository.graph.GraphHelper.getLabels;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
@@ -331,15 +332,28 @@ public class EntityGraphMapper {
}
}
- public void setLabels(AtlasVertex vertex, Set<String> labels) {
- if (CollectionUtils.isNotEmpty(labels)) {
- AtlasGraphUtilsV2.setEncodedProperty(vertex, LABELS_PROPERTY_KEY, getLabelString(labels));
+ public void setLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
+ final Set<String> currentLabels = getLabels(vertex);
+ final Set<String> addedLabels;
+ final Set<String> removedLabels;
+
+ if (CollectionUtils.isEmpty(currentLabels)) {
+ addedLabels = labels;
+ removedLabels = null;
+ } else if (CollectionUtils.isEmpty(labels)) {
+ addedLabels = null;
+ removedLabels = labels;
} else {
- vertex.removeProperty(LABELS_PROPERTY_KEY);
+ addedLabels = new HashSet<String>(CollectionUtils.subtract(labels, currentLabels));
+ removedLabels = new HashSet<String>(CollectionUtils.subtract(currentLabels, labels));
}
+
+ updateLabels(vertex, labels);
+
+ entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), addedLabels, removedLabels);
}
- public void addLabels(AtlasVertex vertex, Set<String> labels) {
+ public void addLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(labels)) {
final Set<String> existingLabels = GraphHelper.getLabels(vertex);
final Set<String> updatedLabels;
@@ -347,25 +361,40 @@ public class EntityGraphMapper {
if (CollectionUtils.isEmpty(existingLabels)) {
updatedLabels = labels;
} else {
- updatedLabels = existingLabels;
+ updatedLabels = new HashSet<>(existingLabels);
updatedLabels.addAll(labels);
}
-
- setLabels(vertex, updatedLabels);
+ if (!updatedLabels.equals(existingLabels)) {
+ updateLabels(vertex, updatedLabels);
+ updatedLabels.removeAll(existingLabels);
+ entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), updatedLabels, null);
+ }
}
}
- public void removeLabels(AtlasVertex vertex, Set<String> labels) {
+ public void removeLabels(AtlasVertex vertex, Set<String> labels) throws AtlasBaseException {
if (CollectionUtils.isNotEmpty(labels)) {
final Set<String> existingLabels = GraphHelper.getLabels(vertex);
- Set<String> updatedLabels = null;
+ Set<String> updatedLabels;
if (CollectionUtils.isNotEmpty(existingLabels)) {
- updatedLabels = existingLabels;
+ updatedLabels = new HashSet<>(existingLabels);
updatedLabels.removeAll(labels);
+
+ if (!updatedLabels.equals(existingLabels)) {
+ updateLabels(vertex, updatedLabels);
+ existingLabels.removeAll(updatedLabels);
+ entityChangeNotifier.onLabelsUpdatedFromEntity(GraphHelper.getGuid(vertex), null, existingLabels);
+ }
}
+ }
+ }
- setLabels(vertex, updatedLabels);
+ private void updateLabels(AtlasVertex vertex, Set<String> labels) {
+ if (CollectionUtils.isNotEmpty(labels)) {
+ AtlasGraphUtilsV2.setEncodedProperty(vertex, LABELS_PROPERTY_KEY, getLabelString(labels));
+ } else {
+ vertex.removeProperty(LABELS_PROPERTY_KEY);
}
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
index aba988d..51d55b9 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2Test.java
@@ -1147,19 +1147,19 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
- Assert.assertNull(tblEntity.getLabels());
+ Assert.assertTrue(tblEntity.getLabels().isEmpty());
}
@Test (dependsOnMethods = "clearLabelsToEntity")
- public void nullLabelsToEntity() throws AtlasBaseException {
+ public void emptyLabelsToEntity() throws AtlasBaseException {
entityStore.setLabels(tblEntityGuid, null);
AtlasEntity tblEntity = getEntityFromStore(tblEntityGuid);
- Assert.assertNull(tblEntity.getLabels());
+ Assert.assertTrue(tblEntity.getLabels().isEmpty());
}
- @Test (dependsOnMethods = "nullLabelsToEntity")
+ @Test (dependsOnMethods = "emptyLabelsToEntity")
public void invalidLabelLengthToEntity() throws AtlasBaseException {
Set<String> labels = new HashSet<>();
labels.add(randomAlphanumeric(50));
@@ -1173,7 +1173,7 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
}
@Test (dependsOnMethods = "invalidLabelLengthToEntity")
- public void invalidLabelCharactersToEntity() throws AtlasBaseException {
+ public void invalidLabelCharactersToEntity() {
Set<String> labels = new HashSet<>();
labels.add("label-1_100_45");
labels.add("LABEL-1_200-55");
@@ -1227,6 +1227,6 @@ public class AtlasEntityStoreV2Test extends AtlasEntityTestBase {
labels.add("label_3_add");
entityStore.removeLabels(tblEntityGuid, labels);
tblEntity = getEntityFromStore(tblEntityGuid);
- Assert.assertNull(tblEntity.getLabels());
+ Assert.assertTrue(tblEntity.getLabels().isEmpty());
}
}
\ No newline at end of file
diff --git a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java
index b2211fc..48f0cd3 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationListenerV2.java
@@ -128,6 +128,16 @@ public class EntityNotificationListenerV2 implements EntityChangeListenerV2 {
// do nothing -> notification not sent out for term removal from entities
}
+ @Override
+ public void onLabelsDeleted(AtlasEntity entity, Set<String> labels) throws AtlasBaseException {
+ // do nothing -> notification not sent out for label removal to entities
+ }
+
+ @Override
+ public void onLabelsAdded(AtlasEntity entity, Set<String> labels) throws AtlasBaseException {
+ // do nothing -> notification not sent out for label assignment to entities
+ }
+
private void notifyEntityEvents(List<AtlasEntity> entities, OperationType operationType) throws AtlasBaseException {
MetricRecorder metric = RequestContext.get().startMetricRecord("entityNotification");
List<EntityNotificationV2> messages = new ArrayList<>();