You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@atlas.apache.org by "Disha Talreja (Jira)" <ji...@apache.org> on 2022/09/13 23:45:00 UTC
[jira] [Updated] (ATLAS-4674) Regression: Classification tagging is not sending appropriate notification to ATLAS_ENTITIES
[ https://issues.apache.org/jira/browse/ATLAS-4674?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Disha Talreja updated ATLAS-4674:
---------------------------------
Description:
When we add/delete a classification from an entity, we send notification to ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based policy. This functionally seems to be broken and is a regression.
On further investigation, it is seen that the issue is caused by deferred action for classification propagation as the right notifications are sent when deferred action is disabled.
Considering the following example:
1. Create two table as shown below at Hive:
=====================================
create table parent_table(ssn varchar(100));
create table child_table as select * from parent_table;
2. Created two classification: "PII" and "sensitive"
3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660308744,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660307360
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{code}
4. Tagged another classification "sensitive" to parent_table and the notification at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660417995,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII",
"sensitive" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
,
{ "typeName": "sensitive", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660417115
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{code}
5. Removed "PII" tag from parent_table and the notification:
------------------------------------------------------------
{code:java}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456406,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
,
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"sensitive" ],
"classifications": [
{ "typeName": "sensitive", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660456206
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456586,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes":
{ "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
,
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{
"source": {},
"version":
{ "version": "1.0.0", "versionParts": [ 1 ] }
,
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456587,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes":
{ "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
,
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{ "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{code}
From the above notification messages, it is clear that the functionality is broken as the messages received are incorrect in case of the second tag (sensitive) being attached and on classification delete.
was:
When we add/delete a classification from an entity, we send notification to ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based policy. This functionally seems to be broken and is a regression.
On further investigation, it is seen that the issue is caused by deferred action for classification propagation as the right notifications are sent when deferred action is disabled.
Considering the following exampe:
1. Create two table as shown below at Hive:
=====================================
create table parent_table(ssn varchar(100));
create table child_table as select * from parent_table;
2. Created two classification: "PII" and "sensitive"
3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660308744,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660307360
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660309419,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
4. Tagged another classification "sensitive" to parent_table and the notification at ATLAS_ENTITIES topic:
-----------------------------------------------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660417995,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"PII",
"sensitive" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false },
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660417115
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660418085,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_ADD",
"eventTime": 1662660308935
}
}
5. Removed "PII" tag from parent_table and the notification:
------------------------------------------------------------
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456406,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602695000,
"qualifiedName": "default.parent_table@cm",
"name": "parent_table" },
"guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"status": "ACTIVE",
"displayText": "parent_table",
"classificationNames": [
"sensitive" ],
"classifications": [
{
"typeName": "sensitive",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660456206
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456586,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_table",
"attributes": {
"owner": "admin",
"createTime": 1662602726000,
"qualifiedName": "default.child_table@cm",
"name": "child_table" },
"guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
"status": "ACTIVE",
"displayText": "child_table",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
{
"source": {},
"version": {
"version": "1.0.0",
"versionParts": [
1
]
},
"msgCompressionKind": "NONE",
"msgSplitIdx": 1,
"msgSplitCount": 1,
"msgSourceIP": "172.27.40.5",
"msgCreatedBy": "",
"msgCreationTime": 1662660456587,
"spooled": false,
"message": {
"type": "ENTITY_NOTIFICATION_V2",
"entity": {
"typeName": "hive_process",
"attributes": {
"qualifiedName": "default.child_table@cm:1662602726000",
"clusterName": "cm",
"name": "default.child_table@cm:1662602726000" },
"guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
"status": "ACTIVE",
"displayText": "default.child_table@cm:1662602726000",
"classificationNames": [
"PII" ],
"classifications": [
{
"typeName": "PII",
"entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
"entityStatus": "ACTIVE",
"propagate": true,
"validityPeriods": [],
"removePropagationsOnEntityDelete": false }
],
"isIncomplete": false },
"operationType": "CLASSIFICATION_DELETE",
"eventTime": 1662660308935
}
}
From the above notification messages, it is clear that the functionality is broken as the messages received are incorrect in case of the second tag (sensitive) being attached and on classification delete.
> Regression: Classification tagging is not sending appropriate notification to ATLAS_ENTITIES
> --------------------------------------------------------------------------------------------
>
> Key: ATLAS-4674
> URL: https://issues.apache.org/jira/browse/ATLAS-4674
> Project: Atlas
> Issue Type: Bug
> Components: atlas-core
> Reporter: Disha Talreja
> Assignee: Disha Talreja
> Priority: Major
>
> When we add/delete a classification from an entity, we send notification to ATLAS_ENTITIES so ranger tag sync works appropriately to facilitate Tag-based policy. This functionally seems to be broken and is a regression.
> On further investigation, it is seen that the issue is caused by deferred action for classification propagation as the right notifications are sent when deferred action is disabled.
> Considering the following example:
> 1. Create two table as shown below at Hive:
> =====================================
> create table parent_table(ssn varchar(100));
> create table child_table as select * from parent_table;
> 2. Created two classification: "PII" and "sensitive"
>
> 3. Tagged PII to parent_table and notification received at ATLAS_ENTITIES topic:
> -----------------------------------------------------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660308744,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660307360
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660309419,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660309419,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {code}
> 4. Tagged another classification "sensitive" to parent_table and the notification at ATLAS_ENTITIES topic:
> -----------------------------------------------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660417995,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "PII",
> "sensitive" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ,
>
> { "typeName": "sensitive", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660417115
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660418085,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660418085,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_ADD",
> "eventTime": 1662660308935
> }
> }
> {code}
> 5. Removed "PII" tag from parent_table and the notification:
> ------------------------------------------------------------
> {code:java}
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456406,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602695000, "qualifiedName": "default.parent_table@cm", "name": "parent_table" }
> ,
> "guid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f",
> "status": "ACTIVE",
> "displayText": "parent_table",
> "classificationNames": [
> "sensitive" ],
> "classifications": [
>
> { "typeName": "sensitive", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660456206
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456586,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_table",
> "attributes":
> { "owner": "admin", "createTime": 1662602726000, "qualifiedName": "default.child_table@cm", "name": "child_table" }
> ,
> "guid": "117bc1c5-748c-4cbd-a86f-8369b7167b1f",
> "status": "ACTIVE",
> "displayText": "child_table",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660308935
> }
> }
> {
> "source": {},
> "version":
> { "version": "1.0.0", "versionParts": [ 1 ] }
> ,
> "msgCompressionKind": "NONE",
> "msgSplitIdx": 1,
> "msgSplitCount": 1,
> "msgSourceIP": "172.27.40.5",
> "msgCreatedBy": "",
> "msgCreationTime": 1662660456587,
> "spooled": false,
> "message": {
> "type": "ENTITY_NOTIFICATION_V2",
> "entity": {
> "typeName": "hive_process",
> "attributes":
> { "qualifiedName": "default.child_table@cm:1662602726000", "clusterName": "cm", "name": "default.child_table@cm:1662602726000" }
> ,
> "guid": "0e180d04-79ed-4cab-bbc3-76af4b876a5b",
> "status": "ACTIVE",
> "displayText": "default.child_table@cm:1662602726000",
> "classificationNames": [
> "PII" ],
> "classifications": [
>
> { "typeName": "PII", "entityGuid": "04ff79d7-b1ff-4354-ba75-a35b28d2f53f", "entityStatus": "ACTIVE", "propagate": true, "validityPeriods": [], "removePropagationsOnEntityDelete": false }
> ],
> "isIncomplete": false },
> "operationType": "CLASSIFICATION_DELETE",
> "eventTime": 1662660308935
> }
> }
> {code}
> From the above notification messages, it is clear that the functionality is broken as the messages received are incorrect in case of the second tag (sensitive) being attached and on classification delete.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)