You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@atlas.apache.org by "Dharshana M Krishnamoorthy (Jira)" <ji...@apache.org> on 2021/07/27 12:42:00 UTC
[jira] [Updated] (ATLAS-4374) [Send lineage only information] When
a ctas table is created, there is an additional notification sent by hive
[ https://issues.apache.org/jira/browse/ATLAS-4374?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Dharshana M Krishnamoorthy updated ATLAS-4374:
----------------------------------------------
Attachment: Screenshot 2021-07-27 at 3.06.59 PM.png
> [Send lineage only information] When a ctas table is created, there is an additional notification sent by hive
> --------------------------------------------------------------------------------------------------------------
>
> Key: ATLAS-4374
> URL: https://issues.apache.org/jira/browse/ATLAS-4374
> Project: Atlas
> Issue Type: Bug
> Components: atlas-core
> Reporter: Dharshana M Krishnamoorthy
> Priority: Major
> Attachments: Screenshot 2021-07-27 at 3.06.59 PM.png, Screenshot 2021-07-27 at 3.22.47 PM.png
>
>
> *Scenario*:
> create a table and create a ctas table from it
> *Expectation*:
> There has to be only 1 notification from hive with *ENTITY_CREATE_V2* type
> *Observation*:
> There are 2 notifications
> one with *ENTITY_CREATE_V2* and other with *ENTITY_PARTIAL_UPDATE_V2*
> {code:java}
> {"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.202.195","msgCreatedBy":"hive","msgCreationTime":1627372412791,"spooled":false,"message":{"type":"ENTITY_PARTIAL_UPDATE_V2","user":"hive","entityId":{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm"}},"entity":{"entity":{"typeName":"hive_table","attributes":{"parameters":{"numRows":"0","rawDataSize":"0","transient_lastDdlTime":"1627372412","bucketing_version":"2","numFilesErasureCoded":"0","totalSize":"0","transactional_properties":"default","COLUMN_STATS_ACCURATE":"{\"BASIC_STATS\":\"true\"}","numFiles":"0","transactional":"true"}},"isIncomplete":false,"provenanceType":0,"version":0,"proxy":false}}}}
> {"version":{"version":"1.0.0","versionParts":[1]},"msgCompressionKind":"NONE","msgSplitIdx":1,"msgSplitCount":1,"msgSourceIP":"172.27.202.195","msgCreatedBy":"hive","msgCreationTime":1627372412906,"spooled":false,"message":{"type":"ENTITY_CREATE_V2","user":"hrt_qa","entities":{"referredEntities":{},"entities":[{"typeName":"hive_table_ddl","attributes":{"serviceType":"hive","qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372386732","execTime":1627372386732,"queryText":"create table db_mtkns.ctas_table_vwsci as select * from db_mtkns.table_gmqnw","name":"create table db_mtkns.ctas_table_vwsci as select * from db_mtkns.table_gmqnw","userName":"hrt_qa"},"guid":"-7790679096914344","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"table":{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm"},"relationshipType":"hive_table_ddl_queries"}},"proxy":false},{"typeName":"hive_process","attributes":{"recentQueries":["create table db_mtkns.ctas_table_vwsci as select * from db_mtkns.table_gmqnw"],"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000","clusterName":"cm","name":"db_mtkns.ctas_table_vwsci@cm:1627372412000","queryText":"","operationType":"CREATETABLE_AS_SELECT","startTime":1627372412905,"queryPlan":"Not Supported","endTime":1627372412905,"userName":"","queryId":""},"guid":"-7790679096914345","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"outputs":[{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm"},"relationshipType":"process_dataset_outputs"}],"inputs":[{"typeName":"hive_table","uniqueAttributes":{"qualifiedName":"db_mtkns.table_gmqnw@cm"},"relationshipType":"dataset_process_inputs"}]},"proxy":false},{"typeName":"hive_process_execution","attributes":{"hostName":"quasar-eotkek-5.quasar-eotkek.root.hwx.site","qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000:1627372386732:1627372412905","name":"db_mtkns.ctas_table_vwsci@cm:1627372412000:1627372386732:1627372412905","queryText":"create table db_mtkns.ctas_table_vwsci as select * from db_mtkns.table_gmqnw","startTime":1627372386732,"queryPlan":"Not Supported","endTime":1627372412905,"userName":"hrt_qa","queryId":"hive_20210727075306_c74d476f-95b0-44bb-80b7-04aa5e73cd9c"},"guid":"-7790679096914346","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"process":{"guid":"-7790679096914345","typeName":"hive_process","relationshipType":"hive_process_process_executions"}},"proxy":false},{"typeName":"hive_column_lineage","attributes":{"expression":null,"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_roll","name":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_roll","depenendencyType":"SIMPLE"},"guid":"-7790679096914347","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci.student_roll@cm"},"relationshipType":"process_dataset_outputs"}],"inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.table_gmqnw.student_roll@cm"},"relationshipType":"dataset_process_inputs"}],"query":{"guid":"-7790679096914345","typeName":"hive_process","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000"},"relationshipType":"hive_process_column_lineage"}},"proxy":false},{"typeName":"hive_column_lineage","attributes":{"expression":null,"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_name","name":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_name","depenendencyType":"SIMPLE"},"guid":"-7790679096914348","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci.student_name@cm"},"relationshipType":"process_dataset_outputs"}],"inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.table_gmqnw.student_name@cm"},"relationshipType":"dataset_process_inputs"}],"query":{"guid":"-7790679096914345","typeName":"hive_process","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000"},"relationshipType":"hive_process_column_lineage"}},"proxy":false},{"typeName":"hive_column_lineage","attributes":{"expression":null,"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_dob","name":"db_mtkns.ctas_table_vwsci@cm:1627372412000:student_dob","depenendencyType":"SIMPLE"},"guid":"-7790679096914349","isIncomplete":false,"provenanceType":0,"version":0,"relationshipAttributes":{"outputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci.student_dob@cm"},"relationshipType":"process_dataset_outputs"}],"inputs":[{"typeName":"hive_column","uniqueAttributes":{"qualifiedName":"db_mtkns.table_gmqnw.student_dob@cm"},"relationshipType":"dataset_process_inputs"}],"query":{"guid":"-7790679096914345","typeName":"hive_process","uniqueAttributes":{"qualifiedName":"db_mtkns.ctas_table_vwsci@cm:1627372412000"},"relationshipType":"hive_process_column_lineage"}},"proxy":false}]}}}{code}
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)