You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@atlas.apache.org by "Saisharath Reddy Kondakindi (Jira)" <ji...@apache.org> on 2021/04/07 04:01:00 UTC
[jira] [Commented] (ATLAS-3507) Apache atlas column lineage not
found
[ https://issues.apache.org/jira/browse/ATLAS-3507?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17315946#comment-17315946 ]
Saisharath Reddy Kondakindi commented on ATLAS-3507:
----------------------------------------------------
same problem, is there any solution ?
can someone share Api process to setup column level lineage in Atlas.
> Apache atlas column lineage not found
> -------------------------------------
>
> Key: ATLAS-3507
> URL: https://issues.apache.org/jira/browse/ATLAS-3507
> Project: Atlas
> Issue Type: Bug
> Components: atlas-webui
> Affects Versions: 2.0.0
> Reporter: dazi
> Priority: Blocker
> Labels: columnLineage
> Attachments: RelationShip.png, columnInfo.png, noColumnLineage.png, tableLineage.png
>
>
> I use Hive2.1.1 and Atlas2.0.0
> I can found table lineage but not found column level lineage
> the detailed information is shown in attchment.
>
> For debug,
>
> I try Restful first,nothing about column lineage.
>
> {{[root@cent1 bin]# curl -X GET -u admin:admin http://cent1:21000/api/atlas/v2/lineage/27c81b16-b422-4479-84b9-2d643b5dba48{"baseEntityGuid":"27c81b16-b422-4479-84b9-2d643b5dba48","lineageDirection":"BOTH","lineageDepth":3,"guidEntityMap":{},"relations":[]}}}
>
> I print Kafka msg from ATLAS_HOOK topic below:
> {code:java}
> {
> "version":{
> "version":"1.0.0",
> "versionParts":[
> 1
> ]
> },
> "msgCompressionKind":"NONE",
> "msgSplitIdx":1,
> "msgSplitCount":1,
> "msgSourceIP":"192.168.10.128",
> "msgCreatedBy":"root",
> "msgCreationTime":1572687183793,
> "message":{
> "type":"ENTITY_CREATE_V2",
> "user":"root",
> "entities":{
> "referredEntities":{
> "-53640556650491":{
> "typeName":"hive_column",
> "attributes":{
> "owner":"root",
> "qualifiedName":"atlas1.pokes.bar@primary",
> "name":"bar",
> "comment":null,
> "position":1,
> "type":"string",
> "table":{
> "guid":"-53640556650488",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes@primary"
> }
> }
> },
> "guid":"-53640556650491",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650492":{
> "typeName":"hive_table",
> "attributes":{
> "owner":"root",
> "temporary":false,
> "lastAccessTime":1572687176000,
> "qualifiedName":"atlas1.pokes_create3@primary",
> "columns":[
> {
> "guid":"-53640556650494",
> "typeName":"hive_column",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3.foo@primary"
> }
> },
> {
> "guid":"-53640556650495",
> "typeName":"hive_column",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3.bar@primary"
> }
> }
> ],
> "tableType":"MANAGED_TABLE",
> "sd":{
> "guid":"-53640556650493",
> "typeName":"hive_storagedesc",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3@primary_storage"
> }
> },
> "createTime":1572687176000,
> "name":"pokes_create3",
> "comment":null,
> "partitionKeys":[
> ],
> "parameters":{
> "totalSize":"5812",
> "numRows":"500",
> "rawDataSize":"5312",
> "COLUMN_STATS_ACCURATE":"{"BASIC_STATS":"true"}",
> "numFiles":"1",
> "transient_lastDdlTime":"1572687178"
> },
> "db":{
> "guid":"-53640556650487",
> "typeName":"hive_db",
> "uniqueAttributes":{
> "qualifiedName":"atlas1@primary"
> }
> },
> "retention":0
> },
> "guid":"-53640556650492",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650490":{
> "typeName":"hive_column",
> "attributes":{
> "owner":"root",
> "qualifiedName":"atlas1.pokes.foo@primary",
> "name":"foo",
> "comment":null,
> "position":0,
> "type":"int",
> "table":{
> "guid":"-53640556650488",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes@primary"
> }
> }
> },
> "guid":"-53640556650490",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650495":{
> "typeName":"hive_column",
> "attributes":{
> "owner":"root",
> "qualifiedName":"atlas1.pokes_create3.bar@primary",
> "name":"bar",
> "comment":null,
> "position":1,
> "type":"string",
> "table":{
> "guid":"-53640556650492",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3@primary"
> }
> }
> },
> "guid":"-53640556650495",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650493":{
> "typeName":"hive_storagedesc",
> "attributes":{
> "qualifiedName":"atlas1.pokes_create3@primary_storage",
> "storedAsSubDirectories":false,
> "location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes_create3",
> "compressed":false,
> "inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
> "parameters":{
> },
> "outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
> "table":{
> "guid":"-53640556650492",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3@primary"
> }
> },
> "serdeInfo":{
> "typeName":"hive_serde",
> "attributes":{
> "serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
> "name":null,
> "parameters":{
> "serialization.format":"1"
> }
> }
> },
> "numBuckets":-1
> },
> "guid":"-53640556650493",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650494":{
> "typeName":"hive_column",
> "attributes":{
> "owner":"root",
> "qualifiedName":"atlas1.pokes_create3.foo@primary",
> "name":"foo",
> "comment":null,
> "position":0,
> "type":"int",
> "table":{
> "guid":"-53640556650492",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3@primary"
> }
> }
> },
> "guid":"-53640556650494",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650488":{
> "typeName":"hive_table",
> "attributes":{
> "owner":"root",
> "temporary":false,
> "lastAccessTime":1572663197000,
> "qualifiedName":"atlas1.pokes@primary",
> "columns":[
> {
> "guid":"-53640556650490",
> "typeName":"hive_column",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes.foo@primary"
> }
> },
> {
> "guid":"-53640556650491",
> "typeName":"hive_column",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes.bar@primary"
> }
> }
> ],
> "tableType":"MANAGED_TABLE",
> "sd":{
> "guid":"-53640556650489",
> "typeName":"hive_storagedesc",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes@primary_storage"
> }
> },
> "createTime":1572663197000,
> "name":"pokes",
> "comment":null,
> "partitionKeys":[
> ],
> "parameters":{
> "transient_lastDdlTime":"1572663225",
> "totalSize":"5812",
> "numRows":"0",
> "rawDataSize":"0",
> "numFiles":"1"
> },
> "db":{
> "guid":"-53640556650487",
> "typeName":"hive_db",
> "uniqueAttributes":{
> "qualifiedName":"atlas1@primary"
> }
> },
> "retention":0
> },
> "guid":"-53640556650488",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650489":{
> "typeName":"hive_storagedesc",
> "attributes":{
> "qualifiedName":"atlas1.pokes@primary_storage",
> "storedAsSubDirectories":false,
> "location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db/pokes",
> "compressed":false,
> "inputFormat":"org.apache.hadoop.mapred.TextInputFormat",
> "parameters":{
> },
> "outputFormat":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
> "table":{
> "guid":"-53640556650488",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes@primary"
> }
> },
> "serdeInfo":{
> "typeName":"hive_serde",
> "attributes":{
> "serializationLib":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe",
> "name":null,
> "parameters":{
> "serialization.format":"1"
> }
> }
> },
> "numBuckets":-1
> },
> "guid":"-53640556650489",
> "provenanceType":0,
> "version":0,
> "proxy":false
> },
> "-53640556650487":{
> "typeName":"hive_db",
> "attributes":{
> "owner":"root",
> "ownerType":"USER",
> "qualifiedName":"atlas1@primary",
> "clusterName":"primary",
> "name":"atlas1",
> "description":null,
> "location":"hdfs://cent1:9000/user/hive/warehouse/atlas1.db",
> "parameters":{
> }
> },
> "guid":"-53640556650487",
> "provenanceType":0,
> "version":0,
> "proxy":false
> }
> },
> "entities":[
> {
> "typeName":"hive_process",
> "attributes":{
> "outputs":[
> {
> "guid":"-53640556650492",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes_create3@primary"
> }
> }
> ],
> "recentQueries":[
> "create table pokes_create3 as select foo,bar from pokes"
> ],
> "qualifiedName":"atlas1.pokes_create3@primary:1572687176000",
> "inputs":[
> {
> "guid":"-53640556650488",
> "typeName":"hive_table",
> "uniqueAttributes":{
> "qualifiedName":"atlas1.pokes@primary"
> }
> }
> ],
> "name":"create table pokes_create3 as select foo,bar from pokes",
> "queryText":"create table pokes_create3 as select foo,bar from pokes",
> "operationType":"CREATETABLE_AS_SELECT",
> "startTime":1572686979104,
> "queryPlan":"Not Supported",
> "endTime":1572687183754,
> "userName":"root",
> "queryId":"root_20191102172939_5de2901c-49e1-473c-ad2f-7e5f5b3251ee"
> },
> "guid":"-53640556650496",
> "provenanceType":0,
> "version":0,
> "proxy":false
> }
> ]
> }
> }
> }
> {code}
>
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)