You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ni...@apache.org on 2020/03/16 07:10:19 UTC
[atlas] branch master updated: ATLAS-3661 :- Create
'spark_column_lineage' type and relationship definition to add support of
column level lineage (#93)
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 9d6f1f6 ATLAS-3661 :- Create 'spark_column_lineage' type and relationship definition to add support of column level lineage (#93)
9d6f1f6 is described below
commit 9d6f1f6baec75a393d2effa7359ae7ac028fa6b5
Author: Vlad Glinsky <61...@users.noreply.github.com>
AuthorDate: Mon Mar 16 09:10:12 2020 +0200
ATLAS-3661 :- Create 'spark_column_lineage' type and relationship definition to add support of column level lineage (#93)
---
addons/models/1000-Hadoop/1100-spark_model.json | 27 +++++++++++++++++++++++++
1 file changed, 27 insertions(+)
diff --git a/addons/models/1000-Hadoop/1100-spark_model.json b/addons/models/1000-Hadoop/1100-spark_model.json
index 28f24d0..ddb9264 100644
--- a/addons/models/1000-Hadoop/1100-spark_model.json
+++ b/addons/models/1000-Hadoop/1100-spark_model.json
@@ -359,6 +359,14 @@
]
},
{
+ "name" : "spark_column_lineage",
+ "superTypes" : [
+ "Process"
+ ],
+ "serviceType": "spark",
+ "typeVersion" : "1.0"
+ },
+ {
"name": "spark_ml_pipeline",
"superTypes": [
"DataSet"
@@ -470,6 +478,25 @@
"cardinality": "SINGLE"
},
"propagateTags": "NONE"
+ },
+ {
+ "name": "spark_process_column_lineages",
+ "serviceType": "spark",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "endDef1": {
+ "type": "spark_column_lineage",
+ "name": "process",
+ "isContainer": false,
+ "cardinality": "SINGLE"
+ },
+ "endDef2": {
+ "type": "spark_process",
+ "name": "columnLineages",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
}
]
}