You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/09/05 04:50:45 UTC
[1/2] atlas git commit: ATLAS-2112: move the Atlas models to
subfolders
Repository: atlas
Updated Branches:
refs/heads/master 9d38c6a94 -> 1dcc3073a
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1030-hive_model.json b/addons/models/1030-hive_model.json
deleted file mode 100644
index f47a7b9..0000000
--- a/addons/models/1030-hive_model.json
+++ /dev/null
@@ -1,625 +0,0 @@
-{
- "enumDefs": [
- {
- "name": "hive_principal_type",
- "typeVersion": "1.0",
- "elementDefs": [
- {
- "ordinal": 1,
- "value": "USER"
- },
- {
- "ordinal": 2,
- "value": "ROLE"
- },
- {
- "ordinal": 3,
- "value": "GROUP"
- }
- ]
- }
- ],
- "structDefs": [
- {
- "name": "hive_order",
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "order",
- "typeName": "int",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "col",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hive_serde",
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "name",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "serializationLib",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "parameters",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "hive_process",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "startTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "endTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "userName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "operationType",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "queryText",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "queryPlan",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "queryId",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "recentQueries",
- "typeName": "array<string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "clusterName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "queryGraph",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hive_table",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "db",
- "typeName": "hive_db",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "createTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "lastAccessTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "comment",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "retention",
- "typeName": "int",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "sd",
- "typeName": "hive_storagedesc",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "partitionKeys",
- "typeName": "array<hive_column>",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "aliases",
- "typeName": "array<string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "columns",
- "typeName": "array<hive_column>",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "parameters",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "viewOriginalText",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "viewExpandedText",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "tableType",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "temporary",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hive_storagedesc",
- "superTypes": [
- "Referenceable"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "table",
- "typeName": "hive_table",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "sd"
- }
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "location",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "inputFormat",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "outputFormat",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "compressed",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "numBuckets",
- "typeName": "int",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "serdeInfo",
- "typeName": "hive_serde",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "bucketCols",
- "typeName": "array<string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "sortCols",
- "typeName": "array<hive_order>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "parameters",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "storedAsSubDirectories",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hive_db",
- "superTypes": [
- "Referenceable",
- "Asset"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "clusterName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "location",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "parameters",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "ownerType",
- "typeName": "hive_principal_type",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hive_column",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "type",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "comment",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "table",
- "typeName": "hive_table",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "columns"
- }
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name" : "hive_column_lineage",
- "superTypes" : [
- "Process"
- ],
- "typeVersion" : "1.0",
- "attributeDefs" : [
- {
- "name": "query",
- "typeName": "hive_process",
- "cardinality" : "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "depenendencyType",
- "typeName": "string",
- "cardinality" : "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "expression",
- "typeName": "string",
- "cardinality" : "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ],
- "relationshipDefs": [
- {
- "name": "hive_db_tables",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hive_db",
- "name": "tables",
- "isContainer": true,
- "cardinality": "SET"
- },
- "endDef2": {
- "type": "hive_table",
- "name": "db",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- },
- {
- "name": "hive_table_columns",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hive_table",
- "name": "columns",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "hive_column",
- "name": "table",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- },
- {
- "name": "hive_table_partitionkeys",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hive_table",
- "name": "partitionKeys",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "hive_column",
- "name": "table",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- },
- {
- "name": "hive_table_storagedesc",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hive_table",
- "name": "sd",
- "isContainer": true,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "hive_storagedesc",
- "name": "table",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- },
- {
- "name": "hive_process_column_lineage",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hive_process",
- "name": "columnLineages",
- "isContainer": true,
- "cardinality": "SET"
- },
- "endDef2": {
- "type": "hive_column_lineage",
- "name": "query",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "NONE"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1040-sqoop_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1040-sqoop_model.json b/addons/models/1040-sqoop_model.json
deleted file mode 100644
index f5c7fd9..0000000
--- a/addons/models/1040-sqoop_model.json
+++ /dev/null
@@ -1,97 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "sqoop_process",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "operation",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "commandlineOpts",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "startTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "endTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "userName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "sqoop_dbdatastore",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "dbStoreType",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "storeUse",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "storeUri",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "source",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1050-falcon_model.json b/addons/models/1050-falcon_model.json
deleted file mode 100644
index 4fa4604..0000000
--- a/addons/models/1050-falcon_model.json
+++ /dev/null
@@ -1,206 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "falcon_feed_replication",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": []
- },
- {
- "name": "falcon_cluster",
- "superTypes": [
- "Infrastructure"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "colo",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "tags",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "falcon_feed",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "frequency",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "stored-in",
- "typeName": "falcon_cluster",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "groups",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "tags",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "falcon_process",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "frequency",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "runs-on",
- "typeName": "falcon_cluster",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "tags",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "pipelines",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "workflow-properties",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "falcon_feed_creation",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "stored-in",
- "typeName": "falcon_cluster",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- }
- ]
- }
- ],
- "relationshipDefs": [
- {
- "name": "falcon_feed_cluster",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "falcon_feed",
- "name": "stored-in",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "falcon_cluster",
- "name": "feeds",
- "isContainer": true,
- "cardinality": "SET"
- },
- "propagateTags": "NONE"
- },
- {
- "name": "falcon_cluster_process",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "falcon_cluster",
- "name": "processes",
- "isContainer": true,
- "cardinality": "SET"
- },
- "endDef2": {
- "type": "falcon_process",
- "name": "runs-on",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "NONE"
- },
- {
- "name": "falcon_cluster_feed_creation",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "falcon_cluster",
- "name": "feedCreations",
- "isContainer": true,
- "cardinality": "SET"
- },
- "endDef2": {
- "type": "falcon_feed_creation",
- "name": "stored-in",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "NONE"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1060-hbase_model.json b/addons/models/1060-hbase_model.json
deleted file mode 100644
index 3e46e06..0000000
--- a/addons/models/1060-hbase_model.json
+++ /dev/null
@@ -1,142 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "hbase_table",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.1",
- "attributeDefs": [
- {
- "name": "uri",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "column_families",
- "typeName": "array<hbase_column_family>",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hbase_column_family",
- "superTypes": [
- "DataSet"
- ],
- "attributeDefs": [
- {
- "name": "table",
- "typeName": "hbase_table",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false,
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "column_families"
- }
- }
- ]
- },
- {
- "name": "columns",
- "typeName": "array<hbase_column>",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ],
- "typeVersion": "1.0"
- },
- {
- "name": "hbase_column",
- "superTypes": [
- "DataSet"
- ],
- "attributeDefs": [
- {
- "name": "column_family",
- "typeName": "hbase_column_family",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false,
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "columns"
- }
- }
- ]
- }
- ],
- "typeVersion": "1.0"
- }
- ],
- "relationshipDefs": [
- {
- "name": "hbase_table_column_families",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hbase_table",
- "name": "column_families",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "hbase_column_family",
- "name": "table",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- },
- {
- "name": "hbase_column_family_columns",
- "typeVersion": "1.0",
- "relationshipCategory": "COMPOSITION",
- "endDef1": {
- "type": "hbase_column_family",
- "name": "columns",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "hbase_column",
- "name": "column_family",
- "isContainer": false,
- "cardinality": "SINGLE",
- "isLegacyAttribute": true
- },
- "propagateTags": "ONE_TO_TWO"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1070-kafka_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1070-kafka_model.json b/addons/models/1070-kafka_model.json
deleted file mode 100644
index b7f6e33..0000000
--- a/addons/models/1070-kafka_model.json
+++ /dev/null
@@ -1,57 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "kafka_topic",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "topic",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": true
- },
- {
- "name": "uri",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- }
- ]
- },
- {
- "name": "jms_topic",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "topic",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": true
- },
- {
- "name": "uri",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1080-storm_model.json b/addons/models/1080-storm_model.json
deleted file mode 100644
index b008c7a..0000000
--- a/addons/models/1080-storm_model.json
+++ /dev/null
@@ -1,167 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "storm_topology",
- "superTypes": [
- "Process"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "id",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": true
- },
- {
- "name": "startTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "endTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "conf",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "clusterName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "nodes",
- "typeName": "array<storm_node>",
- "cardinality": "LIST",
- "constraints": [],
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- }
- ]
- },
- {
- "name": "storm_node",
- "superTypes": [],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "name",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "description",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "driverClass",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "conf",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "storm_spout",
- "superTypes": [
- "storm_node"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "outputs",
- "typeName": "array<string>",
- "cardinality": "LIST",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- }
- ]
- },
- {
- "name": "storm_bolt",
- "superTypes": [
- "storm_node"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "inputs",
- "typeName": "array<string>",
- "cardinality": "LIST",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "outputs",
- "typeName": "array<string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ],
- "relationshipDefs": [
- {
- "name": "storm_topology_nodes",
- "typeVersion": "1.0",
- "relationshipCategory": "ASSOCIATION",
- "endDef1": {
- "type": "storm_topology",
- "name": "nodes",
- "isContainer": false,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "storm_node",
- "name": "topolgies",
- "isContainer": false,
- "cardinality": "SET"
- },
- "propagateTags": "NONE"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/patches/001-hive_column_add_position.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/001-hive_column_add_position.json b/addons/models/patches/001-hive_column_add_position.json
deleted file mode 100644
index eeab74a..0000000
--- a/addons/models/patches/001-hive_column_add_position.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "patches": [
- {
- "action": "ADD_ATTRIBUTE",
- "typeName": "hive_column",
- "applyToVersion": "1.0",
- "updateToVersion": "1.1",
- "params": null,
- "attributeDefs": [
- {
- "name": "position",
- "typeName": "int",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/patches/002-hive_column_table_add_options.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/002-hive_column_table_add_options.json b/addons/models/patches/002-hive_column_table_add_options.json
deleted file mode 100644
index 5f00bd2..0000000
--- a/addons/models/patches/002-hive_column_table_add_options.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "patches": [
- {
- "action": "UPDATE_TYPEDEF_OPTIONS",
- "typeName": "hive_column",
- "applyToVersion": "1.1",
- "updateToVersion": "1.2",
- "typeDefOptions": {
- "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
- }
- },
- {
- "action": "UPDATE_TYPEDEF_OPTIONS",
- "typeName": "hive_table",
- "applyToVersion": "1.0",
- "updateToVersion": "1.1",
- "typeDefOptions": {
- "schemaElementsAttribute": "columns"
- }
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/patches/003-hive_column_update_table_remove_constraint.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/003-hive_column_update_table_remove_constraint.json b/addons/models/patches/003-hive_column_update_table_remove_constraint.json
deleted file mode 100644
index db80722..0000000
--- a/addons/models/patches/003-hive_column_update_table_remove_constraint.json
+++ /dev/null
@@ -1,21 +0,0 @@
-{
- "patches": [
- {
- "action": "UPDATE_ATTRIBUTE",
- "typeName": "hive_column",
- "applyToVersion": "1.2",
- "updateToVersion": "1.3",
- "params": null,
- "attributeDefs": [
- {
- "name": "table",
- "typeName": "hive_table",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ]
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/sqoop-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index b6273b6..17e1136 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -429,10 +429,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/target/models</outputDirectory>
+ <outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../models</directory>
+ <directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/storm-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index a5845ed..73d26f4 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -501,10 +501,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/target/models</outputDirectory>
+ <outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../models</directory>
+ <directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 66cedd3..e11cd12 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -69,6 +69,7 @@ import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONL
@Service
public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
+ public static final String PATCHES_FOLDER_NAME = "patches";
private final AtlasTypeDefStore atlasTypeDefStore;
private final AtlasTypeRegistry atlasTypeRegistry;
@@ -97,57 +98,93 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
LOG.info("<== AtlasTypeDefStoreInitializer.init()");
}
+ /**
+ * This method is looking for folders in alphabetical order in the models directory. It loads each of these folders and their associated patches in order.
+ * It then loads any models in the top level folder and its patches.
+ *
+ * This allows models to be grouped into folders to help managability.
+ *
+ */
private void loadBootstrapTypeDefs() {
LOG.info("==> AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
- String atlasHomeDir = System.getProperty("atlas.home");
- String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
- File typesDir = new File(typesDirName);
- File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
+ String atlasHomeDir = System.getProperty("atlas.home");
+ String modelsDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
- if (typeDefFiles == null || typeDefFiles.length == 0) {
- LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
+ if (modelsDirName == null || modelsDirName.length() == 0) {
+ LOG.info("Types directory {} does not exist or not readable or has no typedef files", modelsDirName);
+ } else {
+ // look for folders we need to load models from
+ File topModeltypesDir = new File(modelsDirName);
+ File[] modelsDirContents = topModeltypesDir.exists() ? topModeltypesDir.listFiles() : null;
+
+ Arrays.sort(modelsDirContents);
- return;
+ for (File folder : modelsDirContents) {
+ if (folder.isFile()) {
+ // ignore files
+ continue;
+ } else if (!folder.getName().equals(PATCHES_FOLDER_NAME)){
+ // load the models alphabetically in the subfolders apart from patches
+ loadModelsInFolder(folder);
+ }
+ }
+
+ // load any files in the top models folder and any associated patches.
+ loadModelsInFolder(topModeltypesDir);
}
+ LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
+ }
- // sort the files by filename
- Arrays.sort(typeDefFiles);
+ /**
+ * Load all the model files in the supplied folder followed by the contents of the patches folder.
+ * @param typesDir
+ */
+ private void loadModelsInFolder(File typesDir) {
+ LOG.info("==> AtlasTypeDefStoreInitializer({})", typesDir);
- for (File typeDefFile : typeDefFiles) {
- if (!typeDefFile.isFile()) {
- continue;
- }
+ String typesDirName = typesDir.getName();
+ File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
- try {
- String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
- AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+ if (typeDefFiles == null || typeDefFiles.length == 0) {
+ LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName );
+ } else {
- if (typesDef == null || typesDef.isEmpty()) {
- LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
+ // sort the files by filename
+ Arrays.sort(typeDefFiles);
- continue;
- }
+ for (File typeDefFile : typeDefFiles) {
+ if (typeDefFile.isFile()) {
+ try {
+ String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
- AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
- AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry);
+ if (typesDef == null || typesDef.isEmpty()) {
+ LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
- if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
- atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
+ continue;
+ }
- LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
- } else {
- LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
- }
+ AtlasTypesDef typesToCreate = getTypesToCreate(typesDef, atlasTypeRegistry);
+ AtlasTypesDef typesToUpdate = getTypesToUpdate(typesDef, atlasTypeRegistry);
- } catch (Throwable t) {
- LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
- }
- }
+ if (!typesToCreate.isEmpty() || !typesToUpdate.isEmpty()) {
+ atlasTypeDefStore.createUpdateTypesDef(typesToCreate, typesToUpdate);
- applyTypePatches(typesDirName);
+ LOG.info("Created/Updated types defined in file {}", typeDefFile.getAbsolutePath());
+ } else {
+ LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
+ }
- LOG.info("<== AtlasTypeDefStoreInitializer.loadBootstrapTypeDefs()");
+ } catch (Throwable t) {
+ LOG.error("error while registering types in file {}", typeDefFile.getAbsolutePath(), t);
+ }
+ }
+ }
+
+ applyTypePatches(typesDir.getPath());
+ }
+ LOG.info("<== AtlasTypeDefStoreInitializer({})", typesDir);
}
public static AtlasTypesDef getTypesToCreate(AtlasTypesDef typesDef, AtlasTypeRegistry typeRegistry) {
@@ -327,68 +364,67 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
}
private void applyTypePatches(String typesDirName) {
- String typePatchesDirName = typesDirName + File.separator + "patches";
+ String typePatchesDirName = typesDirName + File.separator + PATCHES_FOLDER_NAME;
File typePatchesDir = new File(typePatchesDirName);
File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
if (typePatchFiles == null || typePatchFiles.length == 0) {
LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName);
+ } else {
+ LOG.info("Type patches directory {} is being processed", typePatchesDirName);
- return;
- }
-
- // sort the files by filename
- Arrays.sort(typePatchFiles);
+ // sort the files by filename
+ Arrays.sort(typePatchFiles);
- PatchHandler[] patchHandlers = new PatchHandler[] {
- new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
- new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
- new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
- };
+ PatchHandler[] patchHandlers = new PatchHandler[] {
+ new AddAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry),
+ new UpdateTypeDefOptionsPatchHandler(atlasTypeDefStore, atlasTypeRegistry),
+ new UpdateAttributePatchHandler(atlasTypeDefStore, atlasTypeRegistry)
+ };
- Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
+ Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
- for (PatchHandler patchHandler : patchHandlers) {
- for (String supportedAction : patchHandler.getSupportedActions()) {
- patchHandlerRegistry.put(supportedAction, patchHandler);
+ for (PatchHandler patchHandler : patchHandlers) {
+ for (String supportedAction : patchHandler.getSupportedActions()) {
+ patchHandlerRegistry.put(supportedAction, patchHandler);
+ }
}
- }
- for (File typePatchFile : typePatchFiles) {
- if (!typePatchFile.isFile()) {
- continue;
- }
+ for (File typePatchFile : typePatchFiles) {
+ if (typePatchFile.isFile()) {
- LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
+ LOG.info("Applying patches in file {}", typePatchFile.getAbsolutePath());
- try {
- String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
- TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
+ try {
+ String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
+ TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
- if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
- LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
+ if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
+ LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
- continue;
- }
+ continue;
+ }
- for (TypeDefPatch patch : patches.getPatches()) {
- PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
+ for (TypeDefPatch patch : patches.getPatches()) {
+ PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
- if (patchHandler == null) {
- LOG.error("Unknown patch action {} in file {}. Ignored",
- patch.getAction(), typePatchFile.getAbsolutePath());
+ if (patchHandler == null) {
+ LOG.error("Unknown patch action {} in file {}. Ignored",
+ patch.getAction(), typePatchFile.getAbsolutePath());
- continue;
- }
+ continue;
+ }
- try {
- patchHandler.applyPatch(patch);
- } catch (AtlasBaseException excp) {
- LOG.error("Failed to apply {} patch in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath(), excp);
+ try {
+ patchHandler.applyPatch(patch);
+ } catch (AtlasBaseException excp) {
+ LOG.error("Failed to apply {} patch in file {}. Ignored", patch.getAction(), typePatchFile.getAbsolutePath(), excp);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to apply patches in file {}. Ignored", typePatchFile.getAbsolutePath(), t);
}
}
- } catch (Throwable t) {
- LOG.error("Failed to apply patches in file {}. Ignored", typePatchFile.getAbsolutePath(), t);
}
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
index e1461dd..17813d6 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ImportServiceTest.java
@@ -68,7 +68,7 @@ public class ImportServiceTest {
@Test(dataProvider = "sales")
public void importDB1(ZipSource zipSource) throws AtlasBaseException, IOException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
@@ -79,7 +79,7 @@ public class ImportServiceTest {
@Test(dataProvider = "reporting")
public void importDB2(ZipSource zipSource) throws AtlasBaseException, IOException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
@@ -90,7 +90,7 @@ public class ImportServiceTest {
@Test(dataProvider = "logging")
public void importDB3(ZipSource zipSource) throws AtlasBaseException, IOException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runAndVerifyQuickStart_v1_Import(new ImportService(typeDefStore, entityStore, typeRegistry), zipSource);
}
@@ -101,7 +101,7 @@ public class ImportServiceTest {
@Test(dataProvider = "salesNewTypeAttrs", dependsOnMethods = "importDB1")
public void importDB4(ZipSource zipSource) throws AtlasBaseException, IOException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
runImportWithParameters(new ImportService(typeDefStore, entityStore, typeRegistry), getDefaultImportRequest(), zipSource);
}
@@ -150,8 +150,8 @@ public class ImportServiceTest {
@Test(dataProvider = "ctas")
public void importCTAS(ZipSource zipSource) throws IOException, AtlasBaseException {
- loadModelFromJson("0010-base_model.json", typeDefStore, typeRegistry);
- loadModelFromJson("1030-hive_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
runImportWithNoParameters(getImportService(), zipSource);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/webapp/pom.xml
----------------------------------------------------------------------
diff --git a/webapp/pom.xml b/webapp/pom.xml
index 2a06027..58c2e0a 100755
--- a/webapp/pom.xml
+++ b/webapp/pom.xml
@@ -688,10 +688,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/target/models</outputDirectory>
+ <outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../addons/models</directory>
+ <directory>${basedir}/../addons/models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
[2/2] atlas git commit: ATLAS-2112: move the Atlas models to
subfolders
Posted by ma...@apache.org.
ATLAS-2112: move the Atlas models to subfolders
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/1dcc3073
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/1dcc3073
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/1dcc3073
Branch: refs/heads/master
Commit: 1dcc3073abe7b920c9a0fb1fd3055b8daa22cf1e
Parents: 9d38c6a
Author: David Radley <da...@apache.org>
Authored: Mon Sep 4 21:24:15 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Sep 4 21:50:25 2017 -0700
----------------------------------------------------------------------
addons/falcon-bridge/pom.xml | 4 +-
addons/hive-bridge/pom.xml | 4 +-
addons/models/0000-Area0/0010-base_model.json | 153 +++++
addons/models/0010-base_model.json | 153 -----
addons/models/1000-Hadoop/1020-fs_model.json | 191 ++++++
addons/models/1000-Hadoop/1030-hive_model.json | 625 +++++++++++++++++++
addons/models/1000-Hadoop/1040-sqoop_model.json | 97 +++
.../models/1000-Hadoop/1050-falcon_model.json | 206 ++++++
addons/models/1000-Hadoop/1060-hbase_model.json | 142 +++++
addons/models/1000-Hadoop/1070-kafka_model.json | 57 ++
addons/models/1000-Hadoop/1080-storm_model.json | 167 +++++
.../patches/001-hive_column_add_position.json | 21 +
.../002-hive_column_table_add_options.json | 22 +
...e_column_update_table_remove_constraint.json | 21 +
addons/models/1020-fs_model.json | 191 ------
addons/models/1030-hive_model.json | 625 -------------------
addons/models/1040-sqoop_model.json | 97 ---
addons/models/1050-falcon_model.json | 206 ------
addons/models/1060-hbase_model.json | 142 -----
addons/models/1070-kafka_model.json | 57 --
addons/models/1080-storm_model.json | 167 -----
.../patches/001-hive_column_add_position.json | 21 -
.../002-hive_column_table_add_options.json | 22 -
...e_column_update_table_remove_constraint.json | 21 -
addons/sqoop-bridge/pom.xml | 4 +-
addons/storm-bridge/pom.xml | 4 +-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 186 +++---
.../repository/impexp/ImportServiceTest.java | 12 +-
webapp/pom.xml | 4 +-
29 files changed, 1829 insertions(+), 1793 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index e07295d..6482203 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -368,10 +368,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/target/models</outputDirectory>
+ <outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../models</directory>
+ <directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 8ee0881..5a9bfed 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -417,10 +417,10 @@
<goal>copy-resources</goal>
</goals>
<configuration>
- <outputDirectory>${basedir}/target/models</outputDirectory>
+ <outputDirectory>${basedir}/target/models/1000-Hadoop</outputDirectory>
<resources>
<resource>
- <directory>${basedir}/../models</directory>
+ <directory>${basedir}/../models/1000-Hadoop</directory>
<filtering>true</filtering>
</resource>
</resources>
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/0000-Area0/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json
new file mode 100644
index 0000000..5865fb0
--- /dev/null
+++ b/addons/models/0000-Area0/0010-base_model.json
@@ -0,0 +1,153 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [
+ {
+ "name": "TaxonomyTerm",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "atlas.taxonomy",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "entityDefs": [
+ {
+ "name": "Referenceable",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "qualifiedName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ }
+ ]
+ },
+ {
+ "name": "Asset",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "owner",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "DataSet",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Infrastructure",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Process",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "dataset_process_inputs",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "endDef1": {
+ "type": "DataSet",
+ "name": "sourceToProcesses",
+ "isContainer": false,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "Process",
+ "name": "inputs",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "process_dataset_outputs",
+ "typeVersion": "1.0",
+ "relationshipCategory": "AGGREGATION",
+ "endDef1": {
+ "type": "Process",
+ "name": "outputs",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "DataSet",
+ "name": "sinkFromProcesses",
+ "isContainer": false,
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
deleted file mode 100644
index 5865fb0..0000000
--- a/addons/models/0010-base_model.json
+++ /dev/null
@@ -1,153 +0,0 @@
-{
- "enumDefs": [],
- "structDefs": [],
- "classificationDefs": [
- {
- "name": "TaxonomyTerm",
- "superTypes": [],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "atlas.taxonomy",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ],
- "entityDefs": [
- {
- "name": "Referenceable",
- "superTypes": [],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "qualifiedName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": true
- }
- ]
- },
- {
- "name": "Asset",
- "superTypes": [],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "name",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "description",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "owner",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "DataSet",
- "superTypes": [
- "Referenceable",
- "Asset"
- ],
- "typeVersion": "1.0",
- "attributeDefs": []
- },
- {
- "name": "Infrastructure",
- "superTypes": [
- "Referenceable",
- "Asset"
- ],
- "typeVersion": "1.0",
- "attributeDefs": []
- },
- {
- "name": "Process",
- "superTypes": [
- "Referenceable",
- "Asset"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "inputs",
- "typeName": "array<DataSet>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "outputs",
- "typeName": "array<DataSet>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ],
- "relationshipDefs": [
- {
- "name": "dataset_process_inputs",
- "typeVersion": "1.0",
- "relationshipCategory": "AGGREGATION",
- "endDef1": {
- "type": "DataSet",
- "name": "sourceToProcesses",
- "isContainer": false,
- "cardinality": "SET"
- },
- "endDef2": {
- "type": "Process",
- "name": "inputs",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "propagateTags": "NONE"
- },
- {
- "name": "process_dataset_outputs",
- "typeVersion": "1.0",
- "relationshipCategory": "AGGREGATION",
- "endDef1": {
- "type": "Process",
- "name": "outputs",
- "isContainer": true,
- "cardinality": "SET",
- "isLegacyAttribute": true
- },
- "endDef2": {
- "type": "DataSet",
- "name": "sinkFromProcesses",
- "isContainer": false,
- "cardinality": "SET"
- },
- "propagateTags": "NONE"
- }
- ]
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1020-fs_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1020-fs_model.json b/addons/models/1000-Hadoop/1020-fs_model.json
new file mode 100644
index 0000000..ff17261
--- /dev/null
+++ b/addons/models/1000-Hadoop/1020-fs_model.json
@@ -0,0 +1,191 @@
+{
+ "enumDefs": [
+ {
+ "name": "file_action",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "ordinal": 0,
+ "value": "NONE"
+ },
+ {
+ "ordinal": 1,
+ "value": "EXECUTE"
+ },
+ {
+ "ordinal": 2,
+ "value": "WRITE"
+ },
+ {
+ "ordinal": 3,
+ "value": "WRITE_EXECUTE"
+ },
+ {
+ "ordinal": 4,
+ "value": "READ"
+ },
+ {
+ "ordinal": 5,
+ "value": "READ_EXECUTE"
+ },
+ {
+ "ordinal": 6,
+ "value": "READ_WRITE"
+ },
+ {
+ "ordinal": 7,
+ "value": "ALL"
+ }
+ ]
+ }
+ ],
+ "structDefs": [
+ {
+ "name": "fs_permissions",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "group",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "user",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "sticky",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "others",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "fs_path",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "path",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isFile",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isSymlink",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "fileSize",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "group",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "posixPermissions",
+ "typeName": "fs_permissions",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hdfs_path",
+ "superTypes": [
+ "fs_path"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "numberOfReplicas",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "extendedAttributes",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1030-hive_model.json b/addons/models/1000-Hadoop/1030-hive_model.json
new file mode 100644
index 0000000..f47a7b9
--- /dev/null
+++ b/addons/models/1000-Hadoop/1030-hive_model.json
@@ -0,0 +1,625 @@
+{
+ "enumDefs": [
+ {
+ "name": "hive_principal_type",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "ordinal": 1,
+ "value": "USER"
+ },
+ {
+ "ordinal": 2,
+ "value": "ROLE"
+ },
+ {
+ "ordinal": 3,
+ "value": "GROUP"
+ }
+ ]
+ }
+ ],
+ "structDefs": [
+ {
+ "name": "hive_order",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "order",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "col",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_serde",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "serializationLib",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "hive_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operationType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryPlan",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryId",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "recentQueries",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "queryGraph",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_table",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "db",
+ "typeName": "hive_db",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "lastAccessTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "retention",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "sd",
+ "typeName": "hive_storagedesc",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "partitionKeys",
+ "typeName": "array<hive_column>",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "aliases",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "columns",
+ "typeName": "array<hive_column>",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "viewOriginalText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "viewExpandedText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "tableType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "temporary",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_storagedesc",
+ "superTypes": [
+ "Referenceable"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "sd"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "inputFormat",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "outputFormat",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "compressed",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "numBuckets",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "serdeInfo",
+ "typeName": "hive_serde",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "bucketCols",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "sortCols",
+ "typeName": "array<hive_order>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "storedAsSubDirectories",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_db",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "ownerType",
+ "typeName": "hive_principal_type",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_column",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "type",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "columns"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name" : "hive_column_lineage",
+ "superTypes" : [
+ "Process"
+ ],
+ "typeVersion" : "1.0",
+ "attributeDefs" : [
+ {
+ "name": "query",
+ "typeName": "hive_process",
+ "cardinality" : "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "depenendencyType",
+ "typeName": "string",
+ "cardinality" : "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "expression",
+ "typeName": "string",
+ "cardinality" : "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "hive_db_tables",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_db",
+ "name": "tables",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "hive_table",
+ "name": "db",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_columns",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "columns",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "hive_column",
+ "name": "table",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_partitionkeys",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "partitionKeys",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "hive_column",
+ "name": "table",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_table_storagedesc",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_table",
+ "name": "sd",
+ "isContainer": true,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "hive_storagedesc",
+ "name": "table",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hive_process_column_lineage",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hive_process",
+ "name": "columnLineages",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "hive_column_lineage",
+ "name": "query",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "NONE"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1040-sqoop_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1040-sqoop_model.json b/addons/models/1000-Hadoop/1040-sqoop_model.json
new file mode 100644
index 0000000..f5c7fd9
--- /dev/null
+++ b/addons/models/1000-Hadoop/1040-sqoop_model.json
@@ -0,0 +1,97 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "sqoop_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "operation",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "commandlineOpts",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "sqoop_dbdatastore",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "dbStoreType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "storeUse",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "storeUri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "source",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1050-falcon_model.json b/addons/models/1000-Hadoop/1050-falcon_model.json
new file mode 100644
index 0000000..4fa4604
--- /dev/null
+++ b/addons/models/1000-Hadoop/1050-falcon_model.json
@@ -0,0 +1,206 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "falcon_feed_replication",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "falcon_cluster",
+ "superTypes": [
+ "Infrastructure"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "colo",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_feed",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "groups",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "runs-on",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "pipelines",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "workflow-properties",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_feed_creation",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "falcon_feed_cluster",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_feed",
+ "name": "stored-in",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "falcon_cluster",
+ "name": "feeds",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "falcon_cluster_process",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_cluster",
+ "name": "processes",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "falcon_process",
+ "name": "runs-on",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "NONE"
+ },
+ {
+ "name": "falcon_cluster_feed_creation",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "falcon_cluster",
+ "name": "feedCreations",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
+ "endDef2": {
+ "type": "falcon_feed_creation",
+ "name": "stored-in",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "NONE"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1060-hbase_model.json b/addons/models/1000-Hadoop/1060-hbase_model.json
new file mode 100644
index 0000000..3e46e06
--- /dev/null
+++ b/addons/models/1000-Hadoop/1060-hbase_model.json
@@ -0,0 +1,142 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "hbase_table",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.1",
+ "attributeDefs": [
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "column_families",
+ "typeName": "array<hbase_column_family>",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hbase_column_family",
+ "superTypes": [
+ "DataSet"
+ ],
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hbase_table",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "column_families"
+ }
+ }
+ ]
+ },
+ {
+ "name": "columns",
+ "typeName": "array<hbase_column>",
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ],
+ "typeVersion": "1.0"
+ },
+ {
+ "name": "hbase_column",
+ "superTypes": [
+ "DataSet"
+ ],
+ "attributeDefs": [
+ {
+ "name": "column_family",
+ "typeName": "hbase_column_family",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "columns"
+ }
+ }
+ ]
+ }
+ ],
+ "typeVersion": "1.0"
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "hbase_table_column_families",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hbase_table",
+ "name": "column_families",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "hbase_column_family",
+ "name": "table",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ },
+ {
+ "name": "hbase_column_family_columns",
+ "typeVersion": "1.0",
+ "relationshipCategory": "COMPOSITION",
+ "endDef1": {
+ "type": "hbase_column_family",
+ "name": "columns",
+ "isContainer": true,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "hbase_column",
+ "name": "column_family",
+ "isContainer": false,
+ "cardinality": "SINGLE",
+ "isLegacyAttribute": true
+ },
+ "propagateTags": "ONE_TO_TWO"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1070-kafka_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1070-kafka_model.json b/addons/models/1000-Hadoop/1070-kafka_model.json
new file mode 100644
index 0000000..b7f6e33
--- /dev/null
+++ b/addons/models/1000-Hadoop/1070-kafka_model.json
@@ -0,0 +1,57 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "kafka_topic",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "jms_topic",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/1080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/1080-storm_model.json b/addons/models/1000-Hadoop/1080-storm_model.json
new file mode 100644
index 0000000..b008c7a
--- /dev/null
+++ b/addons/models/1000-Hadoop/1080-storm_model.json
@@ -0,0 +1,167 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "storm_topology",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "id",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "nodes",
+ "typeName": "array<storm_node>",
+ "cardinality": "LIST",
+ "constraints": [],
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_node",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "driverClass",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_spout",
+ "superTypes": [
+ "storm_node"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "cardinality": "LIST",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_bolt",
+ "superTypes": [
+ "storm_node"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<string>",
+ "cardinality": "LIST",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "relationshipDefs": [
+ {
+ "name": "storm_topology_nodes",
+ "typeVersion": "1.0",
+ "relationshipCategory": "ASSOCIATION",
+ "endDef1": {
+ "type": "storm_topology",
+ "name": "nodes",
+ "isContainer": false,
+ "cardinality": "SET",
+ "isLegacyAttribute": true
+ },
+ "endDef2": {
+ "type": "storm_node",
+ "name": "topolgies",
+ "isContainer": false,
+ "cardinality": "SET"
+ },
+ "propagateTags": "NONE"
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/patches/001-hive_column_add_position.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/patches/001-hive_column_add_position.json b/addons/models/1000-Hadoop/patches/001-hive_column_add_position.json
new file mode 100644
index 0000000..eeab74a
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/001-hive_column_add_position.json
@@ -0,0 +1,21 @@
+{
+ "patches": [
+ {
+ "action": "ADD_ATTRIBUTE",
+ "typeName": "hive_column",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "position",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/patches/002-hive_column_table_add_options.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/patches/002-hive_column_table_add_options.json b/addons/models/1000-Hadoop/patches/002-hive_column_table_add_options.json
new file mode 100644
index 0000000..5f00bd2
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/002-hive_column_table_add_options.json
@@ -0,0 +1,22 @@
+{
+ "patches": [
+ {
+ "action": "UPDATE_TYPEDEF_OPTIONS",
+ "typeName": "hive_column",
+ "applyToVersion": "1.1",
+ "updateToVersion": "1.2",
+ "typeDefOptions": {
+ "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
+ }
+ },
+ {
+ "action": "UPDATE_TYPEDEF_OPTIONS",
+ "typeName": "hive_table",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "typeDefOptions": {
+ "schemaElementsAttribute": "columns"
+ }
+ }
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json
----------------------------------------------------------------------
diff --git a/addons/models/1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json b/addons/models/1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json
new file mode 100644
index 0000000..db80722
--- /dev/null
+++ b/addons/models/1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json
@@ -0,0 +1,21 @@
+{
+ "patches": [
+ {
+ "action": "UPDATE_ATTRIBUTE",
+ "typeName": "hive_column",
+ "applyToVersion": "1.2",
+ "updateToVersion": "1.3",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/1dcc3073/addons/models/1020-fs_model.json
----------------------------------------------------------------------
diff --git a/addons/models/1020-fs_model.json b/addons/models/1020-fs_model.json
deleted file mode 100644
index ff17261..0000000
--- a/addons/models/1020-fs_model.json
+++ /dev/null
@@ -1,191 +0,0 @@
-{
- "enumDefs": [
- {
- "name": "file_action",
- "typeVersion": "1.0",
- "elementDefs": [
- {
- "ordinal": 0,
- "value": "NONE"
- },
- {
- "ordinal": 1,
- "value": "EXECUTE"
- },
- {
- "ordinal": 2,
- "value": "WRITE"
- },
- {
- "ordinal": 3,
- "value": "WRITE_EXECUTE"
- },
- {
- "ordinal": 4,
- "value": "READ"
- },
- {
- "ordinal": 5,
- "value": "READ_EXECUTE"
- },
- {
- "ordinal": 6,
- "value": "READ_WRITE"
- },
- {
- "ordinal": 7,
- "value": "ALL"
- }
- ]
- }
- ],
- "structDefs": [
- {
- "name": "fs_permissions",
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "group",
- "typeName": "file_action",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "user",
- "typeName": "file_action",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "sticky",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "others",
- "typeName": "file_action",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- }
- ]
- }
- ],
- "classificationDefs": [],
- "entityDefs": [
- {
- "name": "fs_path",
- "superTypes": [
- "DataSet"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "path",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": false,
- "isUnique": false
- },
- {
- "name": "createTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "modifiedTime",
- "typeName": "date",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "isFile",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "isSymlink",
- "typeName": "boolean",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "fileSize",
- "typeName": "long",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "group",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "posixPermissions",
- "typeName": "fs_permissions",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- }
- ]
- },
- {
- "name": "hdfs_path",
- "superTypes": [
- "fs_path"
- ],
- "typeVersion": "1.0",
- "attributeDefs": [
- {
- "name": "clusterName",
- "typeName": "string",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "numberOfReplicas",
- "typeName": "int",
- "cardinality": "SINGLE",
- "isIndexable": true,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "extendedAttributes",
- "typeName": "map<string,string>",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- }
- ]
- }
- ]
-}