You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/08/06 20:43:26 UTC
[1/2] atlas git commit: ATLAS-2806: Using replication attributes
during export and import process.
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 a1b6ba39c -> c3b01a6f1
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/typesdef.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/typesdef.json b/repository/src/test/resources/json/stocksDB-Entities/typesdef.json
new file mode 100644
index 0000000..fcea39c
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/typesdef.json
@@ -0,0 +1,685 @@
+{
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "params": {
+ "attribute": "sd"
+ },
+ "type": "inverseRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "table",
+ "typeName": "hive_table",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "location",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "inputFormat",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "outputFormat",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "compressed",
+ "typeName": "boolean",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "numBuckets",
+ "typeName": "int",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "serdeInfo",
+ "typeName": "hive_serde",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "bucketCols",
+ "typeName": "array<string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "sortCols",
+ "typeName": "array<hive_order>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "storedAsSubDirectories",
+ "typeName": "boolean",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "ENTITY",
+ "createTime": 1528222416164,
+ "createdBy": "atlas",
+ "description": "hive_storagedesc",
+ "guid": "25d90af0-6d83-482b-9e0e-f6d87b6ade18",
+ "name": "hive_storagedesc",
+ "subTypes": [],
+ "superTypes": [
+ "Referenceable"
+ ],
+ "typeVersion": "1.0",
+ "updateTime": 1528222416164,
+ "updatedBy": "atlas",
+ "version": 1
+ },
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "name",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "description",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "owner",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "ENTITY",
+ "createTime": 1528222408418,
+ "createdBy": "atlas",
+ "description": "Asset",
+ "guid": "bdc1e6ea-91f8-4ba3-bd7e-f9a7deea957e",
+ "name": "Asset",
+ "subTypes": [
+ "DataSet",
+ "Infrastructure",
+ "Process",
+ "hive_db"
+ ],
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "updateTime": 1528222408418,
+ "updatedBy": "atlas",
+ "version": 1
+ },
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "db",
+ "typeName": "hive_db",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "createTime",
+ "typeName": "date",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "lastAccessTime",
+ "typeName": "date",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "comment",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "retention",
+ "typeName": "int",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "sd",
+ "typeName": "hive_storagedesc",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "partitionKeys",
+ "typeName": "array<hive_column>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "aliases",
+ "typeName": "array<string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "columns",
+ "typeName": "array<hive_column>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "viewOriginalText",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "viewExpandedText",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "tableType",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "temporary",
+ "typeName": "boolean",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "ENTITY",
+ "createTime": 1528222416110,
+ "createdBy": "atlas",
+ "description": "hive_table",
+ "guid": "bc8441a9-1127-4f8f-b09e-d7b295c4895e",
+ "name": "hive_table",
+ "options": {
+ "schemaElementsAttribute": "columns"
+ },
+ "subTypes": [],
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.1",
+ "updateTime": 1528222420642,
+ "updatedBy": "atlas",
+ "version": 2
+ },
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "type",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "comment",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "position",
+ "typeName": "int",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "table",
+ "typeName": "hive_table",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "ENTITY",
+ "createTime": 1528222416225,
+ "createdBy": "atlas",
+ "description": "hive_column",
+ "guid": "45e871ae-6ef9-4e1c-9b5d-2a66a0ce9fa6",
+ "name": "hive_column",
+ "options": {
+ "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
+ },
+ "subTypes": [],
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.3",
+ "updateTime": 1528222420879,
+ "updatedBy": "atlas",
+ "version": 4
+ },
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "clusterName",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "location",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "ownerType",
+ "typeName": "hive_principal_type",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "ENTITY",
+ "createTime": 1528222416207,
+ "createdBy": "atlas",
+ "description": "hive_db",
+ "guid": "0d3278e5-3e42-46b4-bf85-8c3f40cb3225",
+ "name": "hive_db",
+ "subTypes": [],
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "updateTime": 1528222416207,
+ "updatedBy": "atlas",
+ "version": 1
+ },
+ {
+ "name": "hive_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operationType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryPlan",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryId",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "recentQueries",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "queryGraph",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "enumDefs": [
+ {
+ "category": "ENUM",
+ "createTime": 1528222414371,
+ "createdBy": "atlas",
+ "description": "hive_principal_type",
+ "elementDefs": [
+ {
+ "ordinal": 1,
+ "value": "USER"
+ },
+ {
+ "ordinal": 2,
+ "value": "ROLE"
+ },
+ {
+ "ordinal": 3,
+ "value": "GROUP"
+ }
+ ],
+ "guid": "6ad79f3a-c5b6-448b-95a6-86ddd28f32a3",
+ "name": "hive_principal_type",
+ "typeVersion": "1.0",
+ "updateTime": 1528222414371,
+ "updatedBy": "atlas",
+ "version": 1
+ }
+ ],
+ "structDefs": [
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "name",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "serializationLib",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false,
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 0
+ }
+ ],
+ "category": "STRUCT",
+ "createTime": 1528222415093,
+ "createdBy": "atlas",
+ "description": "hive_serde",
+ "guid": "c8db213b-e08d-4e1e-a373-3567f229ea89",
+ "name": "hive_serde",
+ "typeVersion": "1.0",
+ "updateTime": 1528222415093,
+ "updatedBy": "atlas",
+ "version": 1
+ },
+ {
+ "attributeDefs": [
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "order",
+ "typeName": "int",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ },
+ {
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false,
+ "name": "col",
+ "typeName": "string",
+ "valuesMaxCount": 1,
+ "valuesMinCount": 1
+ }
+ ],
+ "category": "STRUCT",
+ "createTime": 1528222415076,
+ "createdBy": "atlas",
+ "description": "hive_order",
+ "guid": "d65c1a58-1170-4e5c-a8ce-aee0ec920690",
+ "name": "hive_order",
+ "typeVersion": "1.0",
+ "updateTime": 1528222415076,
+ "updatedBy": "atlas",
+ "version": 1
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/server-api/src/main/java/org/apache/atlas/RequestContextV1.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContextV1.java b/server-api/src/main/java/org/apache/atlas/RequestContextV1.java
index 2123166..8506d18 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContextV1.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContextV1.java
@@ -38,10 +38,10 @@ public class RequestContextV1 {
private final Map<String, AtlasEntity> entityCacheV2 = new HashMap<>();
private final Metrics metrics = new Metrics();
private final long requestTime = System.currentTimeMillis();
+ private boolean shouldUpdateModificationTimestamp = true;
private String user;
-
private RequestContextV1() {
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
index e9477ff..d4e7262 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java
@@ -35,7 +35,7 @@ import org.apache.atlas.model.impexp.AtlasExportResult;
import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.metrics.AtlasMetrics;
-import org.apache.atlas.repository.clusterinfo.ClusterService;
+import org.apache.atlas.repository.impexp.ClusterService;
import org.apache.atlas.repository.impexp.ExportImportAuditService;
import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.impexp.ImportService;
[2/2] atlas git commit: ATLAS-2806: Using replication attributes
during export and import process.
Posted by am...@apache.org.
ATLAS-2806: Using replication attributes during export and import process.
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/c3b01a6f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/c3b01a6f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/c3b01a6f
Branch: refs/heads/branch-0.8
Commit: c3b01a6f1e0b8c640c865c82ed22e2e4d66ae190
Parents: a1b6ba3
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Mon Aug 6 12:02:39 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Mon Aug 6 12:02:39 2018 -0700
----------------------------------------------------------------------
.../004-base_model_replication_attributes.json | 29 +
.../org/apache/atlas/repository/Constants.java | 8 +
docs/src/site/twiki/Export-API.twiki | 4 +-
pom.xml | 2 +-
.../repository/clusterinfo/ClusterService.java | 56 --
.../atlas/repository/impexp/AuditHelper.java | 113 ---
.../atlas/repository/impexp/AuditsWriter.java | 185 +++++
.../atlas/repository/impexp/ClusterService.java | 124 ++++
.../atlas/repository/impexp/ExportService.java | 11 +-
.../atlas/repository/impexp/ImportService.java | 8 +-
.../store/graph/v1/EntityGraphRetriever.java | 1 +
.../clusterinfo/ClusterServiceTest.java | 1 +
.../repository/impexp/ExportImportTestBase.java | 5 +
.../impexp/ReplicationEntityAttributeTest.java | 205 ++++++
.../impexp/ZipFileResourceTestUtils.java | 20 +-
.../resources/json/stocksDB-Entities/db.json | 24 +
.../stocksDB-Entities/export-replicatedTo.json | 11 +
.../stocksDB-Entities/export-skip-lineage.json | 11 +
.../import-replicatedFrom.json | 5 +
.../stocksDB-Entities/replicationAttrs.json | 8 +
.../json/stocksDB-Entities/table-columns.json | 284 ++++++++
.../stocksDB-Entities/table-table-lineage.json | 44 ++
.../json/stocksDB-Entities/table-view.json | 107 +++
.../typesdef-new-classification.json | 22 +
.../json/stocksDB-Entities/typesdef.json | 685 +++++++++++++++++++
.../java/org/apache/atlas/RequestContextV1.java | 2 +-
.../atlas/web/resources/AdminResource.java | 2 +-
27 files changed, 1791 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/addons/models/patches/004-base_model_replication_attributes.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/004-base_model_replication_attributes.json b/addons/models/patches/004-base_model_replication_attributes.json
new file mode 100644
index 0000000..bee3718
--- /dev/null
+++ b/addons/models/patches/004-base_model_replication_attributes.json
@@ -0,0 +1,29 @@
+{
+ "patches": [
+ {
+ "action": "ADD_ATTRIBUTE",
+ "typeName": "Referenceable",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "replicatedFromCluster",
+ "typeName": "array<AtlasCluster>",
+ "cardinality": "SET",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "replicatedToCluster",
+ "typeName": "array<AtlasCluster>",
+ "cardinality": "SET",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/common/src/main/java/org/apache/atlas/repository/Constants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java
index 4e179fb..56f341d 100644
--- a/common/src/main/java/org/apache/atlas/repository/Constants.java
+++ b/common/src/main/java/org/apache/atlas/repository/Constants.java
@@ -99,6 +99,14 @@ public final class Constants {
public static final String MAX_FULLTEXT_QUERY_STR_LENGTH = "atlas.graph.fulltext-max-query-str-length";
public static final String MAX_DSL_QUERY_STR_LENGTH = "atlas.graph.dsl-max-query-str-length";
+ /*
+ * replication attributes
+ */
+
+ public static final String ATTR_NAME_REFERENCEABLE = "Referenceable.";
+ public static final String ATTR_NAME_REPLICATED_TO_CLUSTER = "replicatedToCluster";
+ public static final String ATTR_NAME_REPLICATED_FROM_CLUSTER = "replicatedFromCluster";
+
private Constants() {
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/docs/src/site/twiki/Export-API.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/Export-API.twiki b/docs/src/site/twiki/Export-API.twiki
index 0df6629..2074999 100644
--- a/docs/src/site/twiki/Export-API.twiki
+++ b/docs/src/site/twiki/Export-API.twiki
@@ -145,6 +145,6 @@ curl -X POST -u adminuser:password -H "Content-Type: application/json" -H "Cache
{ "typeName": "DB", "uniqueAttributes": { "name": "Logging" }
}
],
- "options": "full"
-}' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip
+ "options": { "full" }
+ }' "http://localhost:21000/api/atlas/admin/export" > quickStartDB.zip
</verbatim>
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3fe0b78..40d483b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -506,7 +506,7 @@
<slf4j.version>1.7.21</slf4j.version>
<jetty.version>8.1.19.v20160209</jetty.version>
<jetty.maven.plugin.version>9.2.12.v20150709</jetty.maven.plugin.version>
- <jersey.version>1.19</jersey.version>
+ <jersey.version>1.19.3</jersey.version>
<tinkerpop.version>2.6.0</tinkerpop.version>
<titan.version>0.5.4</titan.version>
<hadoop.version>2.7.1</hadoop.version>
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java b/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java
deleted file mode 100644
index 9c0afc0..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/clusterinfo/ClusterService.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.clusterinfo;
-
-import org.apache.atlas.annotation.AtlasService;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.clusterinfo.AtlasCluster;
-import org.apache.atlas.repository.ogm.DataAccess;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import javax.inject.Inject;
-
-@AtlasService
-public class ClusterService {
- private static final Logger LOG = LoggerFactory.getLogger(ClusterService.class);
-
- private final DataAccess dataAccess;
-
- @Inject
- public ClusterService(DataAccess dataAccess) {
- this.dataAccess = dataAccess;
- }
-
- public AtlasCluster get(AtlasCluster cluster) {
- try {
- return dataAccess.load(cluster);
- } catch (AtlasBaseException e) {
- LOG.error("dataAccess", e);
- }
-
- return null;
- }
-
- @GraphTransaction
- public AtlasCluster save(AtlasCluster clusterInfo) throws AtlasBaseException {
- return dataAccess.save(clusterInfo);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java
deleted file mode 100644
index 3137537..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditHelper.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.impexp;
-
-import org.apache.atlas.ApplicationProperties;
-import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.clusterinfo.AtlasCluster;
-import org.apache.atlas.model.impexp.AtlasExportRequest;
-import org.apache.atlas.model.impexp.AtlasExportResult;
-import org.apache.atlas.model.impexp.AtlasImportRequest;
-import org.apache.atlas.model.impexp.AtlasImportResult;
-import org.apache.atlas.model.impexp.ExportImportAuditEntry;
-import org.apache.atlas.repository.clusterinfo.ClusterService;
-import org.apache.atlas.type.AtlasType;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import java.util.Map;
-
-@Component
-public class AuditHelper {
- private static final Logger LOG = LoggerFactory.getLogger(AuditHelper.class);
- private static final String CLUSTER_NAME_DEFAULT = "default";
-
- private ClusterService clusterService;
- private ExportImportAuditService auditService;
-
- @Inject
- public AuditHelper(ClusterService clusterService, ExportImportAuditService auditService) {
- this.clusterService = clusterService;
- this.auditService = auditService;
- }
-
-
- public void audit(String userName, AtlasExportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException {
- AtlasExportRequest request = result.getRequest();
- AtlasCluster cluster = saveCluster(getCurrentClusterName());
- String targetClusterName = getClusterNameFromOptions(request.getOptions(), AtlasExportRequest.OPTION_KEY_REPLICATED_TO);
- addAuditEntry(userName,
- cluster.getName(), targetClusterName,
- ExportImportAuditEntry.OPERATION_EXPORT,
- AtlasType.toJson(result), startTime, endTime, hadData);
- }
-
- public void audit(String userName, AtlasImportResult result, long startTime, long endTime, boolean hadData) throws AtlasBaseException {
- AtlasImportRequest request = result.getRequest();
- AtlasCluster cluster = saveCluster(getCurrentClusterName());
- String sourceCluster = getClusterNameFromOptions(request.getOptions(), AtlasImportRequest.OPTION_KEY_REPLICATED_FROM);
- addAuditEntry(userName,
- sourceCluster, cluster.getName(),
- ExportImportAuditEntry.OPERATION_EXPORT, AtlasType.toJson(result), startTime, endTime, hadData);
- }
-
- private String getClusterNameFromOptions(Map options, String key) {
- return options.containsKey(key)
- ? (String) options.get(key)
- : "";
- }
-
- private void addAuditEntry(String userName, String sourceCluster, String targetCluster, String operation,
- String result, long startTime, long endTime, boolean hasData) throws AtlasBaseException {
- if(!hasData) return;
-
- ExportImportAuditEntry entry = new ExportImportAuditEntry();
-
- entry.setUserName(userName);
- entry.setSourceClusterName(sourceCluster);
- entry.setTargetClusterName(targetCluster);
- entry.setOperation(operation);
- entry.setResultSummary(result);
- entry.setStartTime(startTime);
- entry.setEndTime(endTime);
-
- auditService.save(entry);
- LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(),
- entry.getSourceClusterName(), entry.getTargetClusterName(), entry.getOperation());
- }
-
- private AtlasCluster saveCluster(String clusterName) throws AtlasBaseException {
- AtlasCluster cluster = new AtlasCluster(clusterName, clusterName);
- return clusterService.save(cluster);
- }
-
- public static String getCurrentClusterName() {
- try {
- return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, CLUSTER_NAME_DEFAULT);
- } catch (AtlasException e) {
- LOG.error("getCurrentClusterName", e);
- }
-
- return "";
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
new file mode 100644
index 0000000..6a3fbec
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/AuditsWriter.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasConstants;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.clusterinfo.AtlasCluster;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.impexp.AtlasExportResult;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.impexp.AtlasImportResult;
+import org.apache.atlas.model.impexp.ExportImportAuditEntry;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasType;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+import javax.inject.Inject;
+import java.util.List;
+import java.util.Map;
+
+@Component
+public class AuditsWriter {
+ private static final Logger LOG = LoggerFactory.getLogger(AuditsWriter.class);
+ private static final String CLUSTER_NAME_DEFAULT = "default";
+
+ private ClusterService clusterService;
+ private ExportImportAuditService auditService;
+
+ private ExportAudits auditForExport = new ExportAudits();
+ private ImportAudits auditForImport = new ImportAudits();
+
+ @Inject
+ public AuditsWriter(ClusterService clusterService, ExportImportAuditService auditService) {
+ this.clusterService = clusterService;
+ this.auditService = auditService;
+ }
+
+ public void write(String userName, AtlasExportResult result, long startTime, long endTime, List<String> entityCreationOrder) throws AtlasBaseException {
+ auditForExport.add(userName, result, startTime, endTime, entityCreationOrder);
+ }
+
+ public void write(String userName, AtlasImportResult result, long startTime, long endTime, List<String> entityCreationOrder) throws AtlasBaseException {
+ auditForImport.add(userName, result, startTime, endTime, entityCreationOrder);
+ }
+
+ private boolean isReplicationOptionSet(Map<String, ? extends Object> options, String replicatedKey) {
+ return options.containsKey(replicatedKey);
+ }
+
+ private void updateReplicationAttribute(boolean isReplicationSet, String clusterName,
+ List<String> exportedGuids,
+ String attrNameReplicated) throws AtlasBaseException {
+ if (!isReplicationSet) {
+ return;
+ }
+
+ AtlasCluster cluster = saveCluster(clusterName);
+ clusterService.updateEntityWithCluster(cluster, exportedGuids, attrNameReplicated);
+ }
+
+ private String getClusterNameFromOptions(Map options, String key) {
+ return options.containsKey(key)
+ ? (String) options.get(key)
+ : "";
+ }
+
+ private void addAuditEntry(String userName, String sourceCluster, String targetCluster, String operation,
+ String result, long startTime, long endTime, boolean hasData) throws AtlasBaseException {
+ if(!hasData) return;
+
+ ExportImportAuditEntry entry = new ExportImportAuditEntry();
+
+ entry.setUserName(userName);
+ entry.setSourceClusterName(sourceCluster);
+ entry.setTargetClusterName(targetCluster);
+ entry.setOperation(operation);
+ entry.setResultSummary(result);
+ entry.setStartTime(startTime);
+ entry.setEndTime(endTime);
+
+ auditService.save(entry);
+ LOG.info("addAuditEntry: user: {}, source: {}, target: {}, operation: {}", entry.getUserName(),
+ entry.getSourceClusterName(), entry.getTargetClusterName(), entry.getOperation());
+ }
+
+ private AtlasCluster saveCluster(String clusterName) throws AtlasBaseException {
+ AtlasCluster cluster = new AtlasCluster(clusterName, clusterName);
+ return clusterService.save(cluster);
+ }
+
+ public static String getCurrentClusterName() {
+ try {
+ return ApplicationProperties.get().getString(AtlasConstants.CLUSTER_NAME_KEY, CLUSTER_NAME_DEFAULT);
+ } catch (AtlasException e) {
+ LOG.error("getCurrentClusterName", e);
+ }
+
+ return "";
+ }
+
+ private class ExportAudits {
+ private AtlasExportRequest request;
+ private AtlasCluster cluster;
+ private String targetClusterName;
+ private String optionKeyReplicatedTo;
+ private boolean replicationOptionState;
+
+ public void add(String userName, AtlasExportResult result, long startTime, long endTime, List<String> entitityGuids) throws AtlasBaseException {
+ optionKeyReplicatedTo = AtlasExportRequest.OPTION_KEY_REPLICATED_TO;
+ request = result.getRequest();
+ cluster = saveCluster(getCurrentClusterName());
+ replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedTo);
+ targetClusterName = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedTo);
+
+ addAuditEntry(userName,
+ cluster.getName(), targetClusterName,
+ ExportImportAuditEntry.OPERATION_EXPORT,
+ AtlasType.toJson(result), startTime, endTime, !entitityGuids.isEmpty());
+
+ updateReplicationAttributeForExport(entitityGuids, request);
+ }
+
+ private void updateReplicationAttributeForExport(List<String> entityGuids, AtlasExportRequest request) throws AtlasBaseException {
+ if(!replicationOptionState) return;
+
+ updateReplicationAttribute(replicationOptionState, targetClusterName, entityGuids, Constants.ATTR_NAME_REPLICATED_TO_CLUSTER);
+ }
+ }
+
+ private class ImportAudits {
+ private AtlasImportRequest request;
+ private boolean replicationOptionState;
+ private AtlasCluster cluster;
+ private String optionKeyReplicatedFrom;
+ private AtlasImportResult result;
+
+ public void add(String userName, AtlasImportResult result, long startTime, long endTime, List<String> entitityGuids) throws AtlasBaseException {
+ this.result = result;
+ request = result.getRequest();
+ optionKeyReplicatedFrom = AtlasImportRequest.OPTION_KEY_REPLICATED_FROM;
+ replicationOptionState = isReplicationOptionSet(request.getOptions(), optionKeyReplicatedFrom);
+ cluster = saveCluster(getClusterNameFromOptionsState());
+
+ String sourceCluster = getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom);
+ addAuditEntry(userName,
+ sourceCluster, cluster.getName(),
+ ExportImportAuditEntry.OPERATION_EXPORT, AtlasType.toJson(result), startTime, endTime, !entitityGuids.isEmpty());
+
+ updateReplicationAttributeForImport(entitityGuids);
+ }
+
+ private void updateReplicationAttributeForImport(List<String> entityGuids) throws AtlasBaseException {
+ if(!replicationOptionState) return;
+
+ String targetClusterName = cluster.getName();
+ updateReplicationAttribute(replicationOptionState, targetClusterName, entityGuids, Constants.ATTR_NAME_REPLICATED_FROM_CLUSTER);
+ }
+
+ private String getClusterNameFromOptionsState() {
+ return replicationOptionState
+ ? getClusterNameFromOptions(request.getOptions(), optionKeyReplicatedFrom)
+ : getCurrentClusterName();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java
new file mode 100644
index 0000000..fd8e2bf
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ClusterService.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.annotation.AtlasService;
+import org.apache.atlas.annotation.GraphTransaction;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.clusterinfo.AtlasCluster;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.repository.ogm.DataAccess;
+import org.apache.atlas.repository.store.graph.AtlasEntityStore;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityStream;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphRetriever;
+import org.apache.atlas.type.AtlasType;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.List;
+
+@AtlasService
+public class ClusterService {
+ private static final Logger LOG = LoggerFactory.getLogger(ClusterService.class);
+
+ private final DataAccess dataAccess;
+ private final AtlasEntityStore entityStore;
+
+ @Inject
+ public ClusterService(DataAccess dataAccess, AtlasEntityStore entityStore) {
+ this.dataAccess = dataAccess;
+ this.entityStore = entityStore;
+ }
+
+ public AtlasCluster get(AtlasCluster cluster) {
+ try {
+ return dataAccess.load(cluster);
+ } catch (AtlasBaseException e) {
+ LOG.error("dataAccess", e);
+ }
+
+ return null;
+ }
+
+ @GraphTransaction
+ public AtlasCluster save(AtlasCluster clusterInfo) throws AtlasBaseException {
+ return dataAccess.save(clusterInfo);
+ }
+
+ @GraphTransaction
+ public void updateEntityWithCluster(AtlasCluster cluster, List<String> guids, String attributeName) throws AtlasBaseException {
+ if(cluster != null && StringUtils.isEmpty(cluster.getGuid())) return;
+
+ AtlasObjectId objectId = getObjectId(cluster);
+ for (String guid : guids) {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(guid);
+ updateAttribute(entityWithExtInfo, attributeName, objectId);
+ entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true);
+ }
+ }
+
+ private AtlasObjectId getObjectId(AtlasCluster cluster) {
+ return new AtlasObjectId(cluster.getGuid(), AtlasCluster.class.getSimpleName());
+ }
+
+
+ /**
+ * Attribute passed by name is updated with the value passed.
+ * @param entityWithExtInfo Entity to be updated
+ * @param propertyName attribute name
+ * @param value Value to be set for attribute
+ */
+ private void updateAttribute(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo, String propertyName, Object value) {
+ updateAttribute(entityWithExtInfo.getEntity(), propertyName, value);
+ for (AtlasEntity e : entityWithExtInfo.getReferredEntities().values()) {
+ updateAttribute(e, propertyName, value);
+ }
+ }
+
+ private void updateAttribute(AtlasEntity e, String propertyName, Object value) {
+ if(e.hasAttribute(propertyName) == false) return;
+
+ Object oVal = e.getAttribute(propertyName);
+ if (oVal != null && !(oVal instanceof List)) return;
+
+ List list;
+
+ if (oVal == null) {
+ list = new ArrayList();
+ } else {
+ list = (List) oVal;
+ }
+
+ if (!list.contains(value)) {
+ list.add(value);
+ }
+
+ e.setAttribute(propertyName, list);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
index 069f5c0..02d17fb 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ExportService.java
@@ -69,18 +69,18 @@ public class ExportService {
private static final Logger LOG = LoggerFactory.getLogger(ExportService.class);
private final AtlasTypeRegistry typeRegistry;
- private AuditHelper auditHelper;
+ private AuditsWriter auditsWriter;
private final AtlasGraph atlasGraph;
private final EntityGraphRetriever entityGraphRetriever;
private final AtlasGremlinQueryProvider gremlinQueryProvider;
@Inject
- public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AuditHelper auditHelper) {
+ public ExportService(final AtlasTypeRegistry typeRegistry, AtlasGraph atlasGraph, AuditsWriter auditsWriter) {
this.typeRegistry = typeRegistry;
this.entityGraphRetriever = new EntityGraphRetriever(this.typeRegistry);
this.atlasGraph = atlasGraph;
this.gremlinQueryProvider = AtlasGremlinQueryProvider.INSTANCE;
- this.auditHelper = auditHelper;
+ this.auditsWriter = auditsWriter;
}
public AtlasExportResult run(ZipSink exportSink, AtlasExportRequest request, String userName, String hostName,
@@ -113,12 +113,11 @@ public class ExportService {
AtlasExportResult.OperationStatus[] statuses,
long startTime, long endTime) throws AtlasBaseException {
int duration = getOperationDuration(startTime, endTime);
- context.result.setSourceClusterName(AuditHelper.getCurrentClusterName());
+ context.result.setSourceClusterName(AuditsWriter.getCurrentClusterName());
context.result.getData().getEntityCreationOrder().addAll(context.lineageProcessed);
context.sink.setExportOrder(context.result.getData().getEntityCreationOrder());
context.sink.setTypesDef(context.result.getData().getTypesDef());
- auditHelper.audit(userName, context.result, startTime, endTime,
- !context.result.getData().getEntityCreationOrder().isEmpty());
+ auditsWriter.write(userName, context.result, startTime, endTime, context.result.getData().getEntityCreationOrder());
clearContextData(context);
context.result.setOperationStatus(getOverallOperationStatus(statuses));
context.result.incrementMeticsCounter("duration", duration);
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
index 035b669..98ef389 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/ImportService.java
@@ -47,17 +47,17 @@ public class ImportService {
private final AtlasTypeDefStore typeDefStore;
private final AtlasTypeRegistry typeRegistry;
private final BulkImporter bulkImporter;
- private AuditHelper auditHelper;
+ private AuditsWriter auditsWriter;
private long startTimestamp;
private long endTimestamp;
@Inject
- public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter, AuditHelper auditHelper) {
+ public ImportService(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, BulkImporter bulkImporter, AuditsWriter auditsWriter) {
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.bulkImporter = bulkImporter;
- this.auditHelper = auditHelper;
+ this.auditsWriter = auditsWriter;
}
public AtlasImportResult run(ZipSource source, String userName,
@@ -189,7 +189,7 @@ public class ImportService {
endTimestamp = System.currentTimeMillis();
result.incrementMeticsCounter("duration", getDuration(this.endTimestamp, this.startTimestamp));
- auditHelper.audit(userName, result, startTimestamp, endTimestamp, !importSource.getCreationOrder().isEmpty());
+ auditsWriter.write(userName, result, startTimestamp, endTimestamp, importSource.getCreationOrder());
}
private int getDuration(long endTime, long startTime) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 6bb4554..b972946 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.annotation.GraphTransaction;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java
index 65962cc..dba640f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/clusterinfo/ClusterServiceTest.java
@@ -21,6 +21,7 @@ package org.apache.atlas.repository.clusterinfo;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.clusterinfo.AtlasCluster;
+import org.apache.atlas.repository.impexp.ClusterService;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
index bd4d4c9..41c8486 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ExportImportTestBase.java
@@ -24,13 +24,18 @@ import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasException;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.repository.store.graph.v1.DeleteHandlerV1;
+import org.apache.atlas.repository.store.graph.v1.SoftDeleteHandlerV1;
+import static org.mockito.Mockito.mock;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertTrue;
import static org.testng.Assert.fail;
public class ExportImportTestBase {
+ protected DeleteHandlerV1 deleteHandler = mock(SoftDeleteHandlerV1.class);
+
protected void assertAuditEntry(ExportImportAuditService auditService) {
AtlasSearchResult result = null;
try {
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
new file mode 100644
index 0000000..540950a
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ReplicationEntityAttributeTest.java
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.impexp;
+
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.RequestContextV1;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.clusterinfo.AtlasCluster;
+import org.apache.atlas.model.impexp.AtlasExportRequest;
+import org.apache.atlas.model.impexp.AtlasImportRequest;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityChangeNotifier;
+import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
+import org.apache.atlas.repository.store.graph.v1.EntityGraphMapper;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.testng.SkipException;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.io.IOException;
+import java.util.List;
+
+import static org.apache.atlas.model.impexp.AtlasExportRequest.OPTION_KEY_REPLICATED_TO;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createAtlasEntity;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadBaseModel;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadEntity;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadHiveModel;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runExportWithParameters;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.runImportWithParameters;
+import static org.mockito.Mockito.mock;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ReplicationEntityAttributeTest extends ExportImportTestBase {
+ private final String ENTITIES_SUB_DIR = "stocksDB-Entities";
+ private final String EXPORT_REQUEST_FILE = "export-replicatedTo";
+ private final String IMPORT_REQUEST_FILE = "import-replicatedFrom";
+
+ private final String DB_GUID = "1637a33e-6512-447b-ade7-249c8cb5344b";
+ private final String TABLE_GUID = "df122fc3-5555-40f8-a30f-3090b8a622f8";
+
+ private String REPLICATED_TO_CLUSTER_NAME = "";
+ private String REPLICATED_FROM_CLUSTER_NAME = "";
+
+ @Inject
+ AtlasTypeRegistry typeRegistry;
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private EntityGraphMapper graphMapper;
+
+ @Inject
+ ExportService exportService;
+
+ @Inject
+ ImportService importService;
+
+ @Inject
+ ClusterService clusterService;
+
+ private AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
+ private AtlasEntityStoreV1 entityStore;
+ private ZipSource zipSource;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadBaseModel(typeDefStore, typeRegistry);
+ loadHiveModel(typeDefStore, typeRegistry);
+ createEntities();
+ }
+
+ private void createEntities() {
+ entityStore = new AtlasEntityStoreV1(deleteHandler, typeRegistry, mockChangeNotifier, graphMapper);
+
+ createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR,"db"));
+ createAtlasEntity(entityStore, loadEntity(ENTITIES_SUB_DIR, "table-columns"));
+
+ try {
+ AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID));
+ assertEquals(entities.getEntities().size(), 2);
+ } catch (AtlasBaseException e) {
+ throw new SkipException(String.format("getByIds: could not load '%s' & '%s'.", DB_GUID, TABLE_GUID));
+ }
+ }
+
+ @BeforeMethod
+ public void setupTest() {
+ RequestContextV1.clear();
+ RequestContextV1.get().setUser(TestUtilsV2.TEST_USER);
+ }
+
+ @Test
+ public void exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException {
+ final int expectedEntityCount = 2;
+
+ AtlasExportRequest request = getUpdateMetaInfoUpdateRequest();
+ zipSource = runExportWithParameters(exportService, request);
+
+ assertNotNull(zipSource);
+ assertNotNull(zipSource.getCreationOrder());
+ assertEquals(zipSource.getCreationOrder().size(), expectedEntityCount);
+
+ assertClusterInfo(REPLICATED_TO_CLUSTER_NAME);
+ assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_TO_CLUSTER);
+ }
+
+ @Test(dependsOnMethods = "exportWithReplicationToOption_AddsClusterObjectIdToReplicatedFromAttribute")
+ public void importWithReplicationFromOption_AddsClusterObjectIdToReplicatedFromAttribute() throws AtlasBaseException, IOException {
+ AtlasImportRequest request = getImportRequestWithReplicationOption();
+ runImportWithParameters(importService, request, zipSource);
+
+ assertClusterInfo(REPLICATED_FROM_CLUSTER_NAME);
+ assertReplicationAttribute(Constants.ATTR_NAME_REPLICATED_FROM_CLUSTER);
+ }
+
+ private void assertReplicationAttribute(String attrNameReplication) throws AtlasBaseException {
+ AtlasEntity.AtlasEntitiesWithExtInfo entities = entityStore.getByIds(ImmutableList.of(DB_GUID, TABLE_GUID));
+ for (AtlasEntity e : entities.getEntities()) {
+ Object ex = e.getAttribute(attrNameReplication);
+ assertNotNull(ex);
+
+ List<String> clusterNameSyncType = (List) ex;
+ assertEquals(clusterNameSyncType.size(), 1);
+ }
+ }
+
+ private void assertClusterInfo(String name) {
+ AtlasCluster actual = clusterService.get(new AtlasCluster(name, name));
+
+ assertNotNull(actual);
+ assertEquals(actual.getName(), name);
+ }
+
+ private AtlasExportRequest getUpdateMetaInfoUpdateRequest() {
+ AtlasExportRequest request = getExportRequestWithReplicationOption();
+ request.getOptions().put(AtlasExportRequest.OPTION_KEY_REPLICATED_TO, REPLICATED_TO_CLUSTER_NAME);
+
+ return request;
+ }
+
+ private AtlasEntity.AtlasEntityWithExtInfo getEntities(ZipSource source, int expectedCount) {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = new AtlasEntity.AtlasEntityWithExtInfo();
+ try {
+ int count = 0;
+ for(String s : source.getCreationOrder()) {
+ AtlasEntity entity = source.getByGuid(s);
+ entityWithExtInfo.addReferredEntity(s, entity);
+ count++;
+ }
+
+ assertEquals(count, expectedCount);
+ return entityWithExtInfo;
+ } catch (AtlasBaseException e) {
+ throw new SkipException("getEntities: failed!");
+ }
+ }
+
+ private AtlasExportRequest getExportRequestWithReplicationOption() {
+ try {
+ AtlasExportRequest request = TestResourceFileUtils.readObjectFromJson(ENTITIES_SUB_DIR, EXPORT_REQUEST_FILE, AtlasExportRequest.class);
+ REPLICATED_TO_CLUSTER_NAME = (String) request.getOptions().get(OPTION_KEY_REPLICATED_TO);
+ return request;
+ } catch (IOException e) {
+ throw new SkipException(String.format("getExportRequestWithReplicationOption: '%s' could not be laoded.", EXPORT_REQUEST_FILE));
+ }
+ }
+
+ private AtlasImportRequest getImportRequestWithReplicationOption() {
+ try {
+ AtlasImportRequest request = TestResourceFileUtils.readObjectFromJson(ENTITIES_SUB_DIR, IMPORT_REQUEST_FILE, AtlasImportRequest.class);
+ REPLICATED_FROM_CLUSTER_NAME = request.getOptions().get(AtlasImportRequest.OPTION_KEY_REPLICATED_FROM);
+ return request;
+ } catch (IOException e) {
+ throw new SkipException(String.format("getExportRequestWithReplicationOption: '%s' could not be laoded.", IMPORT_REQUEST_FILE));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index 6f01822..014fe75 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -27,16 +27,19 @@ import org.apache.atlas.model.impexp.AtlasImportRequest;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStoreV1;
import org.apache.atlas.repository.store.graph.v1.AtlasEntityStreamForImport;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestResourceFileUtils;
import org.apache.commons.io.FileUtils;
-import org.apache.solr.common.StringUtils;
+import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.SkipException;
@@ -65,7 +68,8 @@ public class ZipFileResourceTestUtils {
public static String getModelJson(String fileName) throws IOException {
final String userDir = System.getProperty("user.dir");
- String filePath = userDir + "/../addons/models/" + fileName;
+ String modelsDir = "/../addons/models/";
+ String filePath = userDir + modelsDir + fileName;
File f = new File(filePath);
String s = FileUtils.readFileToString(f);
assertFalse(StringUtils.isEmpty(s), "Model file read correctly!");
@@ -190,9 +194,21 @@ public class ZipFileResourceTestUtils {
public static void loadModelFromJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
AtlasTypesDef typesFromJson = getAtlasTypesDefFromFile(fileName);
+ addReplicationAttributes(typesFromJson);
createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
}
+ private static void addReplicationAttributes(AtlasTypesDef typesFromJson) throws IOException {
+ AtlasEntityDef ed = typesFromJson.getEntityDefs().get(0);
+ if(!ed.getName().equals("Referenceable")) return;
+
+ String replAttr1Json = TestResourceFileUtils.getJson("stocksDB-Entities","replicationAttrs");
+ String replAttr2Json = StringUtils.replace(replAttr1Json, "From", "To");
+
+ ed.addAttribute(AtlasType.fromJson(replAttr1Json, AtlasStructDef.AtlasAttributeDef.class));
+ ed.addAttribute(AtlasType.fromJson(replAttr2Json, AtlasStructDef.AtlasAttributeDef.class));
+ }
+
public static void loadModelFromResourcesJson(String fileName, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws IOException, AtlasBaseException {
AtlasTypesDef typesFromJson = getAtlasTypesDefFromResourceFile(fileName);
createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/db.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/db.json b/repository/src/test/resources/json/stocksDB-Entities/db.json
new file mode 100644
index 0000000..db581d5
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/db.json
@@ -0,0 +1,24 @@
+{
+ "entity": {
+ "attributes": {
+ "clusterName": "cl1",
+ "description": null,
+ "location": "hdfs://localhost.localdomain:8020/apps/hive/warehouse/stocks_base.db",
+ "name": "stocks_base",
+ "owner": "anonymous",
+ "ownerType": "USER",
+ "parameters": null,
+ "qualifiedName": "stocks_base@cl1"
+ },
+ "classifications": [],
+ "createTime": 1528238690000,
+ "createdBy": "anonymous",
+ "guid": "1637a33e-6512-447b-ade7-249c8cb5344b",
+ "status": "ACTIVE",
+ "typeName": "hive_db",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "referredEntities": {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
new file mode 100644
index 0000000..a69fe9e
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/export-replicatedTo.json
@@ -0,0 +1,11 @@
+{
+ "itemsToExport": [
+ {
+ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks_base@cl1" }
+ }
+ ],
+ "options": {
+ "fetchType": "full",
+ "replicatedTo": "clTarget"
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json b/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json
new file mode 100644
index 0000000..f14eaca
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/export-skip-lineage.json
@@ -0,0 +1,11 @@
+{
+ "itemsToExport": [
+ {
+ "typeName": "hive_db", "uniqueAttributes": { "qualifiedName": "stocks_base@cl1" }
+ }
+ ],
+ "options": {
+ "fetchType": "incremental",
+ "skipLineage": true
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
new file mode 100644
index 0000000..1ce00ad
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/import-replicatedFrom.json
@@ -0,0 +1,5 @@
+{
+ "options": {
+ "replicatedFrom": "clSource"
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json
new file mode 100644
index 0000000..8282638
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/replicationAttrs.json
@@ -0,0 +1,8 @@
+{
+ "name": "replicatedFromCluster",
+ "typeName": "array<AtlasCluster>",
+ "cardinality": "SET",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-columns.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-columns.json b/repository/src/test/resources/json/stocksDB-Entities/table-columns.json
new file mode 100644
index 0000000..1536897
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/table-columns.json
@@ -0,0 +1,284 @@
+{
+ "entity": {
+ "attributes": {
+ "aliases": null,
+ "columns": [
+ {
+ "guid": "08bc6cee-fe25-47d2-9e39-155d4b807765",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "b3a13f06-4585-4fef-a5ae-970c24bd3981",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "f87a5320-1529-4369-8d63-b637ebdf2c1c",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "47d732d5-a6ef-41b8-83bd-a54365750a47",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "92a6a9b8-dbab-4313-a6f6-0035246b0e37",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "7193839e-10d3-4926-83d8-2c675a1068bd",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "c6e27ba6-9f4d-4e07-9a54-0fc631529f51",
+ "typeName": "hive_column"
+ }
+ ],
+ "comment": null,
+ "createTime": 1528238679000,
+ "db": {
+ "guid": "1637a33e-6512-447b-ade7-249c8cb5344b",
+ "typeName": "hive_db"
+ },
+ "description": null,
+ "lastAccessTime": 1528238679000,
+ "name": "stocks_daily",
+ "owner": "anonymous",
+ "parameters": {
+ "COLUMN_STATS_ACCURATE": "{\"BASIC_STATS\":\"true\"}",
+ "numFiles": "0",
+ "numRows": "0",
+ "rawDataSize": "0",
+ "totalSize": "0",
+ "transient_lastDdlTime": "1528238679"
+ },
+ "partitionKeys": null,
+ "qualifiedName": "stocks_base.stocks_daily@cl1",
+ "retention": 0,
+ "sd": {
+ "guid": "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe",
+ "typeName": "hive_storagedesc"
+ },
+ "tableType": "MANAGED_TABLE",
+ "temporary": false,
+ "viewExpandedText": null,
+ "viewOriginalText": null
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "status": "ACTIVE",
+ "typeName": "hive_table",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "referredEntities": {
+ "08bc6cee-fe25-47d2-9e39-155d4b807765": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "dt",
+ "owner": "anonymous",
+ "position": 0,
+ "qualifiedName": "stocks_base.stocks_daily.dt@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "08bc6cee-fe25-47d2-9e39-155d4b807765",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe": {
+ "attributes": {
+ "bucketCols": null,
+ "compressed": false,
+ "inputFormat": "org.apache.hadoop.mapred.TextInputFormat",
+ "location": "hdfs://localhost.localdomain:8020/apps/hive/warehouse/stocks_base.db/stocks_daily",
+ "numBuckets": -1,
+ "outputFormat": "org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat",
+ "parameters": null,
+ "qualifiedName": "stocks_base.stocks_daily@cl1_storage",
+ "serdeInfo": {
+ "attributes": {
+ "name": null,
+ "parameters": {
+ "field.delim": ",",
+ "line.delim": "\n",
+ "serialization.format": ","
+ },
+ "serializationLib": "org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"
+ },
+ "typeName": "hive_serde"
+ },
+ "sortCols": null,
+ "storedAsSubDirectories": false,
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ }
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "1ab9f789-b56c-4b01-a6e4-523f0fe5cdbe",
+ "status": "ACTIVE",
+ "typeName": "hive_storagedesc",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "47d732d5-a6ef-41b8-83bd-a54365750a47": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "low",
+ "owner": "anonymous",
+ "position": 3,
+ "qualifiedName": "stocks_base.stocks_daily.low@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "47d732d5-a6ef-41b8-83bd-a54365750a47",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "7193839e-10d3-4926-83d8-2c675a1068bd": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "volume",
+ "owner": "anonymous",
+ "position": 5,
+ "qualifiedName": "stocks_base.stocks_daily.volume@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "7193839e-10d3-4926-83d8-2c675a1068bd",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "92a6a9b8-dbab-4313-a6f6-0035246b0e37": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "close",
+ "owner": "anonymous",
+ "position": 4,
+ "qualifiedName": "stocks_base.stocks_daily.close@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "92a6a9b8-dbab-4313-a6f6-0035246b0e37",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "b3a13f06-4585-4fef-a5ae-970c24bd3981": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "open",
+ "owner": "anonymous",
+ "position": 1,
+ "qualifiedName": "stocks_base.stocks_daily.open@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "b3a13f06-4585-4fef-a5ae-970c24bd3981",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "c6e27ba6-9f4d-4e07-9a54-0fc631529f51": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "adj_close",
+ "owner": "anonymous",
+ "position": 6,
+ "qualifiedName": "stocks_base.stocks_daily.adj_close@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "c6e27ba6-9f4d-4e07-9a54-0fc631529f51",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "f87a5320-1529-4369-8d63-b637ebdf2c1c": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "high",
+ "owner": "anonymous",
+ "position": 2,
+ "qualifiedName": "stocks_base.stocks_daily.high@cl1",
+ "table": {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1528238679952,
+ "createdBy": "anonymous",
+ "guid": "f87a5320-1529-4369-8d63-b637ebdf2c1c",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1528238690000,
+ "updatedBy": "anonymous",
+ "version": 0
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json b/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json
new file mode 100644
index 0000000..322db8d
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/table-table-lineage.json
@@ -0,0 +1,44 @@
+{
+ "entity": {
+ "attributes": {
+ "clusterName": "cl1",
+ "description": null,
+ "endTime": 1529605199906,
+ "inputs": [
+ {
+ "guid": "df122fc3-5555-40f8-a30f-3090b8a622f8",
+ "typeName": "hive_table"
+ }
+ ],
+ "name": "create view stocks_view as select * from stocks_daily",
+ "operationType": "CREATEVIEW",
+ "outputs": [
+ {
+ "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e",
+ "typeName": "hive_table"
+ }
+ ],
+ "owner": null,
+ "qualifiedName": "stocks.stocks_view@cl1:1529605199000",
+ "queryGraph": null,
+ "queryId": "hive_20180621111959_72f4bec1-585f-4291-bc11-7a184a62a19b",
+ "queryPlan": "Not Supported",
+ "queryText": "create view stocks_view as select * from stocks_daily",
+ "recentQueries": [
+ "create view stocks_view as select * from stocks_daily"
+ ],
+ "startTime": 1529605199002,
+ "userName": "anonymous"
+ },
+ "classifications": [],
+ "createTime": 1529605212691,
+ "createdBy": "anonymous",
+ "guid": "6f3b305a-c459-4ae4-b651-aee0deb0685f",
+ "status": "ACTIVE",
+ "typeName": "hive_process",
+ "updateTime": 1529605221342,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "referredEntities": {}
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/table-view.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/table-view.json b/repository/src/test/resources/json/stocksDB-Entities/table-view.json
new file mode 100644
index 0000000..3efa2ec
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/table-view.json
@@ -0,0 +1,107 @@
+{
+ "entity": {
+ "attributes": {
+ "aliases": null,
+ "columns": [
+ {
+ "guid": "2d3f0239-03cc-41b2-b0cb-c924ff7d096d",
+ "typeName": "hive_column"
+ }
+ ],
+ "comment": null,
+ "createTime": 1529605199000,
+ "db": {
+ "guid": "1637a33e-6512-447b-ade7-249c8cb5344b",
+ "typeName": "hive_db"
+ },
+ "description": null,
+ "lastAccessTime": 1529605199000,
+ "name": "stocks_view",
+ "owner": "anonymous",
+ "parameters": {
+ "transient_lastDdlTime": "1529605199"
+ },
+ "partitionKeys": null,
+ "qualifiedName": "stocks.stocks_view@cl1",
+ "retention": 0,
+ "sd": {
+ "guid": "56415119-7cb0-40dd-ace8-1e50efd54991",
+ "typeName": "hive_storagedesc"
+ },
+ "tableType": "VIRTUAL_VIEW",
+ "temporary": false,
+ "viewExpandedText": "select `stocks_daily`.`dt`, `stocks_daily`.`open`, `stocks_daily`.`high`, `stocks_daily`.`low`, `stocks_daily`.`close`, `stocks_daily`.`volume`, `stocks_daily`.`adj_close` from `stocks`.`stocks_daily`",
+ "viewOriginalText": "select * from stocks_daily"
+ },
+ "classifications": [],
+ "createTime": 1529605205448,
+ "createdBy": "anonymous",
+ "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e",
+ "status": "ACTIVE",
+ "typeName": "hive_table",
+ "updateTime": 1529605221342,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "referredEntities": {
+ "2d3f0239-03cc-41b2-b0cb-c924ff7d096d": {
+ "attributes": {
+ "comment": null,
+ "description": null,
+ "name": "dt",
+ "owner": "anonymous",
+ "position": 0,
+ "qualifiedName": "stocks.stocks_view.dt@cl1",
+ "table": {
+ "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e",
+ "typeName": "hive_table"
+ },
+ "type": "string"
+ },
+ "classifications": [],
+ "createTime": 1529605205448,
+ "createdBy": "anonymous",
+ "guid": "2d3f0239-03cc-41b2-b0cb-c924ff7d096d",
+ "status": "ACTIVE",
+ "typeName": "hive_column",
+ "updateTime": 1529605221342,
+ "updatedBy": "anonymous",
+ "version": 0
+ },
+ "56415119-7cb0-40dd-ace8-1e50efd54991": {
+ "attributes": {
+ "bucketCols": null,
+ "compressed": false,
+ "inputFormat": "org.apache.hadoop.mapred.SequenceFileInputFormat",
+ "location": null,
+ "numBuckets": -1,
+ "outputFormat": "org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat",
+ "parameters": null,
+ "qualifiedName": "stocks.stocks_view@cl1_storage",
+ "serdeInfo": {
+ "attributes": {
+ "name": null,
+ "parameters": null,
+ "serializationLib": null
+ },
+ "typeName": "hive_serde"
+ },
+ "sortCols": null,
+ "storedAsSubDirectories": false,
+ "table": {
+ "guid": "c54400d8-79b7-45ba-9c01-b662b36ceb0e",
+ "typeName": "hive_table"
+ }
+ },
+ "classifications": [],
+ "createTime": 1529605205448,
+ "createdBy": "anonymous",
+ "guid": "56415119-7cb0-40dd-ace8-1e50efd54991",
+ "status": "ACTIVE",
+ "typeName": "hive_storagedesc",
+ "updateTime": 1529605221342,
+ "updatedBy": "anonymous",
+ "version": 0
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/c3b01a6f/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json b/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json
new file mode 100644
index 0000000..3aac4ea
--- /dev/null
+++ b/repository/src/test/resources/json/stocksDB-Entities/typesdef-new-classification.json
@@ -0,0 +1,22 @@
+{
+ "classificationDefs": [
+ {
+ "attributeDefs": [],
+ "category": "CLASSIFICATION",
+ "createTime": 1528240995728,
+ "createdBy": "admin",
+ "description": "T1",
+ "guid": "29fc3eed-e97a-429c-ad9e-45d6ccb99fce",
+ "name": "T1",
+ "subTypes": [],
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "updateTime": 1528240995728,
+ "updatedBy": "admin",
+ "version": 1
+ }
+ ],
+ "entityDefs": [],
+ "enumDefs": [],
+ "structDefs": []
+}