You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/03/11 07:36:22 UTC
[atlas] branch branch-1.0 updated: ATLAS-3065: include removed
legacy attributes in entity.attributes for backward compatibility (in
addition to entity.relationshipAttributes) - #3
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new ef0708e ATLAS-3065: include removed legacy attributes in entity.attributes for backward compatibility (in addition to entity.relationshipAttributes) - #3
ef0708e is described below
commit ef0708e79848dd37523c2607a1e1e908cb350b87
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Fri Mar 8 18:10:12 2019 -0800
ATLAS-3065: include removed legacy attributes in entity.attributes for backward compatibility (in addition to entity.relationshipAttributes) - #3
(cherry picked from commit 10f34ed8adc41929dcdcbaf8ac5fc23e712ae46d)
---
addons/models/1000-Hadoop/1030-hive_model.json | 128 +++++----------------
addons/models/2000-RDBMS/2010-rdbms_model.json | 10 +-
.../migration/TypesWithCollectionsFinder.java | 10 ++
.../atlas/model/typedef/AtlasRelationshipDef.java | 11 +-
.../org/apache/atlas/type/AtlasEntityType.java | 7 +-
.../apache/atlas/type/AtlasRelationshipType.java | 31 +++--
.../java/org/apache/atlas/type/AtlasTypeUtil.java | 18 +++
.../apache/atlas/query/RegistryBasedLookup.java | 39 +++++--
.../converters/AtlasEntityFormatConverter.java | 45 +++++++-
.../converters/AtlasStructFormatConverter.java | 26 ++++-
.../apache/atlas/repository/graph/GraphHelper.java | 28 +++--
.../repository/impexp/TypeAttributeDifference.java | 8 +-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 2 -
.../store/graph/AtlasTypeDefGraphStore.java | 2 +
.../repository/store/graph/v1/DeleteHandlerV1.java | 6 +-
.../store/graph/v2/AtlasEntityStoreV2.java | 48 +++++++-
.../graph/v2/AtlasRelationshipDefStoreV2.java | 8 +-
.../store/graph/v2/AtlasRelationshipStoreV2.java | 2 +-
.../store/graph/v2/AtlasStructDefStoreV2.java | 16 ++-
.../store/graph/v2/EntityGraphMapper.java | 31 -----
.../store/graph/v2/EntityGraphRetriever.java | 42 ++++---
.../org/apache/atlas/query/BasicTestSetup.java | 32 ++++++
.../impexp/TypeAttributeDifferenceTest.java | 16 ++-
.../ClassificationPropagationTest.java | 4 -
.../web/integration/EntityJerseyResourceIT.java | 4 +-
.../web/resources/AdminExportImportTestIT.java | 2 +-
26 files changed, 358 insertions(+), 218 deletions(-)
diff --git a/addons/models/1000-Hadoop/1030-hive_model.json b/addons/models/1000-Hadoop/1030-hive_model.json
index fe16821..872abc3 100644
--- a/addons/models/1000-Hadoop/1030-hive_model.json
+++ b/addons/models/1000-Hadoop/1030-hive_model.json
@@ -175,17 +175,12 @@
"DataSet"
],
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.1",
+ "typeDefOptions": {
+ "schemaElementsAttribute": "columns"
+ },
"attributeDefs": [
{
- "name": "db",
- "typeName": "hive_db",
- "cardinality": "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
"name": "createTime",
"typeName": "date",
"cardinality": "SINGLE",
@@ -218,32 +213,6 @@
"isUnique": false
},
{
- "name": "sd",
- "typeName": "hive_storagedesc",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
- "name": "partitionKeys",
- "typeName": "array<hive_column>",
- "cardinality": "SET",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
"name": "aliases",
"typeName": "array<string>",
"cardinality": "SET",
@@ -252,19 +221,6 @@
"isUnique": false
},
{
- "name": "columns",
- "typeName": "array<hive_column>",
- "cardinality": "SET",
- "constraints": [
- {
- "type": "ownedRef"
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
"name": "parameters",
"typeName": "map<string,string>",
"cardinality": "SINGLE",
@@ -315,22 +271,6 @@
"typeVersion": "1.0",
"attributeDefs": [
{
- "name": "table",
- "typeName": "hive_table",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "sd"
- }
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
- },
- {
"name": "location",
"typeName": "string",
"cardinality": "SINGLE",
@@ -461,7 +401,10 @@
"DataSet"
],
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.3",
+ "typeDefOptions": {
+ "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
+ },
"attributeDefs": [
{
"name": "type",
@@ -480,20 +423,12 @@
"isUnique": false
},
{
- "name": "table",
- "typeName": "hive_table",
- "cardinality": "SINGLE",
- "constraints": [
- {
- "type": "inverseRef",
- "params": {
- "attribute": "columns"
- }
- }
- ],
- "isIndexable": false,
- "isOptional": true,
- "isUnique": false
+ "name": "position",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
}
]
},
@@ -506,14 +441,6 @@
"typeVersion" : "1.0",
"attributeDefs" : [
{
- "name": "query",
- "typeName": "hive_process",
- "cardinality" : "SINGLE",
- "isIndexable": false,
- "isOptional": false,
- "isUnique": false
- },
- {
"name": "depenendencyType",
"typeName": "string",
"cardinality" : "SINGLE",
@@ -536,8 +463,9 @@
{
"name": "hive_table_db",
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
+ "relationshipLabel": "__hive_table.db",
"endDef1": {
"type": "hive_table",
"name": "db",
@@ -556,8 +484,9 @@
{
"name": "hive_table_columns",
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
+ "relationshipLabel": "__hive_table.columns",
"endDef1": {
"type": "hive_table",
"name": "columns",
@@ -577,8 +506,9 @@
{
"name": "hive_table_partitionkeys",
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
+ "relationshipLabel": "__hive_table.partitionkeys",
"endDef1": {
"type": "hive_table",
"name": "partitionKeys",
@@ -598,8 +528,9 @@
{
"name": "hive_table_storagedesc",
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
+ "relationshipLabel": "__hive_table.sd",
"endDef1": {
"type": "hive_table",
"name": "sd",
@@ -619,21 +550,22 @@
{
"name": "hive_process_column_lineage",
"serviceType": "hive",
- "typeVersion": "1.0",
+ "typeVersion": "1.2",
"relationshipCategory": "COMPOSITION",
+ "relationshipLabel": "__hive_column_lineage.query",
"endDef1": {
- "type": "hive_process",
- "name": "columnLineages",
- "isContainer": true,
- "cardinality": "SET"
- },
- "endDef2": {
"type": "hive_column_lineage",
"name": "query",
"isContainer": false,
"cardinality": "SINGLE",
"isLegacyAttribute": true
},
+ "endDef2": {
+ "type": "hive_process",
+ "name": "columnLineages",
+ "isContainer": true,
+ "cardinality": "SET"
+ },
"propagateTags": "NONE"
}
]
diff --git a/addons/models/2000-RDBMS/2010-rdbms_model.json b/addons/models/2000-RDBMS/2010-rdbms_model.json
index e72e13a..fb2aa44 100644
--- a/addons/models/2000-RDBMS/2010-rdbms_model.json
+++ b/addons/models/2000-RDBMS/2010-rdbms_model.json
@@ -148,7 +148,10 @@
"description": "a table in an rdbms database (schema)",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
- "typeVersion": "1.1",
+ "typeVersion": "1.2",
+ "typeDefOptions": {
+ "schemaElementsAttribute": "columns"
+ },
"attributeDefs": [
{
"name": "db",
@@ -254,7 +257,10 @@
"description": "a column in an rdbms table",
"superTypes": ["DataSet"],
"serviceType": "rdbms",
- "typeVersion": "1.1",
+ "typeVersion": "1.2",
+ "typeDefOptions": {
+ "schemaAttributes": "[\"name\", \"description\", \"owner\", \"data_type\", \"comment\", \" isPrimaryKey\", \" isNullable\"]"
+ },
"attributeDefs": [
{
"name": "data_type",
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
index 9b4499b..7f11a75 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
@@ -19,6 +19,7 @@
package org.apache.atlas.repository.graphdb.janus.migration;
import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasStructType;
@@ -72,6 +73,15 @@ public class TypesWithCollectionsFinder {
addIfCollectionAttribute(attr, collectionProperties);
}
+ if (type instanceof AtlasEntityType) {
+ AtlasEntityType entityType = (AtlasEntityType) type;
+
+ for (Map<String, AtlasAttribute> attrs : entityType.getRelationshipAttributes().values()) {
+ for (AtlasAttribute attr : attrs.values()) {
+ addIfCollectionAttribute(attr, collectionProperties);
+ }
+ }
+ }
return collectionProperties;
} catch (Exception e) {
LOG.error("addVertexPropertiesForCollectionAttributes", e);
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
index 7153351..e3de460 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasRelationshipDef.java
@@ -195,7 +195,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
super(TypeCategory.RELATIONSHIP, name, description, typeVersion, attributeDefs, null);
setRelationshipCategory(relationshipCategory);
- setRelationshipLabel(getDefaultRelationshipLabel());
+ setRelationshipLabel(null);
setPropagateTags(propagatetags);
setEndDef1(endDef1);
setEndDef2(endDef2);
@@ -226,7 +226,7 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
}
public String getRelationshipLabel() {
- return relationshipLabel != null ? relationshipLabel : ("r:" + super.getName());
+ return relationshipLabel;
}
public void setPropagateTags(PropagateTags propagateTags) {
@@ -253,13 +253,6 @@ public class AtlasRelationshipDef extends AtlasStructDef implements java.io.Seri
return this.endDef2;
}
- @JsonIgnore
- private String getDefaultRelationshipLabel() {
- String name = super.getName();
-
- return name != null ? ("r:" + name) : null;
- }
-
@Override
public StringBuilder toString(StringBuilder sb) {
if (sb == null) {
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index ce61548..2557bb3 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -34,7 +34,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -335,7 +334,7 @@ public class AtlasEntityType extends AtlasStructType {
attributes.put(relationshipType.getTypeName(), attribute);
}
- public Collection<String> getAttributeRelationshipTypes(String attributeName) {
+ public Set<String> getAttributeRelationshipTypes(String attributeName) {
Map<String, AtlasAttribute> attributes = relationshipAttributes.get(attributeName);
return attributes != null ? attributes.keySet() : null;
@@ -353,6 +352,10 @@ public class AtlasEntityType extends AtlasStructType {
return typeQryStr;
}
+ public boolean hasAttribute(String attributeName) {
+ return allAttributes.containsKey(attributeName);
+ }
+
public boolean hasRelationshipAttribute(String attributeName) {
return relationshipAttributes.containsKey(attributeName);
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
index 304200d..3ea8d80 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasRelationshipType.java
@@ -50,6 +50,7 @@ public class AtlasRelationshipType extends AtlasStructType {
private final AtlasRelationshipDef relationshipDef;
private final boolean hasLegacyAttributeEnd;
+ private String relationshipLabel;
private AtlasEntityType end1Type;
private AtlasEntityType end2Type;
@@ -75,6 +76,10 @@ public class AtlasRelationshipType extends AtlasStructType {
return this.hasLegacyAttributeEnd;
}
+ public String getRelationshipLabel() {
+ return this.relationshipLabel;
+ }
+
@Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferences(typeRegistry);
@@ -110,19 +115,23 @@ public class AtlasRelationshipType extends AtlasStructType {
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
- String relationshipLabel = null;
-
- // if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
- // if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
- // if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
- if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
- relationshipLabel = relationshipDef.getRelationshipLabel();
- } else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
- relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName());
- } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
- relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName());
+ String relationshipLabel = relationshipDef.getRelationshipLabel();
+
+ if (relationshipLabel == null) {
+ // if legacyLabel is not specified at both ends, use relationshipDef name as relationship label.
+ // if legacyLabel is specified in any one end, use it as the relationship label for both ends (legacy case).
+ // if legacyLabel is specified at both ends use the respective end's legacyLabel as relationship label (legacy case).
+ if (!endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = "r:" + getTypeName();
+ } else if (endDef1.getIsLegacyAttribute() && !endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = getLegacyEdgeLabel(end1Type, endDef1.getName());
+ } else if (!endDef1.getIsLegacyAttribute() && endDef2.getIsLegacyAttribute()) {
+ relationshipLabel = getLegacyEdgeLabel(end2Type, endDef2.getName());
+ }
}
+ this.relationshipLabel = relationshipLabel;
+
addRelationshipAttributeToEndType(endDef1, end1Type, end2Type.getTypeName(), typeRegistry, relationshipLabel);
addRelationshipAttributeToEndType(endDef2, end2Type, end1Type.getTypeName(), typeRegistry, relationshipLabel);
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
index 23624eb..5955aec 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeUtil.java
@@ -141,6 +141,24 @@ public class AtlasTypeUtil {
}
+ public static AtlasRelationshipType findRelationshipWithLegacyRelationshipEnd(String entityTypeName, String attributeName, AtlasTypeRegistry typeRegistry) {
+ AtlasRelationshipType ret = null;
+
+ for (AtlasRelationshipDef relationshipDef : typeRegistry.getAllRelationshipDefs()) {
+ AtlasRelationshipEndDef end1Def = relationshipDef.getEndDef1();
+ AtlasRelationshipEndDef end2Def = relationshipDef.getEndDef2();
+
+ if ((end1Def.getIsLegacyAttribute() && StringUtils.equals(end1Def.getType(), entityTypeName) && StringUtils.equals(end1Def.getName(), attributeName)) ||
+ (end2Def.getIsLegacyAttribute() && StringUtils.equals(end2Def.getType(), entityTypeName) && StringUtils.equals(end2Def.getName(), attributeName))) {
+ ret = typeRegistry.getRelationshipTypeByName(relationshipDef.getName());
+
+ break;
+ }
+ }
+
+ return ret;
+ }
+
public static AtlasAttributeDef createOptionalAttrDef(String name, AtlasType dataType) {
return new AtlasAttributeDef(name, dataType.getTypeName(), true,
Cardinality.SINGLE, 0, 1,
diff --git a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
index e0fb4d8..e67eeaf 100644
--- a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
+++ b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
@@ -103,7 +103,7 @@ class RegistryBasedLookup implements Lookup {
return true;
}
- AtlasType at = et.getAttributeType(attributeName);
+ AtlasType at = getAttributeType(et, attributeName);
if(at == null) {
return false;
}
@@ -135,14 +135,17 @@ class RegistryBasedLookup implements Lookup {
return "";
}
- AtlasStructType.AtlasAttribute attr = et.getAttribute(attributeName);
+ AtlasStructType.AtlasAttribute attr = getAttribute(et, attributeName);
+
return (attr != null) ? attr.getRelationshipEdgeLabel() : "";
}
@Override
public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) {
- return (context.getActiveEntityType() != null) &&
- (isSystemAttribute(typeName) || context.getActiveEntityType().getAttribute(typeName) != null);
+ AtlasEntityType entityType = context.getActiveEntityType();
+
+ return (entityType != null) &&
+ (isSystemAttribute(typeName) || entityType.hasAttribute(typeName) || entityType.hasRelationshipAttribute(typeName));
}
@Override
@@ -197,7 +200,7 @@ class RegistryBasedLookup implements Lookup {
return "";
}
- AtlasStructType.AtlasAttribute attr = et.getAttribute(item);
+ AtlasStructType.AtlasAttribute attr = getAttribute(et, item);
if(attr == null) {
return null;
}
@@ -213,7 +216,7 @@ class RegistryBasedLookup implements Lookup {
return getCollectionElementType(mapType.getValueType());
}
- return context.getActiveEntityType().getAttribute(item).getTypeName();
+ return getAttribute(context.getActiveEntityType(), item).getTypeName();
}
private String getCollectionElementType(AtlasType elemType) {
@@ -231,7 +234,7 @@ class RegistryBasedLookup implements Lookup {
return false;
}
- AtlasType attr = et.getAttributeType(attributeName);
+ AtlasType attr = getAttributeType(et, attributeName);
return attr != null && attr.getTypeName().equals(AtlasBaseTypeDef.ATLAS_TYPE_DATE);
}
@@ -242,7 +245,7 @@ class RegistryBasedLookup implements Lookup {
return false;
}
- AtlasType attr = et.getAttributeType(attrName);
+ AtlasType attr = getAttributeType(et, attrName);
boolean ret = attr != null && NUMERIC_ATTRIBUTES.containsKey(attr.getTypeName());
if(ret) {
context.setNumericTypeFormatter(NUMERIC_ATTRIBUTES.get(attr.getTypeName()));
@@ -250,4 +253,24 @@ class RegistryBasedLookup implements Lookup {
return ret;
}
+
+ private AtlasStructType.AtlasAttribute getAttribute(AtlasEntityType entityType, String attrName) {
+ AtlasStructType.AtlasAttribute ret = null;
+
+ if (entityType != null) {
+ ret = entityType.getAttribute(attrName);
+
+ if (ret == null) {
+ ret = entityType.getRelationshipAttribute(attrName, null);
+ }
+ }
+
+ return ret;
+ }
+
+ private AtlasType getAttributeType(AtlasEntityType entityType, String attrName) {
+ AtlasStructType.AtlasAttribute attribute = getAttribute(entityType, attrName);
+
+ return attribute != null ? attribute.getAttributeType() : null;
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
index ab8c153..6c82b93 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasEntityFormatConverter.java
@@ -39,6 +39,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -127,12 +128,30 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND);
}
- final Map v2Attribs = (Map) v2Map.get(ATTRIBUTES_PROPERTY_KEY);
+ final Map v2Attribs = (Map) v2Map.get(ATTRIBUTES_PROPERTY_KEY);
+ final Map v2RelationshipAttribs = (Map) v2Map.get(RELATIONSHIP_ATTRIBUTES_PROPERTY_KEY);
+ final Map attributes;
+
+ if (MapUtils.isNotEmpty(v2RelationshipAttribs)) {
+ if (MapUtils.isNotEmpty(v2Attribs)) {
+ attributes = new HashMap(v2RelationshipAttribs);
+
+ for (Object key : v2Attribs.keySet()) {
+ if (!attributes.containsKey(key)) {
+ attributes.put(key, v2Attribs.get(key));
+ }
+ }
+ } else {
+ attributes = v2RelationshipAttribs;
+ }
+ } else {
+ attributes = v2Attribs;
+ }
if (MapUtils.isEmpty(v2Attribs)) {
ret = new Id(idStr, 0, typeName);
} else {
- ret = new Referenceable(idStr, typeName, super.fromV2ToV1(entityType, v2Attribs, context));
+ ret = new Referenceable(idStr, typeName, super.fromV2ToV1(entityType, attributes, context));
}
} else if (v2Obj instanceof AtlasEntity) {
AtlasEntity entity = (AtlasEntity) v2Obj;
@@ -142,8 +161,28 @@ public class AtlasEntityFormatConverter extends AtlasStructFormatConverter {
status = Status.ACTIVE;
}
+ final Map<String, Object> v2Attribs = entity.getAttributes();
+ final Map<String, Object> v2RelationshipAttribs = entity.getRelationshipAttributes();
+ final Map<String, Object> attributes;
+
+ if (MapUtils.isNotEmpty(v2RelationshipAttribs)) {
+ if (MapUtils.isNotEmpty(v2Attribs)) {
+ attributes = new HashMap(v2RelationshipAttribs);
+
+ for (String key : v2Attribs.keySet()) {
+ if (!attributes.containsKey(key)) {
+ attributes.put(key, v2Attribs.get(key));
+ }
+ }
+ } else {
+ attributes = v2RelationshipAttribs;
+ }
+ } else {
+ attributes = v2Attribs;
+ }
+
Referenceable referenceable = new Referenceable(entity.getGuid(), entity.getTypeName(), status.name(),
- fromV2ToV1(entityType, entity.getAttributes(), context),
+ fromV2ToV1(entityType, attributes, context),
new AtlasSystemAttributes(entity.getCreatedBy(), entity.getUpdatedBy(), entity.getCreateTime(), entity.getUpdateTime()));
if (CollectionUtils.isNotEmpty(entity.getClassifications())) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
index 539afdc..51a6426 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasStructFormatConverter.java
@@ -41,7 +41,8 @@ import java.util.Map;
public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
private static final Logger LOG = LoggerFactory.getLogger(AtlasStructFormatConverter.class);
- public static final String ATTRIBUTES_PROPERTY_KEY = "attributes";
+ public static final String ATTRIBUTES_PROPERTY_KEY = "attributes";
+ public static final String RELATIONSHIP_ATTRIBUTES_PROPERTY_KEY = "relationshipAttributes";
public AtlasStructFormatConverter(AtlasFormatConverters registry, AtlasTypeRegistry typeRegistry) {
this(registry, typeRegistry, TypeCategory.STRUCT);
@@ -136,8 +137,14 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
AtlasAttribute attr = structType.getAttribute(attrName);
if (attr == null) {
- LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
- continue;
+ if (isEntityType) {
+ attr = ((AtlasEntityType) structType).getRelationshipAttribute(attrName, null);
+ }
+
+ if (attr == null) {
+ LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
+ continue;
+ }
}
AtlasType attrType = attr.getAttributeType();
@@ -240,7 +247,8 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
}
protected Map<String, Object> fromV1ToV2(AtlasStructType structType, Map attributes, ConverterContext context) throws AtlasBaseException {
- Map<String, Object> ret = null;
+ Map<String, Object> ret = null;
+ AtlasEntityType entityType = (structType instanceof AtlasEntityType) ? ((AtlasEntityType) structType) : null;
if (MapUtils.isNotEmpty(attributes)) {
ret = new HashMap<>();
@@ -251,8 +259,14 @@ public class AtlasStructFormatConverter extends AtlasAbstractFormatConverter {
AtlasAttribute attr = structType.getAttribute(attrName);
if (attr == null) {
- LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
- continue;
+ if (entityType != null) {
+ attr = entityType.getRelationshipAttribute(attrName, null);
+ }
+
+ if (attr == null) {
+ LOG.warn("ignored unknown attribute {}.{}", structType.getTypeName(), attrName);
+ continue;
+ }
}
AtlasType attrType = attr.getAttributeType();
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index f3ebc43..fde93b5 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -1773,25 +1773,29 @@ public final class GraphHelper {
* to pick the right relationshipDef name
*/
public String getRelationshipTypeName(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) {
- String ret = null;
- Collection<String> relationshipTypes = entityType.getAttributeRelationshipTypes(attributeName);
+ String ret = null;
+ Set<String> relationshipTypes = entityType.getAttributeRelationshipTypes(attributeName);
if (CollectionUtils.isNotEmpty(relationshipTypes)) {
- Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
+ if (relationshipTypes.size() == 1) {
+ ret = relationshipTypes.iterator().next();
+ } else {
+ Iterator<AtlasEdge> iter = entityVertex.getEdges(AtlasEdgeDirection.IN).iterator();
- while (iter.hasNext() && ret == null) {
- String edgeTypeName = AtlasGraphUtilsV2.getTypeName(iter.next());
+ while (iter.hasNext() && ret == null) {
+ String edgeTypeName = AtlasGraphUtilsV2.getTypeName(iter.next());
- if (relationshipTypes.contains(edgeTypeName)) {
- ret = edgeTypeName;
+ if (relationshipTypes.contains(edgeTypeName)) {
+ ret = edgeTypeName;
- break;
+ break;
+ }
}
- }
- if (ret == null) {
- //relationshipTypes will have at least one relationshipDef
- ret = relationshipTypes.iterator().next();
+ if (ret == null) {
+ //relationshipTypes will have at least one relationshipDef
+ ret = relationshipTypes.iterator().next();
+ }
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
index 37071f2..e3d86c8 100644
--- a/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
+++ b/repository/src/main/java/org/apache/atlas/repository/impexp/TypeAttributeDifference.java
@@ -27,7 +27,9 @@ import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,7 +119,11 @@ public class TypeAttributeDifference {
AtlasStructDef.AtlasAttributeDef incoming) throws AtlasBaseException {
AtlasStructDef.AtlasAttributeDef existingAttribute = existing.getAttribute(incoming.getName());
if (existingAttribute == null) {
- difference.add(incoming);
+ AtlasRelationshipType relationshipType = AtlasTypeUtil.findRelationshipWithLegacyRelationshipEnd(existing.getName(), incoming.getName(), typeRegistry);
+
+ if (relationshipType == null) {
+ difference.add(incoming);
+ }
} else {
if (!existingAttribute.getTypeName().equals(incoming.getTypeName())) {
LOG.error("Attribute definition difference found: {}, {}", existingAttribute, incoming);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
index 66162aa..b0c224c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -793,8 +793,6 @@ public class AtlasTypeDefStoreInitializer implements ActiveStateChangeHandler {
updatedDef.setRelationshipCategory(newRelationshipCategory);
}
- end1Def.setIsLegacyAttribute(false);
- end2Def.setIsLegacyAttribute(false);
updatedDef.setTypeVersion(patch.getUpdateToVersion());
AtlasEntityDef updatedEntityDef1 = new AtlasEntityDef(end1Type.getEntityDef());
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index b142179..5cb57a4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -77,6 +77,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore {
protected abstract AtlasDefStore<AtlasRelationshipDef> getRelationshipDefStore(AtlasTypeRegistry typeRegistry);
+ public AtlasTypeRegistry getTypeRegistry() { return typeRegistry; }
+
@Override
public void init() throws AtlasBaseException {
LOG.info("==> AtlasTypeDefGraphStore.init()");
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index d2f5fda..0de9229 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -690,7 +690,11 @@ public abstract class DeleteHandlerV1 {
AtlasEntityType parentEntityType = (AtlasEntityType) parentType;
AtlasStructType.AtlasAttribute attribute = parentEntityType.getAttribute(atlasEdgeLabel.getAttributeName());
- if (attribute.getInverseRefAttribute() != null) {
+ if (attribute == null) {
+ attribute = parentEntityType.getRelationshipAttribute(atlasEdgeLabel.getAttributeName(), AtlasGraphUtilsV2.getTypeName(edge));
+ }
+
+ if (attribute != null && attribute.getInverseRefAttribute() != null) {
deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(), attribute.getInverseRefAttribute());
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index bd3a62a..a62f335 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -26,6 +26,7 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest;
import org.apache.atlas.authorize.AtlasPrivilege;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
@@ -322,7 +323,11 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
AtlasAttribute attr = entityType.getAttribute(attrName);
if (attr == null) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entity.getTypeName());
+ attr = entityType.getRelationshipAttribute(attrName, null);
+
+ if (attr == null) {
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entity.getTypeName());
+ }
}
AtlasType attrType = attr.getAttributeType();
@@ -856,6 +861,14 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
AtlasEntity entity = entityStream.getByGuid(guid);
if (entity != null) { // entity would be null if guid is not in the stream but referenced by an entity in the stream
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+
+ if (entityType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), entity.getTypeName());
+ }
+
+ compactAttributes(entity, entityType);
+
AtlasVertex vertex = getResolvedEntityVertex(discoveryContext, entity);
if (vertex != null) {
@@ -865,8 +878,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
graphDiscoverer.validateAndNormalizeForUpdate(entity);
}
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
- String guidVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex);
+ String guidVertex = AtlasGraphUtilsV2.getIdFromVertex(vertex);
if (!StringUtils.equals(guidVertex, guid)) { // if entity was found by unique attribute
entity.setGuid(guidVertex);
@@ -878,8 +890,6 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
} else {
graphDiscoverer.validateAndNormalize(entity);
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
-
//Create vertices which do not exist in the repository
if (RequestContext.get().isImportInProgress() && AtlasTypeUtil.isAssignedGuid(entity.getGuid())) {
vertex = entityGraphMapper.createVertexWithGuid(entity, entity.getGuid());
@@ -1039,4 +1049,32 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
return ret;
}
+
+ // move/remove relationship-attributes present in 'attributes'
+ private void compactAttributes(AtlasEntity entity, AtlasEntityType entityType) {
+ if (entity != null) {
+ for (String attrName : entityType.getRelationshipAttributes().keySet()) {
+ if (entity.hasAttribute(attrName)) { // relationship attribute is present in 'attributes'
+ Object attrValue = entity.removeAttribute(attrName);
+
+ if (attrValue != null) {
+ // if the attribute doesn't exist in relationshipAttributes, add it
+ Object relationshipAttrValue = entity.getRelationshipAttribute(attrName);
+
+ if (relationshipAttrValue == null) {
+ entity.setRelationshipAttribute(attrName, attrValue);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("moved attribute {}.{} from attributes to relationshipAttributes", entityType.getTypeName(), attrName);
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("attribute {}.{} is present in attributes and relationshipAttributes. Removed from attributes", entityType.getTypeName(), attrName);
+ }
+ }
+ }
+ }
+ }
+ }
+ }
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
index 332c18a..64ef18b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipDefStoreV2.java
@@ -470,9 +470,11 @@ public class AtlasRelationshipDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasRe
// Update RelationshipCategory
vertex.setProperty(Constants.RELATIONSHIPTYPE_CATEGORY_KEY, relationshipCategory);
- vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
-
- vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
+ if (relationshipDef.getRelationshipLabel() == null) {
+ vertex.removeProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY);
+ } else {
+ vertex.setProperty(Constants.RELATIONSHIPTYPE_LABEL_KEY, relationshipDef.getRelationshipLabel());
+ }
if (relationshipDef.getPropagateTags() == null) {
vertex.setProperty(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, AtlasRelationshipDef.PropagateTags.NONE.name());
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 9d372d3..b777dc3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -798,7 +798,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
}
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationshipTypeName);
- String ret = relationshipType.getRelationshipDef().getRelationshipLabel();
+ String ret = relationshipType.getRelationshipLabel();
AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipType.getRelationshipDef().getEndDef2();
Set<String> fromVertexTypes = getTypeAndAllSuperTypes(AtlasGraphUtilsV2.getTypeName(fromVertex));
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
index 8110cc6..9ecc6e3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasStructDefStoreV2.java
@@ -29,15 +29,14 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
import org.apache.atlas.v1.model.typedef.AttributeDefinition;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
-import org.apache.atlas.v1.model.typedef.Multiplicity;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
@@ -425,6 +424,19 @@ public class AtlasStructDefStoreV2 extends AtlasAbstractDefStoreV2<AtlasStructDe
if (CollectionUtils.isNotEmpty(structDef.getAttributeDefs())) {
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) {
+
+ // this could have been an attribute removed by a REMOVE_LEGACY_REF_ATTRIBUTE patch
+ // in such case, don't add this attribute; ignore and continue
+ AtlasRelationshipType relationship = AtlasTypeUtil.findRelationshipWithLegacyRelationshipEnd(structDef.getName(), attributeDef.getName(), typeDefStore.getTypeRegistry());
+
+ if (relationship != null) {
+ attrNames.remove(attributeDef.getName());
+
+ LOG.warn("Ignoring attempt to add legacy attribute {}.{}, which is already present in relationship {}", structDef.getName(), attributeDef.getName(), relationship.getTypeName());
+
+ continue;
+ }
+
// new attribute - only allow if optional
if (!attributeDef.getIsOptional()) {
throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName());
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 601e5f8..a6f1250 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -189,8 +189,6 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = context.getType(guid);
- compactAttributes(createdEntity, entityType);
-
mapRelationshipAttributes(createdEntity, entityType, vertex, CREATE, context);
mapAttributes(createdEntity, entityType, vertex, CREATE, context);
@@ -206,8 +204,6 @@ public class EntityGraphMapper {
AtlasVertex vertex = context.getVertex(guid);
AtlasEntityType entityType = context.getType(guid);
- compactAttributes(updatedEntity, entityType);
-
mapRelationshipAttributes(updatedEntity, entityType, vertex, UPDATE, context);
mapAttributes(updatedEntity, entityType, vertex, UPDATE, context);
@@ -1901,33 +1897,6 @@ public class EntityGraphMapper {
}
}
- // move/remove relationship-attributes present in 'attributes'
- private static void compactAttributes(AtlasEntity entity, AtlasEntityType entityType) {
- if (entity != null) {
- for (String attrName : entityType.getRelationshipAttributes().keySet()) {
-
- if (entity.hasAttribute(attrName)) {
- Object attrValue = entity.getAttribute(attrName);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("relationship attribute {}.{} is present in entity, removing it", entityType.getTypeName(), attrName);
- }
-
- entity.removeAttribute(attrName);
-
- if (attrValue != null) { // relationship attribute is present in 'attributes'
- // if the attribute doesn't exist in relationshipAttributes, add it
- Object relationshipAttrValue = entity.getRelationshipAttribute(attrName);
-
- if (relationshipAttrValue == null) {
- entity.setRelationshipAttribute(attrName, attrValue);
- }
- }
- }
- }
- }
- }
-
private String getIdFromInVertex(AtlasEdge edge) {
return getIdFromVertex(edge.getInVertex());
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 4c85afa..9ada34b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -386,7 +386,7 @@ public class EntityGraphRetriever {
mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
if (!ignoreRelationshipAttr) { // only map when really needed
- mapRelationshipAttributes(entityVertex, entity);
+ mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
}
mapClassifications(entityVertex, entity);
@@ -990,7 +990,7 @@ public class EntityGraphRetriever {
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null;
}
- private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException {
+ private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
if (entityType == null) {
@@ -998,13 +998,11 @@ public class EntityGraphRetriever {
}
for (String attributeName : entityType.getRelationshipAttributes().keySet()) {
- Object attrValue = mapVertexToRelationshipAttribute(entityVertex, entityType, attributeName);
-
- entity.setRelationshipAttribute(attributeName, attrValue);
+ mapVertexToRelationshipAttribute(entityVertex, entityType, attributeName, entity, entityExtInfo, isMinExtInfo);
}
}
- private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName) throws AtlasBaseException {
+ private Object mapVertexToRelationshipAttribute(AtlasVertex entityVertex, AtlasEntityType entityType, String attributeName, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
Object ret = null;
String relationshipTypeName = graphHelper.getRelationshipTypeName(entityVertex, entityType, attributeName);
AtlasRelationshipType relationshipType = relationshipTypeName != null ? typeRegistry.getRelationshipTypeByName(relationshipTypeName) : null;
@@ -1013,6 +1011,7 @@ public class EntityGraphRetriever {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIPDEF_INVALID, "relationshipDef is null");
}
+ AtlasAttribute attribute = entityType.getRelationshipAttribute(attributeName, relationshipTypeName);
AtlasRelationshipDef relationshipDef = relationshipType.getRelationshipDef();
AtlasRelationshipEndDef endDef1 = relationshipDef.getEndDef1();
AtlasRelationshipEndDef endDef2 = relationshipDef.getEndDef2();
@@ -1032,25 +1031,33 @@ public class EntityGraphRetriever {
switch (attributeEndDef.getCardinality()) {
case SINGLE:
- ret = mapRelatedVertexToObjectId(entityVertex, entityType.getRelationshipAttribute(attributeName, relationshipTypeName));
+ ret = mapRelatedVertexToObjectId(entityVertex, attribute, entityExtInfo, isMinExtInfo);
break;
case LIST:
case SET:
- ret = mapRelationshipArrayAttribute(entityVertex, entityType.getRelationshipAttribute(attributeName, relationshipTypeName));
+ ret = mapRelationshipArrayAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo);
break;
}
+ if (ret != null) {
+ entity.setRelationshipAttribute(attributeName, ret);
+
+ if (attributeEndDef.getIsLegacyAttribute() && !entity.hasAttribute(attributeName)) {
+ entity.setAttribute(attributeName, ret);
+ }
+ }
+
return ret;
}
- private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
+ private AtlasObjectId mapRelatedVertexToObjectId(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEdge edge = graphHelper.getEdgeForLabel(entityVertex, attribute.getRelationshipEdgeLabel(), attribute.getRelationshipEdgeDirection());
- return mapVertexToRelatedObjectId(entityVertex, edge);
+ return mapVertexToRelatedObjectId(entityVertex, edge, attribute.isOwnedRef(), entityExtInfo, isMinExtInfo);
}
- private List<AtlasRelatedObjectId> mapRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasAttribute attribute) throws AtlasBaseException {
+ private List<AtlasRelatedObjectId> mapRelationshipArrayAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
List<AtlasRelatedObjectId> ret = new ArrayList<>();
Iterator<AtlasEdge> edges = null;
@@ -1066,7 +1073,7 @@ public class EntityGraphRetriever {
while (edges.hasNext()) {
AtlasEdge relationshipEdge = edges.next();
- AtlasRelatedObjectId relatedObjectId = mapVertexToRelatedObjectId(entityVertex, relationshipEdge);
+ AtlasRelatedObjectId relatedObjectId = mapVertexToRelatedObjectId(entityVertex, relationshipEdge, attribute.isOwnedRef(), entityExtInfo, isMinExtInfo);
ret.add(relatedObjectId);
}
@@ -1075,7 +1082,7 @@ public class EntityGraphRetriever {
return ret;
}
- private AtlasRelatedObjectId mapVertexToRelatedObjectId(AtlasVertex entityVertex, AtlasEdge edge) throws AtlasBaseException {
+ private AtlasRelatedObjectId mapVertexToRelatedObjectId(AtlasVertex entityVertex, AtlasEdge edge, boolean isOwnedRef, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasRelatedObjectId ret = null;
if (GraphHelper.elementExists(edge)) {
@@ -1100,6 +1107,15 @@ public class EntityGraphRetriever {
if (displayText != null) {
ret.setDisplayText(displayText.toString());
}
+
+ if (isOwnedRef && entityExtInfo != null) {
+ if (isMinExtInfo) {
+ mapVertexToAtlasEntityMin(referenceVertex, entityExtInfo);
+ } else {
+ mapVertexToAtlasEntity(referenceVertex, entityExtInfo);
+ }
+ }
+
}
}
diff --git a/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
index ba6616d..0bf8554 100644
--- a/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/query/BasicTestSetup.java
@@ -162,6 +162,9 @@ public abstract class BasicTestSetup {
column("weekDay", "int", "week Day"));
entities.addAll(timeDimColumns);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
"Dimension");
entities.add(timeDim);
@@ -170,14 +173,23 @@ public abstract class BasicTestSetup {
database("Reporting", "reporting database", "Jane BI", "hdfs://host:8000/apps/warehouse/reporting");
entities.add(reportingDB);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity salesFactDaily =
table("sales_fact_daily_mv", "sales fact daily materialized view", reportingDB, sd, "Joe BI", "Managed",
salesFactColumns, "Metric");
entities.add(salesFactDaily);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity circularLineageTable1 = table("table1", "", reportingDB, sd, "Vimal", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable1);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity circularLineageTable2 = table("table2", "", reportingDB, sd, "Vimal 2", "Managed", salesFactColumns, "Metric");
entities.add(circularLineageTable2);
@@ -196,6 +208,9 @@ public abstract class BasicTestSetup {
AtlasEntity logDB = database("Logging", "logging database", "Tim ETL", "hdfs://host:8000/apps/warehouse/logging");
entities.add(logDB);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity loggingFactDaily =
table("log_fact_daily_mv", "log fact daily materialized view", logDB, sd, "Tim ETL", "Managed",
logFactColumns, "Log Data");
@@ -207,6 +222,9 @@ public abstract class BasicTestSetup {
column("brand_name", "int", "brand name"));
entities.addAll(productDimColumns);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity productDim =
table("product_dim", "product dimension table", salesDB, sd, "John Doe 2", "Managed", productDimColumns,
"Dimension");
@@ -221,6 +239,9 @@ public abstract class BasicTestSetup {
column("address", "string", "customer address", "PII"));
entities.addAll(customerDimColumns);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity customerDim =
table("customer_dim", "customer dimension table", salesDB, sd, "fetl", "External", customerDimColumns,
"Dimension");
@@ -229,6 +250,9 @@ public abstract class BasicTestSetup {
AtlasEntity customerDimView = view("customer_dim_view", reportingDB, ImmutableList.of(customerDim), "Dimension", "JdbcAccess");
entities.add(customerDimView);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity salesFactMonthly =
table("sales_fact_monthly_mv", "sales fact monthly materialized view", reportingDB, sd, "Jane BI",
"Managed", salesFactColumns, "Metric");
@@ -238,6 +262,9 @@ public abstract class BasicTestSetup {
ImmutableList.of(salesFactMonthly), "create table as select ", "plan", "id", "graph", "ETL");
entities.add(loadSalesMonthly);
+ sd = storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ entities.add(sd);
+
AtlasEntity loggingFactMonthly =
table("logging_fact_monthly_mv", "logging fact monthly materialized view", logDB, sd, "Tim ETL 2",
"Managed", logFactColumns, "Log Data");
@@ -339,8 +366,13 @@ public abstract class BasicTestSetup {
sd.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "@" + clusterName + "_storage");
+ AtlasObjectId tableId = getAtlasObjectId(table);
+
+ sd.setAttribute("table", tableId);
+
for (AtlasEntity column : columns) {
column.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, dbName + "." + name + "." + column.getAttribute(AtlasClient.NAME).toString() + "@" + clusterName);
+ column.setAttribute("table", tableId);
}
return table;
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
index 87cb1d4..f9b7cd1 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/TypeAttributeDifferenceTest.java
@@ -21,6 +21,11 @@ import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.MockitoAnnotations;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -31,9 +36,18 @@ import java.util.List;
public class TypeAttributeDifferenceTest {
private TypeAttributeDifference typeAttributeDifference;
+ @Mock
+ AtlasTypeDefStore typeDefStore;
+
+ @Mock
+ AtlasTypeRegistry typeRegistry;
+
+
@BeforeClass
public void setup() {
- typeAttributeDifference = new TypeAttributeDifference(null, null);
+ MockitoAnnotations.initMocks(this);
+
+ typeAttributeDifference = new TypeAttributeDifference(typeDefStore, typeRegistry);
}
private List<AtlasEnumDef.AtlasEnumElementDef> getEnumElementDefs(int startIndex, String... names) {
diff --git a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
index d5f36bc..ea441ac 100644
--- a/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/tagpropagation/ClassificationPropagationTest.java
@@ -591,10 +591,6 @@ public class ClassificationPropagationTest {
loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
- loadModelFromJson("1000-Hadoop/patches/001-hive_column_add_position.json", typeDefStore, typeRegistry);
- loadModelFromJson("1000-Hadoop/patches/002-hive_column_table_add_options.json", typeDefStore, typeRegistry);
- loadModelFromJson("1000-Hadoop/patches/003-hive_column_update_table_remove_constraint.json", typeDefStore, typeRegistry);
-
loadSampleClassificationDefs();
runImportWithNoParameters(importService, getZipSource(IMPORT_FILE));
diff --git a/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java
index e4887d2..15a06e3 100755
--- a/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/integration/EntityJerseyResourceIT.java
@@ -998,8 +998,8 @@ public class EntityJerseyResourceIT extends BaseResourceIT {
List<Referenceable> refs = (List<Referenceable>) hiveTableInstance.get("columns");
Assert.assertEquals(refs.size(), 2);
- Assert.assertTrue(refs.get(0).getValuesMap().equals(values1));
- Assert.assertTrue(refs.get(1).getValuesMap().equals(values2));
+ Assert.assertEquals(refs.get(0).getValuesMap(), values1);
+ Assert.assertEquals(refs.get(1).getValuesMap(), values2);
}
@Test
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
index 25c47aa..bfb123d 100644
--- a/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/AdminExportImportTestIT.java
@@ -72,7 +72,7 @@ public class AdminExportImportTestIT extends BaseResourceIT {
ZipSource zs = new ZipSource(exportedStream);
assertNotNull(zs.getExportResult());
- assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE);
+ assertTrue(zs.getCreationOrder().size() > EXPECTED_CREATION_ORDER_SIZE, "expected creationOrderSize > " + EXPECTED_CREATION_ORDER_SIZE + ", but found " + zs.getCreationOrder().size());
}
private void performImport(String fileToImport, int expectedProcessedEntitiesCount) throws AtlasServiceException {