You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2018/05/14 15:53:15 UTC
atlas git commit: ATLAS-2672: Change primitive map type storage in
vertex
Repository: atlas
Updated Branches:
refs/heads/master b7641e742 -> cbca591de
ATLAS-2672: Change primitive map type storage in vertex
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/cbca591d
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/cbca591d
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/cbca591d
Branch: refs/heads/master
Commit: cbca591dea186b88f752f4b66b7fb27f06477f7e
Parents: b7641e7
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Mon May 14 08:52:24 2018 -0700
Committer: Sarath Subramanian <ss...@hortonworks.com>
Committed: Mon May 14 08:52:24 2018 -0700
----------------------------------------------------------------------
.../graphdb/janus/AtlasJanusGraphDatabase.java | 8 +-
.../test/java/org/apache/atlas/TestUtilsV2.java | 58 ++++++++-
.../graph/GraphBackedSearchIndexer.java | 68 ++++++++--
.../atlas/repository/graph/GraphHelper.java | 33 ++---
.../store/graph/v1/DeleteHandlerV1.java | 2 +-
.../store/graph/v1/EntityGraphMapper.java | 125 ++++++-------------
.../store/graph/v1/EntityGraphRetriever.java | 8 +-
.../repository/migration/HiveStocksTest.java | 6 +-
.../graph/v1/AtlasComplexAttributesTest.java | 118 ++++++++++++++++-
.../store/graph/v1/AtlasEntityTestBase.java | 1 +
10 files changed, 287 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 3636ca6..16aecd5 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -70,7 +70,6 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
private static volatile JanusGraph graphInstance;
public AtlasJanusGraphDatabase() {
-
//update registry
GraphSONMapper.build().addRegistry(JanusGraphIoRegistry.getInstance()).create();
}
@@ -79,14 +78,11 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
startLocalSolr();
Configuration configProperties = ApplicationProperties.get();
-
- Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
+ Configuration janusConfig = ApplicationProperties.getSubsetConfiguration(configProperties, GRAPH_PREFIX);
//add serializers for non-standard property value types that Atlas uses
-
janusConfig.addProperty("attributes.custom.attribute1.attribute-class", TypeCategory.class.getName());
- janusConfig.addProperty("attributes.custom.attribute1.serializer-class",
- TypeCategorySerializer.class.getName());
+ janusConfig.addProperty("attributes.custom.attribute1.serializer-class", TypeCategorySerializer.class.getName());
//not ideal, but avoids making large changes to Atlas
janusConfig.addProperty("attributes.custom.attribute2.attribute-class", ArrayList.class.getName());
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index f48ccef..886ce77 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -18,6 +18,7 @@
package org.apache.atlas;
+import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
@@ -67,9 +68,10 @@ public final class TestUtilsV2 {
public static final long TEST_DATE_IN_LONG = 1418265358440L;
- public static final String TEST_USER = "testUser";
+ public static final String TEST_USER = "testUser";
public static final String STRUCT_TYPE = "struct_type";
public static final String ENTITY_TYPE = "entity_type";
+ public static final String ENTITY_TYPE_MAP = "map_entity_type";
private static AtomicInteger seq = new AtomicInteger();
@@ -977,6 +979,21 @@ public final class TestUtilsV2 {
return ret;
}
+ public static AtlasTypesDef defineTypeWithMapAttributes() {
+ AtlasEntityDef entityType = createClassTypeDef(ENTITY_TYPE_MAP, "entity_type_map_description", Collections.emptySet(),
+ createUniqueRequiredAttrDef("mapAttr1", "map<string,string>"),
+ createUniqueRequiredAttrDef("mapAttr2", "map<string,int>"),
+ createUniqueRequiredAttrDef("mapAttr3", "map<string,boolean>"),
+ createOptionalAttrDef("mapAttr4", "map<string,float>"),
+ createOptionalAttrDef("mapAttr5", "map<string,date>"));
+
+ AtlasTypesDef ret = AtlasTypeUtil.getTypesDef(Collections.emptyList(), Collections.emptyList(), Collections.emptyList(), Arrays.asList(entityType));
+
+ populateSystemAttributes(ret);
+
+ return ret;
+ }
+
public static AtlasEntityWithExtInfo createNestedCollectionAttrEntity() {
AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_WITH_NESTED_COLLECTION_ATTR);
@@ -1059,6 +1076,45 @@ public final class TestUtilsV2 {
return ret;
}
+ public static AtlasEntityWithExtInfo createMapAttrEntity() {
+ AtlasEntity entity = new AtlasEntity(ENTITY_TYPE_MAP);
+
+ Map<String, String> map1 = new HashMap<>();
+ map1.put("map1Key1", "value1");
+ map1.put("map1Key2", "value2");
+ map1.put("map1Key3", "value3");
+
+ Map<String, Integer> map2 = new HashMap<>();
+ map2.put("map2Key1", 100);
+ map2.put("map2Key2", 200);
+ map2.put("map2Key3", 300);
+
+ Map<String, Boolean> map3 = new HashMap<>();
+ map3.put("map3Key1", false);
+ map3.put("map3Key2", true);
+ map3.put("map3Key3", false);
+
+ Map<String, Float> map4 = new HashMap<>();
+ map4.put("map4Key1", 1.0f);
+ map4.put("map4Key2", 2.0f);
+ map4.put("map4Key3", 3.0f);
+
+ Map<String, Date> map5 = new HashMap<>();
+ map5.put("map5Key1", new Date());
+ map5.put("map5Key2", new Date());
+ map5.put("map5Key3", new Date());
+
+ entity.setAttribute("mapAttr1", map1);
+ entity.setAttribute("mapAttr2", map2);
+ entity.setAttribute("mapAttr3", map3);
+ entity.setAttribute("mapAttr4", map4);
+ entity.setAttribute("mapAttr5", map5);
+
+ AtlasEntityWithExtInfo ret = new AtlasEntityWithExtInfo(entity);
+
+ return ret;
+ }
+
public static final String randomString() {
return RandomStringUtils.randomAlphanumeric(10);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index f75f786..f289365 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -42,9 +42,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphIndex;
import org.apache.atlas.repository.graphdb.AtlasGraphManagement;
import org.apache.atlas.repository.graphdb.AtlasPropertyKey;
+import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
+import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasEnumType;
+import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
@@ -62,6 +65,7 @@ import java.math.BigInteger;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -71,6 +75,9 @@ import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.LIST;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SET;
import static org.apache.atlas.repository.graphdb.AtlasCardinality.SINGLE;
+import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.isReference;
+import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
+import static org.apache.atlas.type.AtlasTypeUtil.isMapType;
/**
@@ -267,7 +274,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
createVertexIndex(management, TYPENAME_PROPERTY_KEY, String.class, true, SINGLE, true, true);
createVertexIndex(management, VERTEX_TYPE_PROPERTY_KEY, String.class, false, SINGLE, true, true);
createVertexIndex(management, CLASSIFICATION_ENTITY_GUID, String.class, false, SINGLE, true, true);
-
createVertexIndex(management, VERTEX_ID_IN_IMPORT_KEY, Long.class, false, SINGLE, true, false);
// create vertex-centric index
@@ -316,19 +322,42 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
- boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
- boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
+ boolean isArrayType = isArrayType(attribTypeName);
+ boolean isMapType = isMapType(attribTypeName);
try {
AtlasType atlasType = typeRegistry.getType(typeName);
AtlasType attributeType = typeRegistry.getType(attribTypeName);
- if (isMapType || isClassificationType(attributeType)) {
+ if (isClassificationType(attributeType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
- } if (isArrayType) {
+ }
+
+ if (isArrayType) {
createLabelIfNeeded(management, propertyName, attribTypeName);
- } if (isEntityType(attributeType)) {
+
+ AtlasArrayType arrayType = (AtlasArrayType) attributeType;
+ boolean isReference = isReference(arrayType.getElementType());
+
+ if (!isReference) {
+ createPropertyKey(management, propertyName, ArrayList.class, SINGLE);
+ }
+ }
+
+ if (isMapType) {
+ createLabelIfNeeded(management, propertyName, attribTypeName);
+
+ AtlasMapType mapType = (AtlasMapType) attributeType;
+ boolean isReference = isReference(mapType.getValueType());
+
+ if (!isReference) {
+ createPropertyKey(management, propertyName, HashMap.class, SINGLE);
+ }
+ }
+
+ if (isEntityType(attributeType)) {
createEdgeLabel(management, propertyName);
+
} else if (isBuiltInType) {
if (isRelationshipType(atlasType)) {
createEdgeIndex(management, propertyName, getPrimitiveClass(attribTypeName), cardinality, false);
@@ -381,7 +410,9 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
private Class getPrimitiveClass(String attribTypeName) {
- switch (attribTypeName.toLowerCase()) {
+ String attributeTypeName = attribTypeName.toLowerCase();
+
+ switch (attributeTypeName) {
case ATLAS_TYPE_BOOLEAN:
return Boolean.class;
case ATLAS_TYPE_BYTE:
@@ -437,6 +468,16 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
+ private AtlasPropertyKey createPropertyKey(AtlasGraphManagement management, String propertyName, Class propertyClass, AtlasCardinality cardinality) {
+ AtlasPropertyKey propertyKey = management.getPropertyKey(propertyName);
+
+ if (propertyKey == null) {
+ propertyKey = management.makePropertyKey(propertyName, propertyClass, cardinality);
+ }
+
+ return propertyKey;
+ }
+
private AtlasPropertyKey createVertexIndex(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeIndex,
boolean createCompositeIndexWithTypeAndSuperTypes) {
@@ -689,17 +730,17 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
private void cleanupIndexForAttribute(AtlasGraphManagement management, String typeName, AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
- String attribTypeName = attributeDef.getTypeName();
- boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
- boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
- boolean isMapType = AtlasTypeUtil.isMapType(attribTypeName);
+ String attribTypeName = attributeDef.getTypeName();
+ boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
+ boolean isArrayType = isArrayType(attribTypeName);
+ boolean isMapType = isMapType(attribTypeName);
try {
AtlasType atlasType = typeRegistry.getType(attribTypeName);
- if (isMapType || isArrayType || isClassificationType(atlasType) || isEntityType(atlasType)) {
+ if (isClassificationType(atlasType) || isEntityType(atlasType)) {
LOG.warn("Ignoring non-indexable attribute {}", attribTypeName);
- } else if (isBuiltInType || isEnumType(atlasType)) {
+ } else if (isBuiltInType || isEnumType(atlasType) || isArrayType || isMapType) {
cleanupIndex(management, propertyName);
} else if (isStructType(atlasType)) {
AtlasStructDef structDef = typeRegistry.getStructDefByName(attribTypeName);
@@ -714,6 +755,7 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
if (LOG.isDebugEnabled()) {
LOG.debug("Invalidating property key = {}", propertyKey);
}
+
management.deletePropertyKey(propertyKey);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index 7490a15..57dc5bd 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -1567,23 +1567,13 @@ public final class GraphHelper {
return typeName != null && typeName.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX);
}
- public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
- String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
-
- if (!AtlasGraphUtilsV1.isReference(elementType)) {
- return instanceVertex.getProperty(vertexPropertyName, Object.class);
- }
-
- return null;
- }
-
- public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName, AtlasAttribute attribute) {
- String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
+ public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, AtlasAttribute attribute) {
+ String propertyName = attribute.getVertexPropertyName();
if (isReference(elementType)) {
return (List) getCollectionElementsUsingRelationship(instanceVertex, attribute);
} else {
- return (List) instanceVertex.getListProperty(encodedPropertyName);
+ return (List) instanceVertex.getListProperty(propertyName);
}
}
@@ -1593,7 +1583,7 @@ public final class GraphHelper {
if (isReference(mapValueType)) {
return getReferenceMap(instanceVertex, attribute);
} else {
- return getPrimitiveMap(instanceVertex, propertyName, mapValueType);
+ return (Map) instanceVertex.getProperty(propertyName, Map.class);
}
}
@@ -1622,18 +1612,11 @@ public final class GraphHelper {
}
// map elements for primitive types
- public static Map<String,Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName, AtlasType mapValueType) {
- String encodedPropertyName = encodePropertyKey(propertyName);
- List<String> currentKeys = getListProperty(instanceVertex, encodedPropertyName);
- Map<String, Object> ret = new HashMap<>();
+ public static Map<String, Object> getPrimitiveMap(AtlasVertex instanceVertex, String propertyName) {
+ Map<String, Object> ret = instanceVertex.getProperty(encodePropertyKey(propertyName), Map.class);
- if (CollectionUtils.isNotEmpty(currentKeys)) {
- for (String key : currentKeys) {
- String propertyNameForKey = getQualifiedNameForMapKey(encodedPropertyName, encodePropertyKey(key));
- Object propertyValueForKey = getMapValueProperty(mapValueType, instanceVertex, propertyNameForKey);
-
- ret.put(key, propertyValueForKey);
- }
+ if (ret == null) {
+ ret = new HashMap<>();
}
return ret;
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index d285e69..e62c041 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -202,7 +202,7 @@ public abstract class DeleteHandlerV1 {
vertexInfoMap.put(guid, new GraphHelper.VertexInfo(entity, vertex));
for (AtlasStructType.AtlasAttribute attributeInfo : entityType.getAllAttributes().values()) {
- if (! attributeInfo.isOwnedRef()) {
+ if (!attributeInfo.isOwnedRef()) {
continue;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index cd00639..287dacc 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -19,7 +19,6 @@ package org.apache.atlas.repository.store.graph.v1;
import org.apache.atlas.AtlasErrorCode;
-import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TimeBoundary;
@@ -82,9 +81,7 @@ import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY
import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertex;
-import static org.apache.atlas.repository.graph.GraphHelper.getListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.getMapElementsProperty;
-import static org.apache.atlas.repository.graph.GraphHelper.getQualifiedNameForMapKey;
import static org.apache.atlas.repository.graph.GraphHelper.getStatus;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames;
@@ -92,7 +89,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames;
import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled;
import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge;
-import static org.apache.atlas.repository.graph.GraphHelper.setListProperty;
import static org.apache.atlas.repository.graph.GraphHelper.string;
import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromVertex;
@@ -811,51 +807,45 @@ public class EntityGraphMapper {
boolean isReference = isReference(mapType.getValueType());
if (MapUtils.isNotEmpty(newVal)) {
- AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
+ String propertyName = ctx.getVertexProperty();
- for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
- String key = entry.getKey().toString();
- String propertyName = (isReference) ? ctx.getVertexProperty() : getQualifiedNameForMapKey(ctx.getVertexProperty(), GraphHelper.encodePropertyKey(key));
- AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
-
- AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
- propertyName, mapType.getValueType(), existingEdge);
-
- //Add/Update/Remove property value
- Object newEntry = mapCollectionElementsToVertex(mapCtx, context);
+ if (isReference) {
+ for (Map.Entry<Object, Object> entry : newVal.entrySet()) {
+ String key = entry.getKey().toString();
+ AtlasEdge existingEdge = getEdgeIfExists(mapType, currentMap, key);
- if (isReference) {
- AtlasEdge edge = (AtlasEdge) newEntry;
- edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key);
- } else {
- ctx.getReferringVertex().setProperty(propertyName, newEntry);
- }
+ AttributeMutationContext mapCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), attribute, entry.getValue(),
+ propertyName, mapType.getValueType(), existingEdge);
+ // Add/Update/Remove property value
+ Object newEntry = mapCollectionElementsToVertex(mapCtx, context);
- newMap.put(key, newEntry);
+ if (newEntry instanceof AtlasEdge) {
+ AtlasEdge edge = (AtlasEdge) newEntry;
- // If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
- // update the inverse reference value.
- if (isReference && newEntry instanceof AtlasEdge && inverseRefAttribute != null) {
- AtlasEdge newEdge = (AtlasEdge) newEntry;
+ edge.setProperty(ATTRIBUTE_KEY_PROPERTY_KEY, key);
- addInverseReference(context, inverseRefAttribute, newEdge, getRelationshipAttributes(ctx.getValue()));
- }
- }
- }
+ // If value type indicates this attribute is a reference, and the attribute has an inverse reference attribute,
+ // update the inverse reference value.
+ AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
- Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), ctx.getVertexProperty(), currentMap, newMap);
+ if (inverseRefAttribute != null) {
+ addInverseReference(context, inverseRefAttribute, edge, getRelationshipAttributes(ctx.getValue()));
+ }
- for (Object newEntry : newMap.values()) {
- updateInConsistentOwnedMapVertices(ctx, mapType, newEntry);
- }
+ updateInConsistentOwnedMapVertices(ctx, mapType, newEntry);
- // for dereference on way out for primitive map types
- if (!isReference) {
- Set<String> newKeys = new LinkedHashSet<>(newMap.keySet());
+ newMap.put(key, newEntry);
+ }
+ }
- newKeys.addAll(finalMap.keySet());
+ Map<String, Object> finalMap = removeUnusedMapEntries(attribute, ctx.getReferringVertex(), currentMap, newMap);
+ newMap.putAll(finalMap);
+ } else {
+ // primitive type map
+ ctx.getReferringVertex().setProperty(propertyName, new HashMap<>(newVal));
- setListProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(newKeys));
+ newVal.forEach((key, value) -> newMap.put(key.toString(), value));
+ }
}
if (LOG.isDebugEnabled()) {
@@ -1084,60 +1074,23 @@ public class EntityGraphMapper {
return ret;
}
-
- public static Object getMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName) {
- if (isReference(elementType)) {
- return vertex.getProperty(vertexPropertyName, AtlasEdge.class);
- } else if (elementType instanceof AtlasArrayType) {
- return vertex.getProperty(vertexPropertyName, List.class);
- } else if (elementType instanceof AtlasMapType) {
- return vertex.getProperty(vertexPropertyName, Map.class);
- }
- else {
- return vertex.getProperty(vertexPropertyName, String.class).toString();
- }
- }
-
- private static void setMapValueProperty(AtlasType elementType, AtlasVertex vertex, String vertexPropertyName, Object value) {
- if (isReference(elementType)) {
- vertex.setPropertyFromElementId(vertexPropertyName, (AtlasEdge)value);
- }
- else {
- vertex.setProperty(vertexPropertyName, value);
- }
- }
-
- //Remove unused entries from map
- private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName,
- Map<String, Object> currentMap, Map<String, Object> newMap)
- throws AtlasBaseException {
-
- AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
- boolean isReference = isReference(mapType.getValueType());
+ //Remove unused entries for reference map
+ private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, Map<String, Object> currentMap,
+ Map<String, Object> newMap) throws AtlasBaseException {
Map<String, Object> additionalMap = new HashMap<>();
+ AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
for (String currentKey : currentMap.keySet()) {
- boolean shouldDeleteKey = !newMap.containsKey(currentKey);
-
- if (isReference) {
- //Delete the edge reference if its not part of new edges created/updated
- AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey);
+ //Delete the edge reference if its not part of new edges created/updated
+ AtlasEdge currentEdge = (AtlasEdge) currentMap.get(currentKey);
- if (!newMap.values().contains(currentEdge)) {
- boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex);
+ if (!newMap.values().contains(currentEdge)) {
+ boolean deleted = deleteHandler.deleteEdgeReference(currentEdge, mapType.getValueType().getTypeCategory(), attribute.isOwnedRef(), true, vertex);
- if (!deleted) {
- additionalMap.put(currentKey, currentEdge);
- shouldDeleteKey = false;
- }
+ if (!deleted) {
+ additionalMap.put(currentKey, currentEdge);
}
}
-
- if (!isReference && shouldDeleteKey) {
- String propertyNameForKey = getQualifiedNameForMapKey(propertyName, GraphHelper.encodePropertyKey(currentKey));
-
- GraphHelper.setProperty(vertex, propertyNameForKey, null);
- }
}
return additionalMap;
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 183a2f6..7ca51e4 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -619,7 +619,7 @@ public final class EntityGraphRetriever {
ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection);
break;
case ARRAY:
- ret = mapVertexToArray(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
+ ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute);
break;
case MAP:
ret = mapVertexToMap(entityVertex, vertexPropertyName, entityExtInfo, isOwnedAttribute, attribute);
@@ -660,7 +660,7 @@ public final class EntityGraphRetriever {
}
}
} else {
- ret = getPrimitiveMap(entityVertex, propertyName, mapValueType);
+ ret = getPrimitiveMap(entityVertex, propertyName);
}
if (MapUtils.isEmpty(ret)) {
@@ -670,12 +670,12 @@ public final class EntityGraphRetriever {
return ret;
}
- private List<Object> mapVertexToArray(AtlasVertex entityVertex, String propertyName, AtlasEntityExtInfo entityExtInfo,
+ private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
boolean isOwnedAttribute, AtlasAttribute attribute) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
- List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, propertyName, attribute);
+ List<Object> arrayElements = getArrayElementsProperty(arrayElementType, entityVertex, attribute);
if (CollectionUtils.isEmpty(arrayElements)) {
return null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
index 2363fca..e216749 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
@@ -37,9 +37,9 @@ public class HiveStocksTest extends MigrationBaseAsserts {
@Test
public void migrateStocks() throws AtlasBaseException, IOException {
- final int EXPECTED_TOTAL_COUNT = 188;
- final int EXPECTED_DB_COUNT = 1;
- final int EXPECTED_TABLE_COUNT = 1;
+ final int EXPECTED_TOTAL_COUNT = 188;
+ final int EXPECTED_DB_COUNT = 1;
+ final int EXPECTED_TABLE_COUNT = 1;
final int EXPECTED_COLUMN_COUNT = 7;
runFileImporter("stocks_db");
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
index e494728..922f7e0 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasComplexAttributesTest.java
@@ -30,18 +30,21 @@ import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.commons.lang.time.DateUtils;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE;
+import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_MAP;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR;
import static org.apache.atlas.TestUtilsV2.ENTITY_TYPE_WITH_COMPLEX_COLLECTION_ATTR_DELETE;
import static org.apache.atlas.TestUtilsV2.NAME;
@@ -54,18 +57,21 @@ import static org.testng.AssertJUnit.assertEquals;
public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
private AtlasEntityWithExtInfo complexCollectionAttrEntity;
private AtlasEntityWithExtInfo complexCollectionAttrEntityForDelete;
+ private AtlasEntityWithExtInfo mapAttributesEntity;
@BeforeClass
public void setUp() throws Exception {
super.setUp();
// create typeDefs
- AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes() };
+ AtlasTypesDef[] testTypesDefs = new AtlasTypesDef[] { TestUtilsV2.defineTypeWithComplexCollectionAttributes(),
+ TestUtilsV2.defineTypeWithMapAttributes() };
createTypesDef(testTypesDefs);
// create entity
complexCollectionAttrEntity = TestUtilsV2.createComplexCollectionAttrEntity();
complexCollectionAttrEntityForDelete = TestUtilsV2.createComplexCollectionAttrEntity();
+ mapAttributesEntity = TestUtilsV2.createMapAttrEntity();
}
@Test
@@ -78,6 +84,116 @@ public class AtlasComplexAttributesTest extends AtlasEntityTestBase {
validateEntity(complexCollectionAttrEntity, getEntityFromStore(entityCreated));
}
+ @Test
+ public void testPrimitiveMapAttributes() throws Exception {
+ init();
+
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(mapAttributesEntity), false);
+ AtlasEntityHeader entityCreated = response.getFirstCreatedEntityByTypeName(ENTITY_TYPE_MAP);
+ AtlasEntity entityFromStore = getEntityFromStore(entityCreated);
+ validateEntity(mapAttributesEntity, entityFromStore);
+
+ // Modify map of primitives
+ AtlasEntity attrEntity = getEntityFromStore(mapAttributesEntity.getEntity().getGuid());
+
+ Map<String, String> map1 = new HashMap<String, String>() {{ put("map1Key11", "value11");
+ put("map1Key22", "value22");
+ put("map1Key33", "value33"); }};
+
+ Map<String, Integer> map2 = new HashMap<String, Integer>() {{ put("map2Key11", 1100);
+ put("map2Key22", 2200);
+ put("map2Key33", 3300); }};
+
+ Map<String, Boolean> map3 = new HashMap<String, Boolean>() {{ put("map3Key11", true);
+ put("map3Key22", false);
+ put("map3Key33", true); }};
+
+ Map<String, Float> map4 = new HashMap<String, Float>() {{ put("map4Key11", 11.0f);
+ put("map4Key22", 22.0f);
+ put("map4Key33", 33.0f); }};
+
+ Map<String, Date> map5 = new HashMap<String, Date>() {{ put("map5Key11", DateUtils.addHours(new Date(), 1));
+ put("map5Key22", DateUtils.addHours(new Date(), 2));
+ put("map5Key33", DateUtils.addHours(new Date(), 3)); }};
+
+ updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
+
+ AtlasEntitiesWithExtInfo attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
+ response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
+ AtlasEntityHeader updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
+ AtlasEntity updatedFromStore = getEntityFromStore(updatedAttrEntity);
+ validateEntity(attrEntitiesInfo, updatedFromStore);
+
+ // Add new entry to map of primitives
+ map1.put("map1Key44", "value44");
+ map2.put("map2Key44", 4400);
+ map3.put("map3Key44", false);
+ map4.put("map4Key44", 44.0f);
+ map5.put("map5Key44", DateUtils.addHours(new Date(), 4));
+
+ updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
+
+ attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
+ response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
+ updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
+ updatedFromStore = getEntityFromStore(updatedAttrEntity);
+ validateEntity(attrEntitiesInfo, updatedFromStore);
+
+ // Remove an entry from map of primitives
+ map1.remove("map1Key11");
+ map2.remove("map2Key11");
+ map3.remove("map3Key11");
+ map4.remove("map4Key11");
+ map5.remove("map5Key11");
+
+ updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
+
+ attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
+ response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
+ updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
+ updatedFromStore = getEntityFromStore(updatedAttrEntity);
+ validateEntity(attrEntitiesInfo, updatedFromStore);
+
+ // Edit existing entry to map of primitives
+ map1.put("map1Key44", "value44-edit");
+ map2.put("map2Key44", 5555);
+ map3.put("map3Key44", true);
+ map4.put("map4Key44", 55.5f);
+ map5.put("map5Key44", DateUtils.addHours(new Date(), 5));
+
+ updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
+
+ attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
+ response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
+ updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
+ updatedFromStore = getEntityFromStore(updatedAttrEntity);
+ validateEntity(attrEntitiesInfo, updatedFromStore);
+
+ // clear primitive map entries
+ map1.clear();
+ map2.clear();
+ map3.clear();
+ map4.clear();
+ map5.clear();
+
+ updateEntityMapAttributes(attrEntity, map1, map2, map3, map4, map5);
+
+ attrEntitiesInfo = new AtlasEntitiesWithExtInfo(attrEntity);
+ response = entityStore.createOrUpdate(new AtlasEntityStream(attrEntitiesInfo), false);
+ updatedAttrEntity = response.getFirstUpdatedEntityByTypeName(ENTITY_TYPE_MAP);
+ updatedFromStore = getEntityFromStore(updatedAttrEntity);
+ validateEntity(attrEntitiesInfo, updatedFromStore);
+ }
+
+ private void updateEntityMapAttributes(AtlasEntity attrEntity, Map<String, String> map1, Map<String, Integer> map2,
+ Map<String, Boolean> map3, Map<String, Float> map4, Map<String, Date> map5) {
+ attrEntity.setAttribute("mapAttr1", map1);
+ attrEntity.setAttribute("mapAttr2", map2);
+ attrEntity.setAttribute("mapAttr3", map3);
+ attrEntity.setAttribute("mapAttr4", map4);
+ attrEntity.setAttribute("mapAttr5", map5);
+ }
+
@Test(dependsOnMethods = "testCreateComplexAttributeEntity")
public void testStructArray() throws Exception {
init();
http://git-wip-us.apache.org/repos/asf/atlas/blob/cbca591d/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
index 56dc5bd..d810a11 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.impexp.ExportService;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;