You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2018/08/16 18:24:29 UTC
atlas git commit: ATLAS-2813: SoftRef implementation. Part - 2
Repository: atlas
Updated Branches:
refs/heads/branch-0.8 6641a17b1 -> 3021027c9
ATLAS-2813: SoftRef implementation. Part - 2
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/3021027c
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/3021027c
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/3021027c
Branch: refs/heads/branch-0.8
Commit: 3021027c999c56767e65f410492a378f0e5b3a1f
Parents: 6641a17
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Thu Aug 16 11:17:00 2018 -0700
Committer: Ashutosh Mestry <am...@hortonworks.com>
Committed: Thu Aug 16 11:17:00 2018 -0700
----------------------------------------------------------------------
.../graph/v1/AttributeMutationContext.java | 4 -
.../store/graph/v1/EntityGraphMapper.java | 30 ++++---
.../store/graph/v1/EntityGraphRetriever.java | 35 +++++---
.../store/graph/v1/SoftReferenceTest.java | 84 ++++++++++++++++++--
4 files changed, 125 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java
index b32c092..b6d82dd 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AttributeMutationContext.java
@@ -126,10 +126,6 @@ public class AttributeMutationContext {
return value;
}
- public void setValue(Object value) {
- this.value = value;
- }
-
public String getVertexProperty() { return vertexProperty; }
public AtlasVertex getReferringVertex() { return referringVertex; }
http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
index 3811919..fcdd379 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphMapper.java
@@ -67,6 +67,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.string;
@Component
public class EntityGraphMapper {
private static final Logger LOG = LoggerFactory.getLogger(EntityGraphMapper.class);
+ private static final String SOFT_REF_FORMAT = "%s:%s";
private final GraphHelper graphHelper = GraphHelper.getInstance();
private final AtlasGraph graph;
@@ -342,16 +343,24 @@ public class EntityGraphMapper {
}
private Object mapSoftRefValue(AttributeMutationContext ctx, EntityMutationContext context) {
- if(ctx.getValue() == null || !(ctx.getValue() instanceof AtlasObjectId)) {
+ if(ctx.getValue() != null && !(ctx.getValue() instanceof AtlasObjectId)) {
+ LOG.warn("mapSoftRefValue: Was expecting AtlasObjectId, but found: {}", ctx.getValue().getClass());
return null;
}
- AtlasObjectId objectId = (AtlasObjectId) ctx.getValue();
- String resolvedGuid = context.getGuidAssignments().get(objectId.getGuid());
- String softRefValue = String.format("%s:%s", objectId.getTypeName(), resolvedGuid);
+ String softRefValue = null;
+ if(ctx.getValue() != null) {
+ AtlasObjectId objectId = (AtlasObjectId) ctx.getValue();
+ String resolvedGuid = AtlasTypeUtil.isUnAssignedGuid(objectId.getGuid())
+ ? context.getGuidAssignments().get(objectId.getGuid())
+ : objectId.getGuid();
- ctx.setValue(softRefValue);
- return mapPrimitiveValue(ctx);
+ softRefValue = String.format(SOFT_REF_FORMAT, objectId.getTypeName(), resolvedGuid);
+ }
+
+ AtlasGraphUtilsV1.setProperty(ctx.getReferringVertex(), ctx.getVertexProperty(), softRefValue);
+
+ return softRefValue;
}
private void addInverseReference(AttributeMutationContext ctx, AtlasAttribute inverseAttribute, AtlasEdge edge) throws AtlasBaseException {
@@ -588,8 +597,8 @@ public class EntityGraphMapper {
AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType();
AtlasType elementType = arrType.getElementType();
List<Object> currentElements = getArrayElementsProperty(elementType, ctx.getReferringVertex(), ctx.getVertexProperty());
- boolean isReference = AtlasGraphUtilsV1.isReference(elementType);
- boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced();
+ boolean isReference = AtlasGraphUtilsV1.isReference(elementType);
+ boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced();
AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute();
Cardinality cardinality = attribute.getAttributeDef().getCardinality();
List<Object> newElementsCreated = new ArrayList<>();
@@ -614,7 +623,7 @@ public class EntityGraphMapper {
}
}
- if (isReference) {
+ if (isReference && !isSoftReference) {
List<AtlasEdge> additionalEdges = removeUnusedArrayEntries(attribute, (List) currentElements, (List) newElementsCreated);
newElementsCreated.addAll(additionalEdges);
}
@@ -777,13 +786,14 @@ public class EntityGraphMapper {
private Map<String, Object> removeUnusedMapEntries(AtlasAttribute attribute, AtlasVertex vertex, String propertyName,
Map<String, Object> currentMap, Map<String, Object> newMap)
throws AtlasException, AtlasBaseException {
+ boolean isSoftRef = attribute.getAttributeDef().isSoftReferenced();
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
Map<String, Object> additionalMap = new HashMap<>();
for (String currentKey : currentMap.keySet()) {
boolean shouldDeleteKey = !newMap.containsKey(currentKey);
- if (AtlasGraphUtilsV1.isReference(mapType.getValueType())) {
+ if (AtlasGraphUtilsV1.isReference(mapType.getValueType()) && !isSoftRef) {
//Delete the edge reference if its not part of new edges created/updated
AtlasEdge currentEdge = (AtlasEdge)currentMap.get(currentKey);
http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 81b553e..f99b4a9 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -478,7 +478,7 @@ public final class EntityGraphRetriever {
return ret;
}
- private Object mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) {
+ private Map<String, AtlasObjectId> mapVertexToMapForSoftRef(AtlasVertex entityVertex, String propertyName) {
List mapKeys = entityVertex.getListProperty(propertyName);
if (CollectionUtils.isEmpty(mapKeys)) {
return null;
@@ -488,12 +488,12 @@ public final class EntityGraphRetriever {
LOG.debug("Mapping map attribute {} for vertex {}", propertyName, entityVertex);
}
- Map<String, Object> ret = new HashMap<>(mapKeys.size());
+ Map<String, AtlasObjectId> ret = new HashMap<>(mapKeys.size());
for (Object mapKey : mapKeys) {
final String keyPropertyName = String.format(MAP_VALUE_FORMAT, propertyName, mapKey);
- Object mapValue = mapVertexToObjectIdForSoftRef(entityVertex, keyPropertyName);
+ AtlasObjectId mapValue = mapVertexToObjectIdForSoftRef(entityVertex, keyPropertyName);
if (mapValue != null) {
ret.put((String) mapKey, mapValue);
}
@@ -502,29 +502,46 @@ public final class EntityGraphRetriever {
return ret;
}
- private Object mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) {
- List<AtlasObjectId> objectIds = new ArrayList<>();
+ private List<AtlasObjectId> mapVertexToArrayForSoftRef(AtlasVertex entityVertex, String propertyName) {
List list = entityVertex.getListProperty(propertyName);
+ if (CollectionUtils.isEmpty(list)) {
+ return null;
+ }
+
+ List<AtlasObjectId> objectIds = new ArrayList<>();
for (Object o : list) {
- if(!(o instanceof String)) {
+ if (!(o instanceof String)) {
continue;
}
AtlasObjectId objectId = getAtlasObjectIdFromSoftRefFormat((String) o);
+ if(objectId == null) {
+ continue;
+ }
+
objectIds.add(objectId);
}
return objectIds;
}
- private Object mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) {
- Object rawValue = GraphHelper.getSingleValuedProperty(entityVertex, vertexPropertyName, String.class);
- return getAtlasObjectIdFromSoftRefFormat((String) rawValue);
+ private AtlasObjectId mapVertexToObjectIdForSoftRef(AtlasVertex entityVertex, String vertexPropertyName) {
+ String rawValue = GraphHelper.getSingleValuedProperty(entityVertex, vertexPropertyName, String.class);
+ if(StringUtils.isEmpty(rawValue)) {
+ return null;
+ }
+
+ return getAtlasObjectIdFromSoftRefFormat(rawValue);
}
private AtlasObjectId getAtlasObjectIdFromSoftRefFormat(String rawValue) {
+ if(StringUtils.isEmpty(rawValue)) {
+ return null;
+ }
+
String[] objectIdParts = StringUtils.split(rawValue, SOFT_REFERENCE_FORMAT_SEPERATOR);
if(objectIdParts.length < 2) {
+ LOG.warn("Expecting value to be formatted for softRef. Instead found: {}", rawValue);
return null;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/3021027c/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java
index 665f17a..95b99f9 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/SoftReferenceTest.java
@@ -30,6 +30,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.utils.TestResourceFileUtils;
@@ -69,6 +70,8 @@ public class SoftReferenceTest {
private AtlasEntityStore entityStore;
private AtlasType dbType;
+ private String dbGuid;
+ private String storageGuid;
@Test
public void typeCreationFromFile() throws IOException, AtlasBaseException {
@@ -92,16 +95,87 @@ public class SoftReferenceTest {
@Test(dependsOnMethods = "typeCreationFromFile")
public void entityCreationUsingSoftRef() throws IOException, AtlasBaseException {
+ final int EXPECTED_ENTITY_COUNT = 6;
AtlasEntity.AtlasEntityWithExtInfo dbEntity = AtlasType.fromJson(
TestResourceFileUtils.getJson(RDBMS_DB_FILE), AtlasEntity.AtlasEntityWithExtInfo.class);
EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(dbEntity), false);
+
assertNotNull(response);
- assertTrue(response.getCreatedEntities().size() == 6);
+ assertTrue(response.getCreatedEntities().size() == EXPECTED_ENTITY_COUNT);
assertGraphStructure(response.getCreatedEntities().get(0).getGuid(),
response.getCreatedEntities().get(1).getGuid(), RDBMS_SD_PROPERTY);
+
+ dbGuid = response.getCreatedEntities().get(0).getGuid();
+ storageGuid = response.getCreatedEntities().get(1).getGuid();
+ }
+
+ @Test(dependsOnMethods = "entityCreationUsingSoftRef")
+ public void deletetingCollections() throws AtlasBaseException {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid);
+
+ assertNotNull(entityWithExtInfo);
+ List list = (List)entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_TABLES_PROPERTY);
+ list.remove(1);
+
+ Map map = (Map) entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_REGIONS_PROPERTY);
+ map.remove("east");
+
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true);
+ assertNotNull(response);
+ assertTrue(response.getPartialUpdatedEntities().size() > 0);
+ assertAttribute(dbGuid, storageGuid, 1, 1);
+ }
+
+ @Test(dependsOnMethods = "deletetingCollections")
+ public void addingCollections() throws AtlasBaseException {
+ AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid);
+
+ assertNotNull(entityWithExtInfo);
+ addNewTables(entityWithExtInfo);
+ addNewRegions(entityWithExtInfo);
+
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(entityWithExtInfo), true);
+ assertNotNull(response);
+ assertTrue(response.getPartialUpdatedEntities().size() > 0);
+ assertAttribute(dbGuid, storageGuid, 3, 3);
}
+ private void addNewRegions(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException {
+ Map map = (Map) entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_REGIONS_PROPERTY);
+
+ AtlasEntity region1 = getDefaultTableEntity("r1");
+ AtlasEntity region2 = getDefaultTableEntity("r2");
+
+ map.put("north", new AtlasObjectId(region1.getGuid(), region1.getTypeName()));
+ map.put("south", new AtlasObjectId(region2.getGuid(), region2.getTypeName()));
+
+ entityWithExtInfo.addReferredEntity(region1);
+ entityWithExtInfo.addReferredEntity(region2);
+ }
+
+ private void addNewTables(AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo) throws AtlasBaseException {
+ List list = (List)entityWithExtInfo.getEntity().getAttribute(RDBMS_DB_TABLES_PROPERTY);
+ AtlasEntity table1 = getDefaultTableEntity("newTable-1");
+ AtlasEntity table2 = getDefaultTableEntity("newTable-2");
+
+ entityWithExtInfo.addReferredEntity(table1);
+ entityWithExtInfo.addReferredEntity(table2);
+
+ list.add(new AtlasObjectId(table1.getGuid(), table1.getTypeName()));
+ list.add(new AtlasObjectId(table2.getGuid(), table2.getTypeName()));
+ }
+
+ private AtlasEntity getDefaultTableEntity(String name) throws AtlasBaseException {
+ AtlasEntityType type = (AtlasEntityType) typeRegistry.getType(TYPE_RDBMS_TABLES);
+
+ AtlasEntity ret = type.createDefaultValue();
+ ret.setAttribute("name", name);
+
+ return ret;
+ }
+
+
private void assertGraphStructure(String dbGuid, String storageGuid, String propertyName) throws AtlasBaseException {
AtlasVertex vertex = AtlasGraphUtilsV1.findByGuid(dbGuid);
Iterator<AtlasEdge> edgesOut = vertex.getEdges(AtlasEdgeDirection.OUT).iterator();
@@ -110,13 +184,13 @@ public class SoftReferenceTest {
String sd = AtlasGraphUtilsV1.getProperty(vertex, propertyName, String.class);
assertNotNull(sd);
- assertAttribute(dbGuid, storageGuid);
+ assertAttribute(dbGuid, storageGuid, 2, 2);
assertFalse(edgesOut.hasNext());
assertFalse(edgesIn.hasNext());
assertNotNull(vertex);
}
- private void assertAttribute(String dbGuid, String storageGuid) throws AtlasBaseException {
+ private void assertAttribute(String dbGuid, String storageGuid, int expectedTableCount, int expectedRegionCount) throws AtlasBaseException {
AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityStore.getById(dbGuid);
AtlasEntity entity = entityWithExtInfo.getEntity();
@@ -125,8 +199,8 @@ public class SoftReferenceTest {
assertEquals(((AtlasObjectId) val).getTypeName(), TYPE_RDBMS_STORAGE);
assertEquals(((AtlasObjectId) val).getGuid(), storageGuid);
assertNotNull(entity.getAttribute(RDBMS_DB_TABLES_PROPERTY));
- assertEquals(((List) entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)).size(), 2);
+ assertEquals(((List) entity.getAttribute(RDBMS_DB_TABLES_PROPERTY)).size(), expectedTableCount);
assertNotNull(entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY));
- assertEquals(((Map) entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)).size(), 2);
+ assertEquals(((Map) entity.getAttribute(RDBMS_DB_REGIONS_PROPERTY)).size(), expectedRegionCount);
}
}