You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/01/23 18:39:48 UTC
[atlas] branch branch-1.0 updated: ATLAS-3034: Perf enhancement to
avoid unnecessary lookup when creating new relationships
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-1.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-1.0 by this push:
new 7c471dc ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new relationships
7c471dc is described below
commit 7c471dc313daf7c9e30e77024ac3e2c94a9a5403
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Wed Jan 23 10:35:35 2019 -0800
ATLAS-3034: Perf enhancement to avoid unnecessary lookup when creating new relationships
(cherry picked from commit a9aa5b0e1b0a7c2867aef16358e3c3c1977bb140)
---
.../store/graph/AtlasRelationshipStore.java | 4 +
.../store/graph/v2/AtlasRelationshipStoreV2.java | 124 +++++++++++----------
.../store/graph/v2/EntityGraphMapper.java | 32 ++----
3 files changed, 77 insertions(+), 83 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 4468951..53e29e1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -59,6 +59,10 @@ public interface AtlasRelationshipStore {
AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException;
+ AtlasEdge getRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship);
+
+ AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException;
+
/**
* Retrieve a relationship if it exists or creates a new relationship instance.
* @param relationship relationship instance definition
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index 6bc36b3..2791ef3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -81,9 +81,8 @@ import static org.apache.atlas.repository.graph.GraphHelper.getBlockedClassifica
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
-import static org.apache.atlas.repository.graph.GraphHelper.getOutGoingEdgesByLabel;
+import static org.apache.atlas.repository.graph.GraphHelper.getIncomingEdgesByLabel;
import static org.apache.atlas.repository.graph.GraphHelper.getPropagateTags;
-import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getIdFromVertex;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getState;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getTypeName;
@@ -117,8 +116,6 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
- validateRelationship(end1Vertex, end2Vertex, relationship.getTypeName(), relationship.getAttributes());
-
AtlasEdge edge = createRelationship(end1Vertex, end2Vertex, relationship);
AtlasRelationship ret = edge != null ? entityRetriever.mapEdgeToAtlasRelationship(edge) : null;
@@ -289,36 +286,37 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
@Override
public AtlasEdge getOrCreate(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
- AtlasEdge ret = getRelationshipEdge(end1Vertex, end2Vertex, relationship.getTypeName());
+ AtlasEdge ret = getRelationship(end1Vertex, end2Vertex, relationship);
if (ret == null) {
- validateRelationship(end1Vertex, end2Vertex, relationship.getTypeName(), relationship.getAttributes());
-
- ret = createRelationship(end1Vertex, end2Vertex, relationship);
+ ret = createRelationship(end1Vertex, end2Vertex, relationship, false);
}
return ret;
}
@Override
+ public AtlasEdge getRelationship(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
+ String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship.getTypeName());
+
+ return getRelationshipEdge(fromVertex, toVertex, relationshipLabel);
+ }
+
+ @Override
public AtlasRelationship getOrCreate(AtlasRelationship relationship) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getOrCreate({})", relationship);
}
- validateRelationship(relationship);
-
AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1());
AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2());
AtlasRelationship ret = null;
// check if relationship exists
- AtlasEdge relationshipEdge = getRelationshipEdge(end1Vertex, end2Vertex, relationship.getTypeName());
+ AtlasEdge relationshipEdge = getRelationship(end1Vertex, end2Vertex, relationship);
if (relationshipEdge == null) {
- validateRelationship(relationship);
-
- relationshipEdge = createRelationship(end1Vertex, end2Vertex, relationship);
+ relationshipEdge = createRelationship(end1Vertex, end2Vertex, relationship, false);
}
if (relationshipEdge != null){
@@ -332,15 +330,26 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
return ret;
}
- private AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
- AtlasEdge ret = null;
+ @Override
+ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship) throws AtlasBaseException {
+ return createRelationship(end1Vertex, end2Vertex, relationship, true);
+ }
+
+ public AtlasEdge createRelationship(AtlasVertex end1Vertex, AtlasVertex end2Vertex, AtlasRelationship relationship, boolean existingRelationshipCheck) throws AtlasBaseException {
+ AtlasEdge ret;
try {
- ret = getRelationshipEdge(end1Vertex, end2Vertex, relationship.getTypeName());
+ validateRelationship(end1Vertex, end2Vertex, relationship.getTypeName(), relationship.getAttributes());
- if (ret != null) {
- throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(),
- AtlasGraphUtilsV2.getIdFromVertex(end1Vertex), AtlasGraphUtilsV2.getIdFromVertex(end2Vertex));
+ String relationshipLabel = getRelationshipEdgeLabel(end1Vertex, end2Vertex, relationship.getTypeName());
+
+ if (existingRelationshipCheck) {
+ ret = getRelationshipEdge(end1Vertex, end2Vertex, relationshipLabel);
+
+ if (ret != null) {
+ throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_EXISTS, relationship.getTypeName(),
+ AtlasGraphUtilsV2.getIdFromVertex(end1Vertex), AtlasGraphUtilsV2.getIdFromVertex(end2Vertex));
+ }
}
AtlasRelationshipType relationType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
@@ -349,10 +358,40 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasEntityHeader end1Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end1Vertex);
AtlasEntityHeader end2Entity = entityRetriever.toAtlasEntityHeaderWithClassifications(end2Vertex);
- AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD, relationship.getTypeName(), end1Entity, end2Entity));
+ AtlasAuthorizationUtils.verifyAccess(new AtlasRelationshipAccessRequest(typeRegistry, AtlasPrivilege.RELATIONSHIP_ADD,
+ relationship.getTypeName(), end1Entity, end2Entity));
}
- ret = createRelationshipEdge(end1Vertex, end2Vertex, relationship);
+ if (existingRelationshipCheck) {
+ ret = graphHelper.getOrCreateEdge(end1Vertex, end2Vertex, relationshipLabel);
+
+ } else {
+ ret = graphHelper.addEdge(end1Vertex, end2Vertex, relationshipLabel);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Created relationship edge from [{}] --> [{}] using edge label: [{}]", getTypeName(end1Vertex), getTypeName(end2Vertex), relationshipLabel);
+ }
+ }
+
+ // map additional properties to relationship edge
+ if (ret != null) {
+ // Accept a valid (assigned) guid from the supplied relationship, or generate one.
+ String relationshipGuid = relationship.getGuid();
+ PropagateTags tagPropagation = getRelationshipTagPropagation(end1Vertex, end2Vertex, relationship);
+ final String guid = AtlasTypeUtil.isAssignedGuid(relationshipGuid) ? relationshipGuid : UUID.randomUUID().toString();
+
+ AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
+ AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid);
+ AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
+ AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType());
+ AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
+
+ // blocked propagated classifications
+ handleBlockedClassifications(ret, relationship.getBlockedPropagatedClassifications());
+
+ // propagate tags
+ deleteDelegate.getHandler().addTagPropagation(ret, tagPropagation);
+ }
if (MapUtils.isNotEmpty(relationType.getAllAttributes())) {
for (AtlasAttribute attr : relationType.getAllAttributes().values()) {
@@ -691,10 +730,9 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
type.getNormalizedValue(relationship);
}
- public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipType) {
- String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationshipType);
- Iterator<AtlasEdge> edgesIterator = getOutGoingEdgesByLabel(fromVertex, relationshipLabel);
- AtlasEdge ret = null;
+ public AtlasEdge getRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipLabel) {
+ AtlasEdge ret = null;
+ Iterator<AtlasEdge> edgesIterator = getIncomingEdgesByLabel(toVertex, relationshipLabel);
while (edgesIterator != null && edgesIterator.hasNext()) {
AtlasEdge edge = edgesIterator.next();
@@ -702,8 +740,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
if (edge != null) {
Status status = graphHelper.getStatus(edge);
- if ((status == null || status == ACTIVE) &&
- StringUtils.equals(getIdFromVertex(edge.getInVertex()), getIdFromVertex(toVertex))) {
+ if ((status == null || status == ACTIVE) && edge.getOutVertex().equals(fromVertex)) {
ret = edge;
break;
}
@@ -733,37 +770,6 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
return ret;
}
- private AtlasEdge createRelationshipEdge(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) throws RepositoryException, AtlasBaseException {
- String relationshipLabel = getRelationshipEdgeLabel(fromVertex, toVertex, relationship.getTypeName());
- PropagateTags tagPropagation = getRelationshipTagPropagation(fromVertex, toVertex, relationship);
- AtlasEdge ret = graphHelper.getOrCreateEdge(fromVertex, toVertex, relationshipLabel);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Created relationship edge from [{}] --> [{}] using edge label: [{}]", getTypeName(fromVertex), getTypeName(toVertex), relationshipLabel);
- }
-
- // map additional properties to relationship edge
- if (ret != null) {
- // Accept a valid (assigned) guid from the supplied relationship, or generate one.
- String relationshipGuid = relationship.getGuid();
- final String guid = AtlasTypeUtil.isAssignedGuid(relationshipGuid) ? relationshipGuid : UUID.randomUUID().toString();
-
- AtlasGraphUtilsV2.setEncodedProperty(ret, ENTITY_TYPE_PROPERTY_KEY, relationship.getTypeName());
- AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, guid);
- AtlasGraphUtilsV2.setEncodedProperty(ret, VERSION_PROPERTY_KEY, getRelationshipVersion(relationship));
- AtlasGraphUtilsV2.setEncodedProperty(ret, PROVENANCE_TYPE_KEY, relationship.getProvenanceType());
- AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, tagPropagation.name());
-
- // blocked propagated classifications
- handleBlockedClassifications(ret, relationship.getBlockedPropagatedClassifications());
-
- // propagate tags
- deleteDelegate.getHandler().addTagPropagation(ret, tagPropagation);
- }
-
- return ret;
- }
-
private PropagateTags getRelationshipTagPropagation(AtlasVertex fromVertex, AtlasVertex toVertex, AtlasRelationship relationship) {
AtlasRelationshipType relationshipType = typeRegistry.getRelationshipTypeByName(relationship.getTypeName());
AtlasRelationshipEndDef endDef1 = relationshipType.getRelationshipDef().getEndDef1();
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 98734cc..afa471f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -819,7 +819,6 @@ public class EntityGraphMapper {
AtlasType type = typeRegistry.getType(AtlasGraphUtilsV2.getTypeName(entityVertex));
AtlasRelationshipEdgeDirection edgeDirection = ctx.getAttribute().getRelationshipEdgeDirection();
- String edgeLabel = ctx.getAttribute().getRelationshipEdgeLabel();
if (type instanceof AtlasEntityType) {
AtlasEntityType entityType = (AtlasEntityType) type;
@@ -844,10 +843,17 @@ public class EntityGraphMapper {
fromVertex = entityVertex;
toVertex = attributeVertex;
}
- boolean relationshipExists = isRelationshipExists(fromVertex, toVertex, edgeLabel);
ret = getOrCreateRelationship(fromVertex, toVertex, relationshipName, relationshipAttributes);
+ boolean isCreated = GraphHelper.getCreatedTime(ret) == RequestContext.get().getRequestTime();
+
+ if (isCreated) {
+ // if relationship did not exist before and new relationship was created
+ // record entity update on both relationship vertices
+ recordEntityUpdate(attributeVertex);
+ }
+
// for import use the relationship guid provided
if (RequestContext.get().isImportInProgress()) {
String relationshipGuid = getRelationshipGuid(ctx.getValue());
@@ -856,12 +862,6 @@ public class EntityGraphMapper {
AtlasGraphUtilsV2.setEncodedProperty(ret, RELATIONSHIP_GUID_PROPERTY_KEY, relationshipGuid);
}
}
-
- // if relationship did not exist before and new relationship was created
- // record entity update on both relationship vertices
- if (!relationshipExists) {
- recordEntityUpdate(attributeVertex);
- }
}
} else {
// use legacy way to create/update edges
@@ -1851,22 +1851,6 @@ public class EntityGraphMapper {
return relationshipStore.getOrCreate(end1Vertex, end2Vertex, new AtlasRelationship(relationshipName, relationshipAttributes));
}
- private boolean isRelationshipExists(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
- boolean ret = false;
- Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(fromVertex, edgeLabel);
-
- while (edges != null && edges.hasNext()) {
- AtlasEdge edge = edges.next();
- AtlasVertex inVertex = edge.getInVertex();
-
- if (inVertex != null && StringUtils.equals(getIdFromVertex(inVertex), getIdFromVertex(toVertex))) {
- ret = true;
- }
- }
-
- return ret;
- }
-
private void recordEntityUpdate(AtlasVertex vertex) throws AtlasBaseException {
RequestContext req = RequestContext.get();