You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/06/17 07:07:44 UTC
[atlas] branch master updated: ATLAS-3036: Improve FullTextMapper
performance during entity retrieval
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 66c2964 ATLAS-3036: Improve FullTextMapper performance during entity retrieval
66c2964 is described below
commit 66c2964e05f8421d8dfb395f1f0cf61d1f33a8e4
Author: Sarath Subramanian <ss...@hortonworks.com>
AuthorDate: Sun Jun 16 12:34:21 2019 -0700
ATLAS-3036: Improve FullTextMapper performance during entity retrieval
(cherry picked from commit a08a5a39a83fcfee0965471a3a1b1c29279feea3)
(cherry picked from commit 7ff396af047c8ff829dbe5b8a4094177fb3e9686)
(cherry picked from commit 2fdbc851db06f7ebe96cc3fde4d8507d088e7ec6)
---
.../atlas/repository/graph/FullTextMapperV2.java | 8 +-
.../store/graph/v2/EntityGraphRetriever.java | 92 ++++++++++++++++------
2 files changed, 76 insertions(+), 24 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 7de7b29..caa6604 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -131,7 +131,7 @@ public class FullTextMapperV2 {
entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
entityExtInfo = entityWithExtInfo;
} else {
- entity = getAndCacheEntity(guid);
+ entity = getAndCacheEntity(guid, false);
entityExtInfo = null;
}
@@ -272,11 +272,15 @@ public class FullTextMapperV2 {
}
private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
+ return getAndCacheEntity(guid, true);
+ }
+
+ private AtlasEntity getAndCacheEntity(String guid, boolean includeReferences) throws AtlasBaseException {
RequestContext context = RequestContext.get();
AtlasEntity entity = context.getEntity(guid);
if (entity == null) {
- entity = entityGraphRetriever.toAtlasEntity(guid);
+ entity = entityGraphRetriever.toAtlasEntity(guid, includeReferences);
if (entity != null) {
context.cache(entity);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index fa7c5a1..02638c3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -44,6 +44,7 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasMapType;
import org.apache.atlas.type.AtlasRelationshipType;
@@ -155,6 +156,10 @@ public class EntityGraphRetriever {
this.ignoreRelationshipAttr = ignoreRelationshipAttr;
}
+ public AtlasEntity toAtlasEntity(String guid, boolean includeReferences) throws AtlasBaseException {
+ return mapVertexToAtlasEntity(getEntityVertex(guid), null, false, includeReferences);
+ }
+
public AtlasEntity toAtlasEntity(String guid) throws AtlasBaseException {
return toAtlasEntity(getEntityVertex(guid));
}
@@ -429,6 +434,10 @@ public class EntityGraphRetriever {
}
private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
+ return mapVertexToAtlasEntity(entityVertex, entityExtInfo, isMinExtInfo, true);
+ }
+
+ private AtlasEntity mapVertexToAtlasEntity(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
String guid = GraphHelper.getGuid(entityVertex);
AtlasEntity entity = entityExtInfo != null ? entityExtInfo.getEntity(guid) : null;
@@ -445,7 +454,7 @@ public class EntityGraphRetriever {
mapSystemAttributes(entityVertex, entity);
- mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
+ mapAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo, includeReferences);
if (!ignoreRelationshipAttr) { // only map when really needed
mapRelationshipAttributes(entityVertex, entity, entityExtInfo, isMinExtInfo);
@@ -595,6 +604,10 @@ public class EntityGraphRetriever {
}
private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
+ mapAttributes(entityVertex, struct, entityExtInfo, isMinExtInfo, true);
+ }
+
+ private void mapAttributes(AtlasVertex entityVertex, AtlasStruct struct, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasType objType = typeRegistry.getType(struct.getTypeName());
if (!(objType instanceof AtlasStructType)) {
@@ -604,7 +617,7 @@ public class EntityGraphRetriever {
AtlasStructType structType = (AtlasStructType) objType;
for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
- Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ Object attrValue = mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, includeReferences);
struct.setAttribute(attribute.getName(), attrValue);
}
@@ -721,6 +734,10 @@ public class EntityGraphRetriever {
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo) throws AtlasBaseException {
+ return mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, true);
+ }
+
+ private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null;
AtlasType attrType = attribute.getAttributeType();
String edgeLabel = attribute.getRelationshipEdgeLabel();
@@ -742,25 +759,56 @@ public class EntityGraphRetriever {
ret = mapVertexToStruct(entityVertex, edgeLabel, null, entityExtInfo, isMinExtInfo);
break;
case OBJECT_ID_TYPE:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ if (includeReferences) {
+ ret = attribute.getAttributeDef().isSoftReferenced() ? mapVertexToObjectIdForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo) :
+ mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
} else {
- ret = mapVertexToObjectId(entityVertex, edgeLabel, null, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
- }
+ ret = null;
+ }
break;
- case ARRAY:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ case ARRAY: {
+ final boolean skipAttribute;
+
+ if (!includeReferences) {
+ AtlasType elementType = ((AtlasArrayType) attrType).getElementType();
+
+ skipAttribute = (elementType instanceof AtlasObjectIdType || elementType instanceof AtlasEntityType);
+ } else {
+ skipAttribute = false;
+ }
+
+ if (skipAttribute) {
+ ret = null;
} else {
- ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
- }
+ if (attribute.getAttributeDef().isSoftReferenced()) {
+ ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ } else {
+ ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
+ }
+ }
+ }
break;
- case MAP:
- if(attribute.getAttributeDef().isSoftReferenced()) {
- ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ case MAP: {
+ final boolean skipAttribute;
+
+ if (!includeReferences) {
+ AtlasType valueType = ((AtlasMapType) attrType).getValueType();
+
+ skipAttribute = (valueType instanceof AtlasObjectIdType || valueType instanceof AtlasEntityType);
} else {
- ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo);
- }
+ skipAttribute = false;
+ }
+
+ if (skipAttribute) {
+ ret = null;
+ } else {
+ if (attribute.getAttributeDef().isSoftReferenced()) {
+ ret = mapVertexToMapForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
+ } else {
+ ret = mapVertexToMap(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
+ }
+ }
+ }
break;
case CLASSIFICATION:
// do nothing
@@ -848,7 +896,7 @@ public class EntityGraphRetriever {
}
private Map<String, Object> mapVertexToMap(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
- boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
+ boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Map<String, Object> ret = null;
AtlasMapType mapType = (AtlasMapType) attribute.getAttributeType();
@@ -868,7 +916,7 @@ public class EntityGraphRetriever {
String mapKey = entry.getKey();
Object keyValue = entry.getValue();
Object mapValue = mapVertexToCollectionEntry(entityVertex, mapValueType, keyValue, attribute.getRelationshipEdgeLabel(),
- entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo);
+ entityExtInfo, isOwnedAttribute, attribute.getRelationshipEdgeDirection(), isMinExtInfo, includeReferences);
if (mapValue != null) {
ret.put(mapKey, mapValue);
}
@@ -882,7 +930,7 @@ public class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
- boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo) throws AtlasBaseException {
+ boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
@@ -909,7 +957,7 @@ public class EntityGraphRetriever {
}
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
- entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
+ entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo, includeReferences);
if (arrValue != null) {
arrValues.add(arrValue);
@@ -921,7 +969,7 @@ public class EntityGraphRetriever {
private Object mapVertexToCollectionEntry(AtlasVertex entityVertex, AtlasType arrayElement, Object value,
String edgeLabel, AtlasEntityExtInfo entityExtInfo, boolean isOwnedAttribute,
- AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo) throws AtlasBaseException {
+ AtlasRelationshipEdgeDirection edgeDirection, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
Object ret = null;
switch (arrayElement.getTypeCategory()) {
@@ -940,7 +988,7 @@ public class EntityGraphRetriever {
break;
case OBJECT_ID_TYPE:
- ret = mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo);
+ ret = includeReferences ? mapVertexToObjectId(entityVertex, edgeLabel, (AtlasEdge) value, entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo) : null;
break;
default: