You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/09/18 21:06:23 UTC
atlas git commit: ATLAS-2878: avoid retrieval of entiyWithExtInfo
when extInfo is not needed
Repository: atlas
Updated Branches:
refs/heads/master 48e522497 -> a064e0924
ATLAS-2878: avoid retrieval of entiyWithExtInfo when extInfo is not needed
(cherry picked from commit 8e7ecf72f32ef6ca282a314d85761742e229a48b)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a064e092
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a064e092
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a064e092
Branch: refs/heads/master
Commit: a064e09242a0b1259d7b1227913786d2fb98f695
Parents: 48e5224
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Sep 18 11:16:50 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Sep 18 11:35:24 2018 -0700
----------------------------------------------------------------------
.../repository/audit/EntityAuditListenerV2.java | 6 +--
.../converters/AtlasInstanceConverter.java | 30 ++++++++++---
.../repository/graph/FullTextMapperV2.java | 45 ++++++++++++++++----
.../graph/v2/AtlasEntityChangeNotifier.java | 9 ++--
.../store/graph/v2/EntityGraphMapper.java | 14 +++---
.../java/org/apache/atlas/RequestContext.java | 24 ++++++++---
6 files changed, 90 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
index 91e1f63..8ca8c9a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
@@ -180,8 +180,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
List<EntityAuditEventV2> events = new ArrayList<>();
for (AtlasRelatedObjectId relatedObjectId : entities) {
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid());
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid());
if (entity != null) {
events.add(createEvent(entity, TERM_ADD, "Added term: " + term.toAuditString()));
@@ -198,8 +197,7 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
List<EntityAuditEventV2> events = new ArrayList<>();
for (AtlasRelatedObjectId relatedObjectId : entities) {
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid());
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(relatedObjectId.getGuid());
if (entity != null) {
events.add(createEvent(entity, TERM_DELETE, "Deleted term: " + term.toAuditString()));
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
index b26b353..1f0cc86 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
@@ -28,6 +28,7 @@ import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
import org.apache.atlas.model.instance.EntityMutations.EntityOperation;
@@ -94,12 +95,12 @@ public class AtlasInstanceConverter {
}
public Referenceable getReferenceable(String guid) throws AtlasBaseException {
- AtlasEntity.AtlasEntityWithExtInfo entity = getAndCacheEntity(guid);
+ AtlasEntityWithExtInfo entity = getAndCacheEntityExtInfo(guid);
return getReferenceable(entity);
}
- public Referenceable getReferenceable(AtlasEntity.AtlasEntityWithExtInfo entity) throws AtlasBaseException {
+ public Referenceable getReferenceable(AtlasEntityWithExtInfo entity) throws AtlasBaseException {
AtlasFormatConverter.ConverterContext ctx = new AtlasFormatConverter.ConverterContext();
ctx.addEntity(entity.getEntity());
@@ -291,10 +292,29 @@ public class AtlasInstanceConverter {
return ret;
}
+ public AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
+ RequestContext context = RequestContext.get();
+ AtlasEntity entity = context.getEntity(guid);
- public AtlasEntity.AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException {
- RequestContext context = RequestContext.get();
- AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid);
+ if (entity == null) {
+ entity = entityGraphRetriever.toAtlasEntity(guid);
+
+ if (entity != null) {
+ context.cache(entity);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cache miss -> GUID = {}", guid);
+ }
+ }
+ }
+
+ return entity;
+ }
+
+
+ public AtlasEntityWithExtInfo getAndCacheEntityExtInfo(String guid) throws AtlasBaseException {
+ RequestContext context = RequestContext.get();
+ AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid);
if (entityWithExtInfo == null) {
entityWithExtInfo = entityGraphRetriever.toAtlasEntityWithExtInfo(guid);
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index b85dc9b..08ccd9c 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -74,8 +74,16 @@ public class FullTextMapperV2 {
* @throws AtlasBaseException
*/
public String getIndexTextForClassifications(String guid, List<AtlasClassification> classifications) throws AtlasBaseException {
- String ret = null;
- AtlasEntityWithExtInfo entityWithExtInfo = getAndCacheEntity(guid);
+ String ret = null;
+ final AtlasEntityWithExtInfo entityWithExtInfo;
+
+ if (followReferences) {
+ entityWithExtInfo = getAndCacheEntityWithExtInfo(guid);
+ } else {
+ AtlasEntity entity = getAndCacheEntity(guid);
+
+ entityWithExtInfo = entity != null ? new AtlasEntityWithExtInfo(entity) : null;
+ }
if (entityWithExtInfo != null) {
StringBuilder sb = new StringBuilder();
@@ -101,13 +109,13 @@ public class FullTextMapperV2 {
}
public String getIndexTextForEntity(String guid) throws AtlasBaseException {
- String ret = null;
- AtlasEntityWithExtInfo entity = getAndCacheEntity(guid);
+ String ret = null;
+ AtlasEntity entity = getAndCacheEntity(guid);
if (entity != null) {
StringBuilder sb = new StringBuilder();
- map(entity.getEntity(), entity, sb, new HashSet<String>());
+ map(entity, null, sb, new HashSet<String>());
ret = sb.toString();
}
@@ -166,7 +174,7 @@ public class FullTextMapperV2 {
private void mapAttribute(Object value, AtlasEntityExtInfo entityExtInfo, StringBuilder sb, Set<String> processedGuids) throws AtlasBaseException {
if (value instanceof AtlasObjectId) {
- if (followReferences) {
+ if (followReferences && entityExtInfo != null) {
AtlasObjectId objectId = (AtlasObjectId) value;
AtlasEntity entity = entityExtInfo.getEntity(objectId.getGuid());
@@ -203,9 +211,28 @@ public class FullTextMapperV2 {
}
}
- private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException {
+ private AtlasEntity getAndCacheEntity(String guid) throws AtlasBaseException {
+ RequestContext context = RequestContext.get();
+ AtlasEntity entity = context.getEntity(guid);
+
+ if (entity == null) {
+ entity = entityGraphRetriever.toAtlasEntity(guid);
+
+ if (entity != null) {
+ context.cache(entity);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cache miss -> GUID = {}", guid);
+ }
+ }
+ }
+
+ return entity;
+ }
+
+ private AtlasEntityWithExtInfo getAndCacheEntityWithExtInfo(String guid) throws AtlasBaseException {
RequestContext context = RequestContext.get();
- AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid);
+ AtlasEntityWithExtInfo entityWithExtInfo = context.getEntityWithExtInfo(guid);
if (entityWithExtInfo == null) {
// Only map ownedRef and relationship attr when follow references is set to true
@@ -245,4 +272,4 @@ public class FullTextMapperV2 {
return ret;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index 9d8afdf..a8c3363 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -254,8 +254,7 @@ public class AtlasEntityChangeNotifier {
continue;
}
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid);
- AtlasEntity entity = entityWithExtInfo != null ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
if (entity == null) {
continue;
@@ -413,9 +412,7 @@ public class AtlasEntityChangeNotifier {
entity.setGuid(entityGuid);
} else {
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
-
- entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ entity = instanceConverter.getAndCacheEntity(entityGuid);
}
if (entity != null) {
@@ -497,4 +494,4 @@ public class AtlasEntityChangeNotifier {
doFullTextMapping(Collections.singletonList(entityHeader));
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index d1e95f0..fa69fdc 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -1422,8 +1422,7 @@ public class EntityGraphMapper {
for (AtlasVertex vertex : notificationVertices) {
String entityGuid = GraphHelper.getGuid(vertex);
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
List<AtlasClassification> addedClassifications = StringUtils.equals(entityGuid, guid) ? addClassifications : propagations.get(vertex);
if (CollectionUtils.isNotEmpty(addedClassifications)) {
@@ -1529,8 +1528,7 @@ public class EntityGraphMapper {
for (Map.Entry<AtlasVertex, List<AtlasClassification>> entry : removedClassifications.entrySet()) {
String guid = GraphHelper.getGuid(entry.getKey());
List<AtlasClassification> deletedClassificationNames = entry.getValue();
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(guid);
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(guid);
entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassificationNames);
}
@@ -1686,9 +1684,8 @@ public class EntityGraphMapper {
}
for (AtlasVertex vertex : notificationVertices) {
- String entityGuid = GraphHelper.getGuid(vertex);
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ String entityGuid = GraphHelper.getGuid(vertex);
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
entityChangeNotifier.onClassificationUpdatedToEntity(entity, updatedClassifications);
@@ -1700,8 +1697,7 @@ public class EntityGraphMapper {
AtlasVertex vertex = entry.getKey();
List<AtlasClassification> removedClassifications = entry.getValue();
String entityGuid = GraphHelper.getGuid(vertex);
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
- AtlasEntity entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ AtlasEntity entity = instanceConverter.getAndCacheEntity(entityGuid);
if (isActive(entity)) {
entityChangeNotifier.onClassificationDeletedFromEntity(entity, removedClassifications);
http://git-wip-us.apache.org/repos/asf/atlas/blob/a064e092/server-api/src/main/java/org/apache/atlas/RequestContext.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index e23eaec..8fe10fe 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -35,7 +35,8 @@ public class RequestContext {
private final Map<String, AtlasObjectId> updatedEntities = new HashMap<>();
private final Map<String, AtlasObjectId> deletedEntities = new HashMap<>();
- private final Map<String, AtlasEntityWithExtInfo> entityCacheV2 = new HashMap<>();
+ private final Map<String, AtlasEntity> entityCache = new HashMap<>();
+ private final Map<String, AtlasEntityWithExtInfo> entityExtInfoCache = new HashMap<>();
private final Map<String, List<AtlasClassification>> addedPropagations = new HashMap<>();
private final Map<String, List<AtlasClassification>> removedPropagations = new HashMap<>();
private final long requestTime = System.currentTimeMillis();
@@ -70,7 +71,8 @@ public class RequestContext {
if (instance != null) {
instance.updatedEntities.clear();
instance.deletedEntities.clear();
- instance.entityCacheV2.clear();
+ instance.entityCache.clear();
+ instance.entityExtInfoCache.clear();
instance.addedPropagations.clear();
instance.removedPropagations.clear();
@@ -174,10 +176,18 @@ public class RequestContext {
*/
public void cache(AtlasEntityWithExtInfo entity) {
if (entity != null && entity.getEntity() != null && entity.getEntity().getGuid() != null) {
- entityCacheV2.put(entity.getEntity().getGuid(), entity);
+ entityExtInfoCache.put(entity.getEntity().getGuid(), entity);
+ entityCache.put(entity.getEntity().getGuid(), entity.getEntity());
}
}
+ public void cache(AtlasEntity entity) {
+ if (entity != null && entity.getGuid() != null) {
+ entityCache.put(entity.getGuid(), entity);
+ }
+ }
+
+
public Collection<AtlasObjectId> getUpdatedEntities() {
return updatedEntities.values();
}
@@ -193,8 +203,12 @@ public class RequestContext {
* @param guid the guid to find
* @return Either the instance or null if it is not in the cache.
*/
- public AtlasEntityWithExtInfo getInstanceV2(String guid) {
- return entityCacheV2.get(guid);
+ public AtlasEntityWithExtInfo getEntityWithExtInfo(String guid) {
+ return entityExtInfoCache.get(guid);
+ }
+
+ public AtlasEntity getEntity(String guid) {
+ return entityCache.get(guid);
}
public long getRequestTime() {