You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/08/27 21:57:37 UTC
[atlas] branch master updated: ATLAS-4403: Improve find entity by
unique attributes logic - when multiple unique attributes exist for entity
type
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 097f50e ATLAS-4403: Improve find entity by unique attributes logic - when multiple unique attributes exist for entity type
097f50e is described below
commit 097f50eae35e927411af3593fa401a335e61a0e3
Author: Sarath Subramanian <sa...@apache.org>
AuthorDate: Fri Aug 27 14:11:52 2021 -0700
ATLAS-4403: Improve find entity by unique attributes logic - when multiple unique attributes exist for entity type
---
.../store/graph/v2/AtlasGraphUtilsV2.java | 201 ++++++++-------------
1 file changed, 80 insertions(+), 121 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index e73f084..2fce123 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -59,6 +59,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -70,6 +71,7 @@ import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_D
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.SUPER_TYPES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance;
@@ -316,48 +318,29 @@ public class AtlasGraphUtilsV2 {
}
public static AtlasVertex findByUniqueAttributes(AtlasGraph graph, AtlasEntityType entityType, Map<String, Object> attrValues) {
- MetricRecorder metric = RequestContext.get().startMetricRecord("findByUniqueAttributes");
-
- AtlasVertex vertex = null;
-
+ MetricRecorder metric = RequestContext.get().startMetricRecord("findByUniqueAttributes");
+ AtlasVertex vertex = null;
final Map<String, AtlasAttribute> uniqueAttributes = entityType.getUniqAttributes();
if (MapUtils.isNotEmpty(uniqueAttributes) && MapUtils.isNotEmpty(attrValues)) {
- for (AtlasAttribute attribute : uniqueAttributes.values()) {
- Object attrValue = attrValues.get(attribute.getName());
+ Map<String, Object> uniqAttrValues = populateUniqueAttributesMap(uniqueAttributes, attrValues);
+ Map<String, Object> attrNameValues = populateAttributesMap(uniqueAttributes, attrValues);
+ String typeName = entityType.getTypeName();
+ Set<String> entitySubTypes = entityType.getAllSubTypes();
- if (attrValue == null) {
- continue;
- }
-
- if (canUseIndexQuery(graph, entityType, attribute.getName())) {
- vertex = AtlasGraphUtilsV2.getAtlasVertexFromIndexQuery(graph, entityType, attribute, attrValue);
- } else {
- if (USE_UNIQUE_INDEX_PROPERTY_TO_FIND_ENTITY && attribute.getVertexUniquePropertyName() != null) {
- vertex = AtlasGraphUtilsV2.findByTypeAndUniquePropertyName(graph, entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
-
- // if no instance of given typeName is found, try to find an instance of type's sub-type
- if (vertex == null && !entityType.getAllSubTypes().isEmpty()) {
- vertex = AtlasGraphUtilsV2.findBySuperTypeAndUniquePropertyName(graph, entityType.getTypeName(), attribute.getVertexUniquePropertyName(), attrValue);
- }
- } else {
- vertex = AtlasGraphUtilsV2.findByTypeAndPropertyName(graph, entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
-
- // if no instance of given typeName is found, try to find an instance of type's sub-type
- if (vertex == null && !entityType.getAllSubTypes().isEmpty()) {
- vertex = AtlasGraphUtilsV2.findBySuperTypeAndPropertyName(graph, entityType.getTypeName(), attribute.getVertexPropertyName(), attrValue);
- }
- }
+ if (USE_UNIQUE_INDEX_PROPERTY_TO_FIND_ENTITY && MapUtils.isNotEmpty(uniqAttrValues)) {
+ vertex = findByTypeAndUniquePropertyName(graph, typeName, uniqAttrValues);
+ // if no instance of given typeName is found, try to find an instance of type's sub-type
+ if (vertex == null && !entitySubTypes.isEmpty()) {
+ vertex = findBySuperTypeAndUniquePropertyName(graph, typeName, uniqAttrValues);
}
+ } else {
+ vertex = findByTypeAndPropertyName(graph, typeName, attrNameValues);
- if (vertex != null) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("findByUniqueAttributes(type={}, attrName={}, attrValue={}: found vertex {}",
- entityType.getTypeName(), attribute.getName(), attrValue, vertex);
- }
-
- break;
+ // if no instance of given typeName is found, try to find an instance of type's sub-type
+ if (vertex == null && !entitySubTypes.isEmpty()) {
+ vertex = findBySuperTypeAndPropertyName(graph, typeName, attrNameValues);
}
}
}
@@ -484,50 +467,87 @@ public class AtlasGraphUtilsV2 {
return vertex;
}
- public static AtlasVertex findBySuperTypeAndUniquePropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
- MetricRecorder metric = RequestContext.get().startMetricRecord("findBySuperTypeAndUniquePropertyName");
+ private static Map<String, Object> populateUniqueAttributesMap(Map<String, AtlasAttribute> uniqueAttributes, Map<String, Object> attrValues) {
+ return populateAttributesMap(uniqueAttributes, attrValues, true);
+ }
- AtlasGraphQuery query = graph.query()
- .has(Constants.SUPER_TYPES_PROPERTY_KEY, typeName)
- .has(propertyName, attrVal);
+ private static Map<String, Object> populateAttributesMap(Map<String, AtlasAttribute> uniqueAttributes, Map<String, Object> attrValues) {
+ return populateAttributesMap(uniqueAttributes, attrValues, false);
+ }
- Iterator<AtlasVertex> results = query.vertices().iterator();
+ private static Map<String, Object> populateAttributesMap(Map<String, AtlasAttribute> uniqueAttributes, Map<String, Object> attrValues, boolean isUnique) {
+ Map<String, Object> ret = new HashMap<>();
- AtlasVertex vertex = results.hasNext() ? results.next() : null;
+ for (AtlasAttribute attribute : uniqueAttributes.values()) {
+ String attrName = isUnique ? attribute.getVertexUniquePropertyName() : attribute.getVertexPropertyName();
+ Object attrValue = attrValues.get(attribute.getName());
- RequestContext.get().endMetricRecord(metric);
+ if (attrName != null && attrValue != null) {
+ ret.put(attrName, attrValue);
+ }
+ }
- return vertex;
+ return ret;
}
- public static AtlasVertex findByTypeAndPropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
- MetricRecorder metric = RequestContext.get().startMetricRecord("findByTypeAndPropertyName");
+ public static AtlasVertex findByTypeAndUniquePropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues) {
+ return findByTypeAndUniquePropertyName(graph, typeName, attributeValues, false);
+ }
- AtlasGraphQuery query = graph.query()
- .has(ENTITY_TYPE_PROPERTY_KEY, typeName)
- .has(propertyName, attrVal)
- .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ public static AtlasVertex findBySuperTypeAndUniquePropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues) {
+ return findByTypeAndUniquePropertyName(graph, typeName, attributeValues, true);
+ }
- Iterator<AtlasVertex> results = query.vertices().iterator();
+ public static AtlasVertex findByTypeAndUniquePropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues, boolean isSuperType) {
+ String metricName = isSuperType ? "findBySuperTypeAndUniquePropertyName" : "findByTypeAndUniquePropertyName";
+ MetricRecorder metric = RequestContext.get().startMetricRecord(metricName);
+ String typePropertyKey = isSuperType ? SUPER_TYPES_PROPERTY_KEY : ENTITY_TYPE_PROPERTY_KEY;
+ AtlasGraphQuery query = graph.query().has(typePropertyKey, typeName);
- AtlasVertex vertex = results.hasNext() ? results.next() : null;
+ for (Map.Entry<String, Object> entry : attributeValues.entrySet()) {
+ String attrName = entry.getKey();
+ Object attrValue = entry.getValue();
+
+ if (attrName != null && attrValue != null) {
+ query.has(attrName, attrValue);
+ }
+ }
+
+ Iterator<AtlasVertex> results = query.vertices().iterator();
+ AtlasVertex vertex = results.hasNext() ? results.next() : null;
RequestContext.get().endMetricRecord(metric);
return vertex;
}
- public static AtlasVertex findBySuperTypeAndPropertyName(AtlasGraph graph, String typeName, String propertyName, Object attrVal) {
- MetricRecorder metric = RequestContext.get().startMetricRecord("findBySuperTypeAndPropertyName");
+ public static AtlasVertex findByTypeAndPropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues) {
+ return findByTypeAndPropertyName(graph, typeName, attributeValues, false);
+ }
- AtlasGraphQuery query = graph.query()
- .has(Constants.SUPER_TYPES_PROPERTY_KEY, typeName)
- .has(propertyName, attrVal)
- .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
+ public static AtlasVertex findBySuperTypeAndPropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues) {
+ return findByTypeAndPropertyName(graph, typeName, attributeValues, true);
+ }
- Iterator<AtlasVertex> results = query.vertices().iterator();
+ public static AtlasVertex findByTypeAndPropertyName(AtlasGraph graph, String typeName, Map<String, Object> attributeValues, boolean isSuperType) {
+ String metricName = isSuperType ? "findBySuperTypeAndPropertyName" : "findByTypeAndPropertyName";
+ MetricRecorder metric = RequestContext.get().startMetricRecord(metricName);
+ String typePropertyKey = isSuperType ? SUPER_TYPES_PROPERTY_KEY : ENTITY_TYPE_PROPERTY_KEY;
+ AtlasGraphQuery query = graph.query()
+ .has(typePropertyKey, typeName)
+ .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name());
- AtlasVertex vertex = results.hasNext() ? results.next() : null;
+ for (Map.Entry<String, Object> entry : attributeValues.entrySet()) {
+ String attrName = entry.getKey();
+ Object attrValue = entry.getValue();
+
+ if (attrName != null && attrValue != null) {
+ query.has(attrName, attrValue);
+ }
+ }
+
+ Iterator<AtlasVertex> results = query.vertices().iterator();
+ AtlasVertex vertex = results.hasNext() ? results.next() : null;
RequestContext.get().endMetricRecord(metric);
@@ -646,67 +666,6 @@ public class AtlasGraphUtilsV2 {
return element.getProperty(STATE_PROPERTY_KEY, String.class);
}
- private static boolean canUseIndexQuery(AtlasGraph graph, AtlasEntityType entityType, String attributeName) {
- boolean ret = false;
-
- if (USE_INDEX_QUERY_TO_FIND_ENTITY_BY_UNIQUE_ATTRIBUTES) {
- final String typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
-
- ret = typeAndSubTypesQryStr.length() <= SearchProcessor.MAX_QUERY_STR_LENGTH_TYPES;
-
- if (ret) {
- Set<String> indexSet = graph.getVertexIndexKeys();
- try {
- ret = indexSet.contains(entityType.getVertexPropertyName(attributeName));
- }
- catch (AtlasBaseException ex) {
- ret = false;
- }
- }
- }
-
- return ret;
- }
-
- private static AtlasVertex getAtlasVertexFromIndexQuery(AtlasGraph graph, AtlasEntityType entityType, AtlasAttribute attribute, Object attrVal) {
- String propertyName = attribute.getVertexPropertyName();
- AtlasIndexQuery query = getIndexQuery(graph, entityType, propertyName, attrVal.toString());
-
- for (Iterator<Result> iter = query.vertices(); iter.hasNext(); ) {
- Result result = iter.next();
- AtlasVertex vertex = result.getVertex();
-
- // skip non-entity vertices, if any got returned
- if (vertex == null || !vertex.getPropertyKeys().contains(Constants.GUID_PROPERTY_KEY)) {
- continue;
- }
-
- // verify the typeName
- String typeNameInVertex = getTypeName(vertex);
-
- if (!entityType.getTypeAndAllSubTypes().contains(typeNameInVertex)) {
- LOG.warn("incorrect vertex type from index-query: expected='{}'; found='{}'", entityType.getTypeName(), typeNameInVertex);
-
- continue;
- }
-
- if (attrVal.getClass() == String.class) {
- String s = (String) attrVal;
- String vertexVal = vertex.getProperty(propertyName, String.class);
-
- if (!s.equalsIgnoreCase(vertexVal)) {
- LOG.warn("incorrect match from index-query for property {}: expected='{}'; found='{}'", propertyName, s, vertexVal);
-
- continue;
- }
- }
-
- return vertex;
- }
-
- return null;
- }
-
private static AtlasIndexQuery getIndexQuery(AtlasGraph graph, AtlasEntityType entityType, String propertyName, String value) {
StringBuilder sb = new StringBuilder();