You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by kb...@apache.org on 2020/03/05 13:27:10 UTC
[atlas] branch branch-2.0 updated: ATLAS-3600 : Some System
Attribute of Entity filter doesn't work
This is an automated email from the ASF dual-hosted git repository.
kbhatt pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 66c95d1 ATLAS-3600 : Some System Attribute of Entity filter doesn't work
66c95d1 is described below
commit 66c95d12419ecd6bf4429171eecf5d9039eacdb1
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Mon Mar 2 14:50:42 2020 +0530
ATLAS-3600 : Some System Attribute of Entity filter doesn't work
Signed-off-by: kevalbhatt <kb...@apache.org>
(cherry picked from commit e50372820f146636ba606695ea4bd8ae91d82e55)
---
.../atlas/model/discovery/SearchParameters.java | 1 +
.../atlas/discovery/EntitySearchProcessor.java | 5 ++
.../apache/atlas/discovery/SearchProcessor.java | 77 ++++++++++++++++++++--
.../org/apache/atlas/util/SearchPredicateUtil.java | 45 +++++++++++++
4 files changed, 123 insertions(+), 5 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index c74cf51..fcc4494 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -455,6 +455,7 @@ public class SearchParameters implements Serializable {
STARTS_WITH(new String[]{"startsWith", "STARTSWITH", "begins_with", "BEGINS_WITH"}),
ENDS_WITH(new String[]{"endsWith", "ENDSWITH", "ends_with", "ENDS_WITH"}),
CONTAINS(new String[]{"contains", "CONTAINS"}),
+ NOT_CONTAINS(new String[]{"not_contains", "NOT_CONTAINS"}),
CONTAINS_ANY(new String[]{"containsAny", "CONTAINSANY", "contains_any", "CONTAINS_ANY"}),
CONTAINS_ALL(new String[]{"containsAll", "CONTAINSALL", "contains_all", "CONTAINS_ALL"}),
IS_NULL(new String[]{"isNull", "ISNULL", "is_null", "IS_NULL"}),
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 8f53187..ebd5992 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -310,6 +310,11 @@ public class EntitySearchProcessor extends SearchProcessor {
getVertices(queryResult, entityVertices);
isLastResultPage = entityVertices.size() < limit;
+
+ //incase when operator is NEQ in pipeSeperatedSystemAttributes
+ if (graphQueryPredicate != null) {
+ CollectionUtils.filter(entityVertices, graphQueryPredicate);
+ }
}
super.filter(entityVertices);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 356363d..cd01443 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -25,6 +25,7 @@ import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
@@ -51,6 +52,7 @@ import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
@@ -76,6 +78,7 @@ public abstract class SearchProcessor {
public static final String ALL_TYPE_QUERY = "[* TO *]";
public static final char CUSTOM_ATTR_SEPARATOR = '=';
public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT_GRAPH = "\"%s\":\"%s\"";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
@@ -115,6 +118,8 @@ public abstract class SearchProcessor {
OPERATOR_MAP.put(SearchParameters.Operator.CONTAINS, INDEX_SEARCH_PREFIX + "\"%s\": (*%s*)");
OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.CONTAINS, getContainsPredicateGenerator());
+ OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_CONTAINS, getNotContainsPredicateGenerator());
+
// TODO: Add contains any, contains all mappings here
OPERATOR_MAP.put(SearchParameters.Operator.IS_NULL, "(*:* NOT " + INDEX_SEARCH_PREFIX + "\"%s\":[* TO *])");
@@ -481,13 +486,64 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates);
}
}
- } else if (indexAttributes.contains(criteria.getAttributeName()) && !isPipeSeparatedSystemAttribute(criteria.getAttributeName())){
- return toInMemoryPredicate(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+ } else if (indexAttributes.contains(criteria.getAttributeName())) {
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
+
+ //process attribute value and attribute operator for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
+
+ return toInMemoryPredicate(type, attrName, operator, attrValue);
}
return null;
}
+ private FilterCriteria processPipeSeperatedSystemAttribute(String attrName, SearchParameters.Operator op, String attrVal) {
+
+ FilterCriteria ret = new FilterCriteria();
+
+ if (op != null && attrVal != null) {
+ switch (op) {
+ case STARTS_WITH:
+ attrVal = CLASSIFICATION_NAME_DELIMITER + attrVal;
+ op = SearchParameters.Operator.CONTAINS;
+ break;
+ case ENDS_WITH:
+ attrVal = attrVal + CLASSIFICATION_NAME_DELIMITER;
+ op = SearchParameters.Operator.CONTAINS;
+ break;
+ case EQ:
+ attrVal = GraphHelper.getDelimitedClassificationNames(Collections.singleton(attrVal));
+ op = SearchParameters.Operator.CONTAINS;
+ break;
+ case NEQ:
+ attrVal = GraphHelper.getDelimitedClassificationNames(Collections.singleton(attrVal));
+ op = SearchParameters.Operator.NOT_CONTAINS;
+ break;
+ case CONTAINS:
+ if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
+ attrVal = getCustomAttributeIndexQueryValue(attrVal, true);
+ }
+ break;
+ default:
+ LOG.warn("{}: unsupported operator. Ignored", op);
+ break;
+ }
+ }
+
+ ret.setAttributeName(attrName);
+ ret.setOperator(op);
+ ret.setAttributeValue(attrVal);
+
+ return ret;
+ }
+
private String toIndexExpression(AtlasStructType type, String attrName, SearchParameters.Operator op, String attrVal) {
String ret = EMPTY_STRING;
@@ -499,7 +555,7 @@ public abstract class SearchProcessor {
// map '__customAttributes' 'CONTAINS' operator to 'EQ' operator (solr limitation for json serialized string search)
// map '__customAttributes' value from 'key1=value1' to '\"key1\":\"value1\"' (escape special characters and surround with quotes)
if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && op == SearchParameters.Operator.CONTAINS) {
- ret = String.format(OPERATOR_MAP.get(SearchParameters.Operator.EQ), qualifiedName, getCustomAttributeIndexQueryValue(escapeIndexQueryValue));
+ ret = String.format(OPERATOR_MAP.get(op), qualifiedName, getCustomAttributeIndexQueryValue(escapeIndexQueryValue, false));
} else {
ret = String.format(OPERATOR_MAP.get(op), qualifiedName, escapeIndexQueryValue);
}
@@ -511,7 +567,7 @@ public abstract class SearchProcessor {
return ret;
}
- private String getCustomAttributeIndexQueryValue(String attrValue) {
+ private String getCustomAttributeIndexQueryValue(String attrValue, boolean forGraphQuery) {
String ret = null;
if (StringUtils.isNotEmpty(attrValue)) {
@@ -520,7 +576,11 @@ public abstract class SearchProcessor {
String value = key != null ? attrValue.substring(separatorIdx + 1) : null;
if (key != null && value != null) {
- ret = String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value);
+ if (forGraphQuery) {
+ ret = String.format(CUSTOM_ATTR_SEARCH_FORMAT_GRAPH, key, value);
+ } else {
+ ret = String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value);
+ }
} else {
ret = attrValue;
}
@@ -632,6 +692,13 @@ public abstract class SearchProcessor {
String attrValue = criteria.getAttributeValue();
SearchParameters.Operator operator = criteria.getOperator();
+ //process attribute value and attribute operator for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
+
try {
final String qualifiedName = type.getQualifiedAttributeName(attrName);
diff --git a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
index bb1e9f6..b5ede0b 100644
--- a/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
+++ b/repository/src/main/java/org/apache/atlas/util/SearchPredicateUtil.java
@@ -542,6 +542,43 @@ public class SearchPredicateUtil {
return ret;
}
+ public static VertexAttributePredicateGenerator getNotContainsPredicateGenerator() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("==> getNotContainsPredicateGenerator");
+ }
+
+ VertexAttributePredicateGenerator ret = new VertexAttributePredicateGenerator() {
+ @Override
+ public Predicate generatePredicate(final String attrName, final Object attrVal, final Class attrClass) {
+ final Predicate ret;
+
+ if (attrName == null || attrClass == null || attrVal == null) {
+ ret = ALWAYS_FALSE;
+ } else if (String.class.isAssignableFrom(attrClass)) {
+ ret = StringPredicate.getNotContainsPredicate(attrName, attrClass, (String) attrVal);
+ } else if (Collection.class.isAssignableFrom(attrClass)) {
+ // Check if the provided value is present in the list of stored values
+ ret = new VertexAttributePredicate(attrName, attrClass) {
+ @Override
+ protected boolean compareValue(final Object vertexAttrVal) {
+ return !((Collection) vertexAttrVal).contains(attrVal);
+ }
+ };
+ } else {
+ ret = ALWAYS_FALSE;
+ }
+
+ return ret;
+ }
+ };
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("<== getNotContainsPredicateGenerator");
+ }
+
+ return ret;
+ }
+
public static VertexAttributePredicateGenerator getIsNullPredicateGenerator() {
if (LOG.isDebugEnabled()) {
LOG.debug("==> getIsNullPredicateGenerator");
@@ -1293,6 +1330,14 @@ public class SearchPredicateUtil {
};
}
+ static VertexAttributePredicate getNotContainsPredicate(String attrName, Class attrClass, String value) {
+ return new StringPredicate(attrName, attrClass, value) {
+ protected boolean compareValue(Object vertexAttrVal) {
+ return !((String) vertexAttrVal).contains(value);
+ }
+ };
+ }
+
static VertexAttributePredicate getStartsWithPredicate(String attrName, Class attrClass, String value) {
return new StringPredicate(attrName, attrClass, value) {
protected boolean compareValue(Object vertexAttrVal) {