You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ni...@apache.org on 2020/03/31 11:13:37 UTC
[atlas] branch branch-2.0 updated (761172b -> 0bd819a)
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from 761172b ATLAS-3667: Option to store Ldap/AD bind password in jceks keystore file - #2 (fix for NPE)
new 4a3ac00 ATLAS-3703 : BasicSearch: Search by terms/Search by text in combination of Classification filter, filter doesn't work
new 0bd819a ATLAS-3672 : Search with Classification without filters and Entity with filters doesn't fetch expected results.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../discovery/ClassificationSearchProcessor.java | 93 +++++++++++++--------
.../apache/atlas/discovery/SearchProcessor.java | 97 +++++++++++-----------
2 files changed, 109 insertions(+), 81 deletions(-)
[atlas] 02/02: ATLAS-3672 : Search with Classification without
filters and Entity with filters doesn't fetch expected results.
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 0bd819a9142446aa031c981fc76f27b75c1b2e58
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Tue Mar 31 16:13:28 2020 +0530
ATLAS-3672 : Search with Classification without filters and Entity with filters doesn't fetch expected results.
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit 4d0f5ce213a49fcd8509a2ea5f6a0bb733c0c186)
---
.../apache/atlas/discovery/SearchProcessor.java | 88 +++++++++++-----------
1 file changed, 42 insertions(+), 46 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 074ad8b..5d7c508 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -738,52 +738,48 @@ public abstract class SearchProcessor {
operator = processedCriteria.getOperator();
}
- try {
- final String qualifiedName = type.getQualifiedAttributeName(attrName);
-
- switch (operator) {
- case LT:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
- break;
- case LTE:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
- break;
- case GT:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
- break;
- case GTE:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
- break;
- case EQ:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
- break;
- case NEQ:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
- break;
- case LIKE:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
- break;
- case CONTAINS:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
- break;
- case STARTS_WITH:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
- break;
- case ENDS_WITH:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
- break;
- case IS_NULL:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
- break;
- case NOT_NULL:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
- break;
- default:
- LOG.warn("{}: unsupported operator. Ignored", operator);
- break;
- }
- } catch (AtlasBaseException e) {
- LOG.error("toGraphFilterQuery(): failed for attrName=" + attrName + "; operator=" + operator + "; attrValue=" + attrValue, e);
+ final String qualifiedName = type.getAttribute(attrName).getVertexPropertyName();
+
+ switch (operator) {
+ case LT:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+ break;
+ case LTE:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+ break;
+ case GT:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+ break;
+ case GTE:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+ break;
+ case EQ:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+ break;
+ case NEQ:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+ break;
+ case LIKE:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
+ break;
+ case CONTAINS:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+ break;
+ case STARTS_WITH:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+ break;
+ case ENDS_WITH:
+ query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+ break;
+ case IS_NULL:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
+ break;
+ case NOT_NULL:
+ query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
+ break;
+ default:
+ LOG.warn("{}: unsupported operator. Ignored", operator);
+ break;
}
}
}
[atlas] 01/02: ATLAS-3703 : BasicSearch: Search by terms/Search by
text in combination of Classification filter, filter doesn't work
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 4a3ac0020d867026859c7b4c394f9417e544dce0
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Mon Mar 30 11:15:35 2020 +0530
ATLAS-3703 : BasicSearch: Search by terms/Search by text in combination of Classification filter, filter doesn't work
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit 61c2fe3ebeb7a71620bc9d0a231710d2b42aa800)
---
.../discovery/ClassificationSearchProcessor.java | 93 ++++++++++++++--------
.../apache/atlas/discovery/SearchProcessor.java | 9 ++-
2 files changed, 67 insertions(+), 35 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 1343c9b..74c088c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -42,13 +42,7 @@ import org.slf4j.LoggerFactory;
import javax.script.ScriptEngine;
import javax.script.ScriptException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
@@ -109,13 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
/* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well.
*/
- final boolean useIndexSearchForClassification = (classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
- classificationType != MATCH_ALL_CLASSIFIED &&
- classificationType != MATCH_ALL_NOT_CLASSIFIED && !isWildcardSearch) &&
+ final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
- CollectionUtils.isEmpty(graphAttributes) &&
+ CollectionUtils.isNotEmpty(indexAttributes) &&
canApplyIndexFilter(classificationType, filterCriteria, false);
+ traitPredicate = buildTraitPredict(classificationType);
+ isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
+
AtlasGraph graph = context.getGraph();
// index query directly on entity
@@ -148,15 +143,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
LOG.debug("Using query string '{}'.", indexQuery);
-
- isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
- traitPredicate = buildTraitPredict(classificationType);
- inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate);
-
} else {
- indexQuery = null;
- traitPredicate = null;
- isEntityPredicate = null;
+ indexQuery = null;
}
// index query directly on classification
@@ -286,25 +274,27 @@ public class ClassificationSearchProcessor extends SearchProcessor {
getVerticesFromIndexQueryResult(queryResult, entityVertices);
isLastResultPage = entityVertices.size() < limit;
- if (isEntityPredicate != null) {
- // Do in-memory filtering
- CollectionUtils.filter(entityVertices, isEntityPredicate);
- }
- } else {
- if (classificationIndexQuery != null) {
- Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
-
- getVerticesFromIndexQueryResult(queryResult, classificationVertices);
+ // Do in-memory filtering
+ CollectionUtils.filter(entityVertices, traitPredicate);
+ CollectionUtils.filter(entityVertices, isEntityPredicate);
- // Do in-memory filtering before the graph query
- CollectionUtils.filter(classificationVertices, inMemoryPredicate);
+ } else {
+ if (tagGraphQueryWithAttributes != null) {
- } else if (context.getSearchParameters().getTagFilters() != null) {
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
+
+ } else if (classificationIndexQuery != null){
+
+ Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
+
+ getVerticesFromIndexQueryResult(queryResult, classificationVertices);
+
+ // Do in-memory filtering before the graph query
+ CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
@@ -333,6 +323,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (whiteSpaceFilter) {
filterWhiteSpaceClassification(entityVertices);
}
+ // Do in-memory filtering
+ CollectionUtils.filter(entityVertices, isEntityPredicate);
super.filter(entityVertices);
@@ -358,7 +350,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
-
+ //in case of classification type + graph attributes
if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
// Now filter on the tag attributes
Set<String> guids = getGuids(entityVertices);
@@ -381,8 +373,43 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.warn(e.getMessage(), e);
}
}
- } else if (traitPredicate != null) {
+ } else if (inMemoryPredicate != null) {
+ //in case of classification type + index attributes
+ CollectionUtils.filter(entityVertices, traitPredicate);
+
+ //filter attributes (filterCriteria). Find classification vertex(typeName = classification) from entity vertex (traitName = classification)
+ final Set<String> processedGuids = new HashSet<>();
+ List<AtlasVertex> matchEntityVertices = new ArrayList<>();
+ if (CollectionUtils.isNotEmpty(entityVertices)) {
+ for (AtlasVertex entityVertex : entityVertices) {
+ Iterable<AtlasEdge> edges = entityVertex.getEdges(AtlasEdgeDirection.OUT, Constants.CLASSIFICATION_LABEL);
+
+ for (AtlasEdge edge : edges) {
+ AtlasVertex classificationVertex = edge.getInVertex();
+
+ AtlasVertex matchVertex = (AtlasVertex) CollectionUtils.find(Collections.singleton(classificationVertex), inMemoryPredicate);
+ if (matchVertex != null) {
+ String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex);
+
+ if (processedGuids.contains(guid)) {
+ continue;
+ }
+
+ matchEntityVertices.add(entityVertex);
+ processedGuids.add(guid);
+ break;
+
+ }
+ }
+ }
+ }
+ entityVertices.clear();
+ entityVertices.addAll(matchEntityVertices);
+
+ } else {
+ //in case of only classsification type
CollectionUtils.filter(entityVertices, traitPredicate);
+ CollectionUtils.filter(entityVertices, isEntityPredicate);
}
super.filter(entityVertices);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ddd3e60..074ad8b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -79,7 +79,7 @@ public abstract class SearchProcessor {
public static final char CUSTOM_ATTR_SEPARATOR = '=';
public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
public static final String CUSTOM_ATTR_SEARCH_FORMAT_GRAPH = "\"%s\":\"%s\"";
-
+ public static final String WILDCARD_CHAR = "*";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
@@ -195,12 +195,17 @@ public abstract class SearchProcessor {
protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
Predicate traitPredicate;
- if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch() || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
+ if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ } else if (context.isWildCardSearch()) {
+ String wildcardName = context.getClassificationName();
+ wildcardName = wildcardName.replace(WILDCARD_CHAR,"");
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, wildcardName, String.class),
+ SearchPredicateUtil.getContainsPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, wildcardName, String.class));
} else {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));