You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/09/26 20:39:24 UTC
[atlas] branch master updated: ATLAS-3257: Enhance Classification
Basic Search
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/master by this push:
new 9d432d5 ATLAS-3257: Enhance Classification Basic Search
9d432d5 is described below
commit 9d432d5db2ce2e3469b0e00bfb5685982ef59f85
Author: Le Ma <lm...@cloudera.com>
AuthorDate: Thu Sep 26 13:38:56 2019 -0700
ATLAS-3257: Enhance Classification Basic Search
Signed-off-by: Sarath Subramanian <sa...@apache.org>
---
.../graphdb/AtlasIndexQueryParameter.java | 2 -
.../janusgraph/diskstorage/solr/Solr6Index.java | 16 ++
.../graphdb/janus/AbstractGraphDatabaseTest.java | 10 -
.../test/java/org/apache/atlas/TestUtilsV2.java | 2 +
.../discovery/ClassificationSearchProcessor.java | 247 ++++++++++-----------
.../atlas/discovery/EntitySearchProcessor.java | 7 +-
.../atlas/discovery/GraphIndexQueryBuilder.java | 108 +++++++++
.../org/apache/atlas/discovery/SearchContext.java | 28 ++-
.../apache/atlas/discovery/SearchProcessor.java | 50 +++--
.../store/graph/v2/AtlasGraphUtilsV2.java | 21 +-
.../atlas/web/adapters/TestEntitiesREST.java | 226 ++++++++++++++++---
.../json/search-parameters/tag-filters.json | 17 --
12 files changed, 522 insertions(+), 212 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
index 347f216..ce0d5fe 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQueryParameter.java
@@ -18,8 +18,6 @@
package org.apache.atlas.repository.graphdb;
-import java.util.Iterator;
-
/**
* Represents an index query parameter for use with AtlasGraph queries.
*/
diff --git a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
index 9d81a12..a272ab9 100644
--- a/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
+++ b/graphdb/janus/src/main/java/org/janusgraph/diskstorage/solr/Solr6Index.java
@@ -370,6 +370,11 @@ public class Solr6Index implements IndexProvider {
String analyzer = ParameterType.STRING_ANALYZER.findParameter(information.getParameters(), null);
if (analyzer != null) {
//If the key have a tokenizer, we try to get it by reflection
+ // some referred classes might not be able to be found via SystemClassLoader
+ // since they might be associated with other classloader, in this situation
+ // ClassNotFound exception will be thrown. instead of using SystemClassLoader
+ // for all classes, we find its classloader first and then load the class, please
+ // call - instantiateUsingClassLoader()
try {
((Constructor<Tokenizer>) ClassLoader.getSystemClassLoader().loadClass(analyzer)
.getConstructor()).newInstance();
@@ -389,6 +394,17 @@ public class Solr6Index implements IndexProvider {
}
}
+ private void instantiateUsingClassLoader(String analyzer) throws PermanentBackendException {
+ if (analyzer == null) return;
+ try {
+ Class analyzerClass = Class.forName(analyzer);
+ ClassLoader cl = analyzerClass.getClassLoader();
+ ((Constructor<Tokenizer>) cl.loadClass(analyzer).getConstructor()).newInstance();
+ } catch (final ReflectiveOperationException e) {
+ throw new PermanentBackendException(e.getMessage(),e);
+ }
+ }
+
@Override
public void mutate(Map<String, Map<String, IndexMutation>> mutations, KeyInformation.IndexRetriever information,
BaseTransaction tx) throws BackendException {
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
index 7d1260c..3500415 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AbstractGraphDatabaseTest.java
@@ -33,10 +33,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-
-/**
- *
- */
public abstract class AbstractGraphDatabaseTest {
protected static final String WEIGHT_PROPERTY = "weight";
@@ -112,13 +108,8 @@ public abstract class AbstractGraphDatabaseTest {
//ok
t.printStackTrace();
}
-
-
}
-
-
-
protected final <V, E> AtlasGraph<V, E> getGraph() {
if (graph == null) {
graph = new AtlasJanusGraph();
@@ -184,5 +175,4 @@ public abstract class AbstractGraphDatabaseTest {
return exceptionThrown;
}
}
-
}
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index a109f6a..530d5cd 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -544,6 +544,8 @@ public final class TestUtilsV2 {
public static final String COLUMN_TYPE = "column_type";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
+ public static final String FETL_CLASSIFICATION = "fetl" + CLASSIFICATION;
+
public static final String PII = "PII";
public static final String PHI = "PHI";
public static final String SUPER_TYPE_NAME = "Base";
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 724c0f6..0aa229c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -31,11 +30,10 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
-import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
-import org.apache.commons.collections.Predicate;
-import org.apache.commons.collections.PredicateUtils;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,7 +43,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -53,95 +50,139 @@ import java.util.Set;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
-import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
-import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
-
+/**
+ * This class is needed when this is a registered classification type or wildcard search,
+ * registered classification includes special type as well. (tag filters will be ignored, and front-end should not enable
+ * tag-filter for special classification types, including wildcard search - classification name contains *)
+ */
public class ClassificationSearchProcessor extends SearchProcessor {
- private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
- private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
- private final AtlasIndexQuery indexQuery;
- private final AtlasGraphQuery tagGraphQueryWithAttributes;
- private final AtlasGraphQuery entityGraphQueryTraitNames;
- private Predicate entityPredicateTraitNames;
+ private static final Logger LOG = LoggerFactory.getLogger(ClassificationSearchProcessor.class);
+ private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("ClassificationSearchProcessor");
+
+ private final AtlasIndexQuery indexQuery;
+ private final AtlasIndexQuery classificationIndexQuery;
+ private final AtlasGraphQuery tagGraphQueryWithAttributes;
+ private final Map<String, Object> gremlinQueryBindings;
+ private final String gremlinTagFilterQuery;
- private final String gremlinTagFilterQuery;
- private final Map<String, Object> gremlinQueryBindings;
+ // Some index engines may take space as a delimiter, when basic search is
+ // executed, unsatisfying results may be returned.
+ // eg, an entity A has classification "cls" and B has "cls 1"
+ // when user execute a exact search for "cls", only A should be returned
+ // but both A and B are returned. To avoid this, we should filter the res.
+ private boolean whiteSpaceFilter = false;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
- final AtlasClassificationType classificationType = context.getClassificationType();
- final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
- final Set<String> indexAttributes = new HashSet<>();
- final Set<String> graphAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
+ final AtlasClassificationType classificationType = context.getClassificationType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
+ final Set<String> indexAttributes = new HashSet<>();
+ final Set<String> graphAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
final Set<String> typeAndSubTypes = context.getClassificationTypes();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
- final String sortBy = context.getSearchParameters().getSortBy();
- final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+ final boolean isBuiltInType = context.isBuiltInClassificationType();
+ final boolean isWildcardSearch = context.isWildCardSearch();
processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
- // for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
- boolean useIndexSearch = classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
- classificationType != MATCH_ALL_CLASSIFIED &&
- classificationType != MATCH_ALL_NOT_CLASSIFIED &&
- typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS &&
- CollectionUtils.isEmpty(graphAttributes) &&
- canApplyIndexFilter(classificationType, filterCriteria, false);
+ /* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
+ There are four cases in the classification type :
+ 1. unique classification type, including not classified, single wildcard (*), match all classified
+ 2. wildcard search, including starting/ending/mid wildcard, like cls*, *c*, *ion.
+ 3. registered classification type, like PII, PHI
+ 4. classification is not present in the search parameter
+ each of above cases with either has empty/or not tagFilters
+ */
+ final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) &&
+ filterCriteria == null &&
+ (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
+
+ /* If classification's attributes can be applied index filter, we can use direct index
+ * to query classification index as well.
+ */
+ final boolean useIndexSearchForClassification = (classificationType != MATCH_ALL_WILDCARD_CLASSIFICATION &&
+ classificationType != MATCH_ALL_CLASSIFIED &&
+ classificationType != MATCH_ALL_NOT_CLASSIFIED && !isWildcardSearch) &&
+ (typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
+ CollectionUtils.isEmpty(graphAttributes) &&
+ canApplyIndexFilter(classificationType, filterCriteria, false);
AtlasGraph graph = context.getGraph();
- if (useIndexSearch) {
- StringBuilder indexQuery = new StringBuilder();
+ // index query directly on entity
+ if (useIndexSearchForEntity) {
- constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
- constructFilterQuery(indexQuery, classificationType, filterCriteria, indexAttributes);
+ StringBuilder queryString = new StringBuilder();
+ graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
- String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
+ if (isWildcardSearch) {
- indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
- indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+ // tagFilters is not allowed in wildcard search
+ graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
+ } else {
+ if (isBuiltInType) {
- this.indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
+ // tagFilters is not allowed in unique classificationType search
+ graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
- Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator()
- .generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
- Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
- if (attributePredicate != null) {
- inMemoryPredicate = PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
- } else {
- inMemoryPredicate = typeNamePredicate;
+ } else {
+
+ // only registered classification will search for subtypes
+ graphIndexQueryBuilder.addClassificationAndSubTypesQueryFilter(queryString);
+ whiteSpaceFilter = true;
+ }
}
+
+ String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
+ indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
+ indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+ indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
+
+ LOG.debug("Using query string '{}'.", indexQuery);
} else {
indexQuery = null;
}
- if (context.getSearchParameters().getTagFilters() != null) {
- // Now filter on the tag attributes
- AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
+ // index query directly on classification
+ if (useIndexSearchForClassification) {
+
+ StringBuilder queryString = new StringBuilder();
+
+ graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
+ graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
+
+ constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes);
+
+ String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
+ indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
+ indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
+
+ this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
+ } else {
+ classificationIndexQuery = null;
+ }
- tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes));
- entityGraphQueryTraitNames = null;
- entityPredicateTraitNames = null;
+ // only registered classification will search with tag filters
+ if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) {
- gremlinQueryBindings = new HashMap<>();
+ AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
+ tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes,
+ graph.query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes));
+ gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
+
gremlinQuery.append("g.V().has('__guid', T.in, guids)");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
gremlinQuery.append(".as('e').out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
+
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(".back('e').toList()");
@@ -154,50 +195,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
}
} else {
- tagGraphQueryWithAttributes = null;
- List<AtlasGraphQuery> orConditions = new LinkedList<>();
-
- if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
- orConditions.add(graph.query().createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
- orConditions.add(graph.query().createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, NOT_EQUAL, null));
-
- entityGraphQueryTraitNames = graph.query().or(orConditions);
- entityPredicateTraitNames = PredicateUtils.orPredicate(
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
- } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
- orConditions.add(graph.query().createChildQuery().has(GUID_PROPERTY_KEY, NOT_EQUAL, null).has(ENTITY_TYPE_PROPERTY_KEY, NOT_EQUAL, null)
- .has(TRAIT_NAMES_PROPERTY_KEY, EQUAL, null).has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, EQUAL, null));
-
- entityGraphQueryTraitNames = graph.query().or(orConditions);
- entityPredicateTraitNames = PredicateUtils.andPredicate(
- SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
- } else {
- orConditions.add(graph.query().createChildQuery().in(TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
- orConditions.add(graph.query().createChildQuery().in(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, typeAndSubTypes));
-
- entityGraphQueryTraitNames = graph.query().or(orConditions);
- entityPredicateTraitNames = PredicateUtils.orPredicate(
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes(), List.class));
- }
-
- if (context.getSearchParameters().getExcludeDeletedEntities()) {
- entityGraphQueryTraitNames.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
-
- final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator().generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
-
- entityPredicateTraitNames = PredicateUtils.andPredicate(entityPredicateTraitNames, activePredicate);
- }
-
- if (sortBy != null && !sortBy.isEmpty()) {
- AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
- entityGraphQueryTraitNames.orderBy(sortBy, qrySortOrder);
- }
-
+ tagGraphQueryWithAttributes = null;
gremlinTagFilterQuery = null;
- gremlinQueryBindings = null;
+ gremlinQueryBindings = null;
}
}
@@ -218,7 +218,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
try {
final int startIdx = context.getSearchParameters().getOffset();
final int limit = context.getSearchParameters().getLimit();
- final boolean activeOnly = context.getSearchParameters().getExcludeDeletedEntities();
// query to start at 0, even though startIdx can be higher - because few results in earlier retrieval could
// have been dropped: like non-active-entities or duplicate-entities (same entity pointed to by multiple
@@ -232,6 +231,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
final List<AtlasVertex> entityVertices = new ArrayList<>();
final List<AtlasVertex> classificationVertices = new ArrayList<>();
+ final String sortBy = context.getSearchParameters().getSortBy();
+ final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
+
for (; ret.size() < limit; qryOffset += limit) {
entityVertices.clear();
classificationVertices.clear();
@@ -242,34 +244,31 @@ public class ClassificationSearchProcessor extends SearchProcessor {
break;
}
- final boolean isLastResultPage;
+ boolean isLastResultPage = true;
if (indexQuery != null) {
- Iterator<AtlasIndexQuery.Result> queryResult = indexQuery.vertices(qryOffset, limit);
-
- getVerticesFromIndexQueryResult(queryResult, classificationVertices);
-
- isLastResultPage = classificationVertices.size() < limit;
+ Iterator<AtlasIndexQuery.Result> queryResult;
+ if (StringUtils.isNotEmpty(sortBy)) {
+ Order qrySortOrder = sortOrder == SortOrder.ASCENDING ? Order.asc : Order.desc;
+ queryResult = indexQuery.vertices(qryOffset, limit, sortBy, qrySortOrder);
+ } else {
+ queryResult = indexQuery.vertices(qryOffset, limit);
+ }
- // Do in-memory filtering before the graph query
- CollectionUtils.filter(classificationVertices, inMemoryPredicate);
+ getVerticesFromIndexQueryResult(queryResult, entityVertices);
+ isLastResultPage = entityVertices.size() < limit;
} else {
- if (context.getSearchParameters().getTagFilters() == null) {
- // We can use single graph query to determine in this case
- Iterator<AtlasVertex> queryResult = entityGraphQueryTraitNames.vertices(qryOffset, limit).iterator();
+ if (classificationIndexQuery != null) {
+ Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
- getVertices(queryResult, entityVertices);
+ getVerticesFromIndexQueryResult(queryResult, classificationVertices);
- isLastResultPage = entityVertices.size() < limit;
- } else {
+ } else if (context.getSearchParameters().getTagFilters() != null) {
Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
-
- // Do in-memory filtering before the graph query
- CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
@@ -282,10 +281,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
for (AtlasEdge edge : edges) {
AtlasVertex entityVertex = edge.getOutVertex();
- if (activeOnly && AtlasGraphUtilsV2.getState(entityVertex) != AtlasEntity.Status.ACTIVE) {
- continue;
- }
-
String guid = AtlasGraphUtilsV2.getIdFromVertex(entityVertex);
if (processedGuids.contains(guid)) {
@@ -299,6 +294,10 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
}
+ if (whiteSpaceFilter) {
+ filterWhiteSpaceClassification(entityVertices);
+ }
+
super.filter(entityVertices);
resultIdx = collectResultVertices(ret, startIdx, limit, resultIdx, entityVertices);
@@ -346,8 +345,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.warn(e.getMessage(), e);
}
}
- } else if (entityPredicateTraitNames != null) {
- CollectionUtils.filter(entityVertices, entityPredicateTraitNames);
}
super.filter(entityVertices);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index b211074..03eb92b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -111,7 +111,7 @@ public class EntitySearchProcessor extends SearchProcessor {
StringBuilder indexQuery = new StringBuilder();
if (typeSearchByIndex) {
- constructTypeTestQuery(indexQuery, typeAndSubTypesQryStr);
+ graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr);
// TypeName check to be done in-memory as well to address ATLAS-2121 (case sensitivity)
inMemoryPredicate = typeNamePredicate;
@@ -131,9 +131,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (indexQuery.length() > 0) {
- if (context.getSearchParameters().getExcludeDeletedEntities()) {
- constructStateTestQuery(indexQuery);
- }
+
+ graphIndexQueryBuilder.addActiveStateQueryFilter(indexQuery);
String indexQueryString = STRAY_AND_PATTERN.matcher(indexQuery).replaceAll(")");
diff --git a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
new file mode 100644
index 0000000..3f58acb
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
+import static org.apache.atlas.discovery.SearchProcessor.INDEX_SEARCH_PREFIX;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.commons.lang3.StringUtils;
+
+public class GraphIndexQueryBuilder {
+ SearchContext context;
+
+ GraphIndexQueryBuilder(SearchContext context) {
+ this.context = context;
+ }
+
+ void addClassificationTypeFilter(StringBuilder indexQuery) {
+ if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+ String classificationName = context.getSearchParameters().getClassification();
+ if (indexQuery.length() != 0) {
+ indexQuery.append(" AND ");
+ }
+
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
+ .append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
+ .append('\"').append(':').append(classificationName).append(")");
+ }
+ }
+
+ void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
+ if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+ String classificationTypesQryStr = context.getClassificationTypesQryStr();
+
+ if (indexQuery.length() != 0) {
+ indexQuery.append(" AND ");
+ }
+
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
+ .append("\"").append(":" + classificationTypesQryStr).append(" OR ").append(INDEX_SEARCH_PREFIX)
+ .append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + classificationTypesQryStr).append(")");
+ }
+ }
+
+ void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
+ if (indexQuery != null && context.getClassificationType() != null) {
+ if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
+ if (indexQuery.length() != 0) {
+ indexQuery.append(" AND ");
+ }
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
+ .append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
+ .append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
+ .append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
+
+ } else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
+ if (indexQuery.length() != 0) {
+ indexQuery.append(" AND ");
+ }
+ indexQuery.append("(").append("-").append(INDEX_SEARCH_PREFIX).append("\"").append(CLASSIFICATION_NAMES_KEY)
+ .append("\"").append(":" + "[* TO *]").append(" AND ").append("-")
+ .append(INDEX_SEARCH_PREFIX).append("\"").append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
+ .append("\"").append(":" + "[* TO *]").append(")");
+ }
+ }
+ }
+
+ void addActiveStateQueryFilter(StringBuilder indexQuery){
+ if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery != null) {
+ if (indexQuery.length() != 0) {
+ indexQuery.append(" AND ");
+ }
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"").append(STATE_PROPERTY_KEY)
+ .append("\"").append(":" + "ACTIVE").append(")");
+ }
+ }
+
+ void addTypeAndSubTypesQueryFilter(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
+ if (indexQuery != null && StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
+ if (indexQuery.length() > 0) {
+ indexQuery.append(" AND ");
+ }
+
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY)
+ .append("\":").append(typeAndAllSubTypesQryStr).append(")");
+ }
+ }
+}
\ No newline at end of file
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 49f9f98..7ad32bd 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -63,6 +63,7 @@ public class SearchContext {
private final Set<String> entityAttributes;
private final AtlasEntityType entityType;
private final AtlasClassificationType classificationType;
+ private final String classificationName;
private SearchProcessor searchProcessor;
private boolean terminateSearch = false;
private final Set<String> typeAndSubTypes;
@@ -74,10 +75,8 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
-
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
- String classificationName = searchParameters.getClassification();
-
+ this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
@@ -92,7 +91,7 @@ public class SearchContext {
}
// Validate if the classification exists
- if (StringUtils.isNotEmpty(classificationName) && classificationType == null) {
+ if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
}
@@ -239,7 +238,21 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
- return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
+ return (classificationType != null || isWildCardSearch());
+ }
+
+ boolean isBuiltInClassificationType() {
+ return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
+ || getClassificationType() == MATCH_ALL_CLASSIFIED
+ || getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
+ }
+
+ boolean isWildCardSearch () {
+ String classification = getSearchParameters().getClassification();
+ if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
+ return classification.contains("*");
+ }
+ return false;
}
boolean needEntityProcessor() {
@@ -263,7 +276,10 @@ public class SearchContext {
private void validateAttributes(final AtlasStructType structType, final String... attributeNames) throws AtlasBaseException {
for (String attributeName : attributeNames) {
- if (StringUtils.isNotEmpty(attributeName) && structType.getAttributeType(attributeName) == null) {
+ if (StringUtils.isNotEmpty(attributeName) && (structType == null || structType.getAttributeType(attributeName) == null)) {
+ if (structType == null) {
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
+ }
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
}
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index a12ce3f..015aade 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -110,13 +110,14 @@ public abstract class SearchProcessor {
OPERATOR_PREDICATE_MAP.put(SearchParameters.Operator.NOT_NULL, getNotNullPredicateGenerator());
}
- protected final SearchContext context;
- protected SearchProcessor nextProcessor;
- protected Predicate inMemoryPredicate;
-
+ protected final SearchContext context;
+ protected SearchProcessor nextProcessor;
+ protected Predicate inMemoryPredicate;
+ protected GraphIndexQueryBuilder graphIndexQueryBuilder;
protected SearchProcessor(SearchContext context) {
this.context = context;
+ this.graphIndexQueryBuilder = new GraphIndexQueryBuilder(context);
}
public void addProcessor(SearchProcessor processor) {
@@ -239,13 +240,30 @@ public abstract class SearchProcessor {
return ret;
}
- protected void constructTypeTestQuery(StringBuilder indexQuery, String typeAndAllSubTypesQryStr) {
- if (StringUtils.isNotEmpty(typeAndAllSubTypesQryStr)) {
- if (indexQuery.length() > 0) {
- indexQuery.append(AND_STR);
- }
+ protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
+ if (CollectionUtils.isNotEmpty(entityVertices)) {
+
+ boolean hasExactMatch = false;
+ Iterator<AtlasVertex> it = entityVertices.iterator();
+
+ while (it.hasNext()) {
+ AtlasVertex entityVertex = it.next();
+ List<String> classificationNames = AtlasGraphUtilsV2.getClassificationNames(entityVertex);
+ if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
+ hasExactMatch = true;
+ }
+
+ if (hasExactMatch) continue;
+
+ classificationNames = AtlasGraphUtilsV2.getPropagatedClassificationNames(entityVertex);
+ if (CollectionUtils.isNotEmpty(classificationNames) && classificationNames.contains(context.getClassificationType().getTypeName())) {
+ hasExactMatch = true;
+ }
- indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\":").append(typeAndAllSubTypesQryStr);
+ if (!hasExactMatch) {
+ it.remove();
+ }
+ }
}
}
@@ -322,14 +340,6 @@ public abstract class SearchProcessor {
}
}
- protected void constructStateTestQuery(StringBuilder indexQuery) {
- if (indexQuery.length() > 0) {
- indexQuery.append(AND_STR);
- }
-
- indexQuery.append(INDEX_SEARCH_PREFIX + "\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
- }
-
private boolean isIndexSearchable(FilterCriteria filterCriteria, AtlasStructType structType) throws AtlasBaseException {
String qualifiedName = structType.getQualifiedAttributeName(filterCriteria.getAttributeName());
Set<String> indexedKeys = context.getIndexedKeys();
@@ -747,7 +757,7 @@ public abstract class SearchProcessor {
return false;
}
- protected List<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, List<AtlasVertex> vertices) {
+ protected Collection<AtlasVertex> getVerticesFromIndexQueryResult(Iterator<AtlasIndexQuery.Result> idxQueryResult, Collection<AtlasVertex> vertices) {
if (idxQueryResult != null) {
while (idxQueryResult.hasNext()) {
AtlasVertex vertex = idxQueryResult.next().getVertex();
@@ -759,7 +769,7 @@ public abstract class SearchProcessor {
return vertices;
}
- protected List<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, List<AtlasVertex> vertices) {
+ protected Collection<AtlasVertex> getVertices(Iterator<AtlasVertex> iterator, Collection<AtlasVertex> vertices) {
if (iterator != null) {
while (iterator.hasNext()) {
AtlasVertex vertex = iterator.next();
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
index b90d67f..f07cff1 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java
@@ -50,7 +50,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collection;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.Iterator;
@@ -58,9 +58,11 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_DEFAULT;
import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_PROPERTY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
@@ -634,4 +636,21 @@ public class AtlasGraphUtilsV2 {
public static String getIndexSearchPrefix() {
return INDEX_SEARCH_PREFIX;
}
+
+ public static List<String> getClassificationNames(AtlasVertex entityVertex) {
+ return getClassificationNamesHelper(entityVertex, CLASSIFICATION_NAMES_KEY);
+ }
+
+ public static List<String> getPropagatedClassificationNames(AtlasVertex entityVertex) {
+ return getClassificationNamesHelper(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY);
+ }
+
+ private static List<String> getClassificationNamesHelper(AtlasVertex entityVertex, String propertyKey) {
+ List<String> classificationNames = null;
+ String classificationNamesString = entityVertex.getProperty(propertyKey, String.class);
+ if (StringUtils.isNotEmpty(classificationNamesString)) {
+ classificationNames = Arrays.asList(classificationNamesString.split("\\|"));
+ }
+ return classificationNames;
+ }
}
diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
index b62938f..1e1c1ff 100644
--- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
+++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
@@ -17,10 +17,16 @@
*/
package org.apache.atlas.web.adapters;
+import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
+import static org.apache.atlas.TestUtilsV2.DATABASE_TYPE;
+import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
+
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.model.discovery.AtlasSearchResult;
+import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
@@ -36,6 +42,7 @@ import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.web.rest.DiscoveryREST;
import org.apache.atlas.web.rest.EntityREST;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,28 +60,27 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-
@Guice(modules = {TestModules.TestOnlyModule.class})
public class TestEntitiesREST {
private static final Logger LOG = LoggerFactory.getLogger(TestEntitiesREST.class);
@Inject
- AtlasTypeRegistry typeRegistry;
-
+ private AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStore typeStore;
-
@Inject
- private EntityREST entityREST;
-
- private List<String> createdGuids = new ArrayList<>();
-
- private AtlasEntity dbEntity;
-
- private AtlasEntity tableEntity;
+ private DiscoveryREST discoveryREST;
+ @Inject
+ private EntityREST entityREST;
- private List<AtlasEntity> columns;
+ private AtlasEntity dbEntity;
+ private AtlasEntity tableEntity;
+ private AtlasEntity tableEntity2;
+ private List<AtlasEntity> columns;
+ private List<AtlasEntity> columns2;
+ private SearchParameters searchParameters = new SearchParameters();
+ private Map<String, List<String>> createdGuids = new HashMap<>();
@BeforeClass
public void setUp() throws Exception {
@@ -88,12 +94,17 @@ public class TestEntitiesREST {
}
}
- dbEntity = TestUtilsV2.createDBEntity();
- tableEntity = TestUtilsV2.createTableEntity(dbEntity);
+ dbEntity = TestUtilsV2.createDBEntity();
+ tableEntity = TestUtilsV2.createTableEntity(dbEntity);
+ tableEntity2 = TestUtilsV2.createTableEntity(dbEntity);
+
+ final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
+ final AtlasEntity colEntity2 = TestUtilsV2.createColumnEntity(tableEntity2);
+ columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
+ columns2 = new ArrayList<AtlasEntity>() {{ add(colEntity2); }};
- final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
- columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
tableEntity.setAttribute("columns", getObjIdList(columns));
+ tableEntity2.setAttribute("columns", getObjIdList(columns2));
}
@AfterMethod
@@ -107,35 +118,196 @@ public class TestEntitiesREST {
entities.addEntity(dbEntity);
entities.addEntity(tableEntity);
+ entities.addEntity(tableEntity2);
+
for (AtlasEntity column : columns) {
entities.addReferredEntity(column);
}
+ for (AtlasEntity column : columns2) {
+ entities.addReferredEntity(column);
+ }
+
EntityMutationResponse response = entityREST.createOrUpdate(entities);
List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
Assert.assertNotNull(guids);
- Assert.assertEquals(guids.size(), 3);
+ Assert.assertEquals(guids.size(), 5);
for (AtlasEntityHeader header : guids) {
- createdGuids.add(header.getGuid());
+ if (!createdGuids.containsKey(header.getTypeName())) {
+ createdGuids.put(header.getTypeName(), new ArrayList<>());
+ }
+ createdGuids.get(header.getTypeName()).add(header.getGuid());
}
}
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testTagToMultipleEntities() throws Exception{
AtlasClassification tag = new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }});
- ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids, tag);
+
+ // tag with table entities, leave rest for comparison
+ ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids.get(TABLE_TYPE), tag);
entityREST.addClassification(classificationAssociateRequest);
- for (String guid : createdGuids) {
- final AtlasClassification result_tag = entityREST.getClassification(guid, TestUtilsV2.CLASSIFICATION);
+
+ for (int i = 0; i < createdGuids.get(TABLE_TYPE).size() - 1; i++) {
+ final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(TABLE_TYPE).get(i), TestUtilsV2.CLASSIFICATION);
Assert.assertNotNull(result_tag);
Assert.assertEquals(result_tag, tag);
}
}
- @Test
+ @Test(dependsOnMethods = "testTagToMultipleEntities")
+ public void testBasicSearchWithSub() throws Exception {
+ // search entities with classification named classification
+ searchParameters = new SearchParameters();
+ searchParameters.setIncludeSubClassifications(true);
+ searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+ }
+
+ @Test(dependsOnMethods = "testTagToMultipleEntities")
+ public void testWildCardBasicSearch() throws Exception {
+ searchParameters = new SearchParameters();
+
+ searchParameters.setClassification("*");
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters.setClassification("_CLASSIFIED");
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ // Test wildcard usage of basic search
+ searchParameters.setClassification("cl*");
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters.setClassification("*ion");
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters.setClassification("*l*");
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters.setClassification("_NOT_CLASSIFIED");
+ searchParameters.setTypeName(DATABASE_TYPE);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 1);
+ }
+
+ @Test(dependsOnMethods = "testWildCardBasicSearch")
+ public void testBasicSearchAddCls() throws Exception {
+ AtlasClassification cls = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
+ put("stringAttr", "sample_string");
+ put("booleanAttr", true);
+ put("integerAttr", 100);
+ }});
+
+ ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), cls);
+ entityREST.addClassification(clsAssRequest);
+
+ final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.PHI);
+ Assert.assertNotNull(result_tag);
+
+ // search entities associated with phi
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification(TestUtilsV2.PHI);
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 1);
+ }
+
+ private void addPHICls() throws Exception {
+ AtlasClassification clsPHI = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
+ put("stringAttr", "string");
+ put("booleanAttr", true);
+ put("integerAttr", 100);
+ }});
+
+ // add clsPHI to col entities
+ ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(COLUMN_TYPE), clsPHI);
+ entityREST.addClassification(clsAssRequest);
+
+ final AtlasClassification result_PHI = entityREST.getClassification(createdGuids.get(COLUMN_TYPE).get(0), TestUtilsV2.PHI);
+ Assert.assertNotNull(result_PHI);
+ }
+
+ @Test(dependsOnMethods = "testBasicSearchAddCls")
+ public void testBasicSearch() throws Exception{
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification("PH*");
+ searchParameters.setTypeName(DATABASE_TYPE);
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 1);
+
+ addPHICls();
+
+ // basic search with tag filterCriteria
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification("PHI");
+
+ SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria();
+ filterCriteria.setAttributeName("stringAttr");
+ filterCriteria.setOperator(SearchParameters.Operator.CONTAINS);
+ filterCriteria.setAttributeValue("str");
+ searchParameters.setTagFilters(filterCriteria);
+
+ res = discoveryREST.searchWithParameters(searchParameters);
+
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 3);
+
+ filterCriteria.setAttributeName("stringAttr");
+ filterCriteria.setOperator(SearchParameters.Operator.EQ);
+ filterCriteria.setAttributeValue("string");
+
+ res = discoveryREST.searchWithParameters(searchParameters);
+
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+ }
+
+ @Test(dependsOnMethods = "testWildCardBasicSearch")
+ public void testBasicSearchWithSubTypes() throws Exception{
+ AtlasClassification fetlCls = new AtlasClassification(TestUtilsV2.FETL_CLASSIFICATION, new HashMap<String, Object>() {{
+ put("tag", "sample_tag");
+ }});
+
+ ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), fetlCls);
+ entityREST.addClassification(clsAssRequest);
+
+ final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.PHI);
+ Assert.assertNotNull(result_tag);
+
+ // basic search with subtypes
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
+ searchParameters.setIncludeSubTypes(true);
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 3);
+ }
+
+ @Test(dependsOnMethods = "testBasicSearchWithSubTypes")
public void testUpdateWithSerializedEntities() throws Exception {
+
//Check with serialization and deserialization of entity attributes for the case
// where attributes which are de-serialized into a map
AtlasEntity dbEntity = TestUtilsV2.createDBEntity();
@@ -165,11 +337,11 @@ public class TestEntitiesREST {
@Test(dependsOnMethods = "testCreateOrUpdateEntities")
public void testGetEntities() throws Exception {
- final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids, false, false);
+ final AtlasEntitiesWithExtInfo response = entityREST.getByGuids(createdGuids.get(DATABASE_TYPE), false, false);
final List<AtlasEntity> entities = response.getEntities();
Assert.assertNotNull(entities);
- Assert.assertEquals(entities.size(), 3);
+ Assert.assertEquals(entities.size(), 1);
verifyAttributes(entities);
}
@@ -192,11 +364,11 @@ public class TestEntitiesREST {
AtlasEntity retrievedTableEntity = null;
AtlasEntity retrievedColumnEntity = null;
for (AtlasEntity entity: retrievedEntities ) {
- if ( entity.getTypeName().equals(TestUtilsV2.DATABASE_TYPE)) {
+ if ( entity.getTypeName().equals(DATABASE_TYPE)) {
retrievedDBEntity = entity;
}
- if ( entity.getTypeName().equals(TestUtilsV2.TABLE_TYPE)) {
+ if ( entity.getTypeName().equals(TABLE_TYPE)) {
retrievedTableEntity = entity;
}
@@ -261,4 +433,4 @@ public class TestEntitiesREST {
return ret;
}
-}
+}
\ No newline at end of file
diff --git a/webapp/src/test/resources/json/search-parameters/tag-filters.json b/webapp/src/test/resources/json/search-parameters/tag-filters.json
index 829dc3b..5e74328 100644
--- a/webapp/src/test/resources/json/search-parameters/tag-filters.json
+++ b/webapp/src/test/resources/json/search-parameters/tag-filters.json
@@ -12,22 +12,5 @@
"classification":"fooTag"
},
"expectedCount": 1
- },
- {
- "testDescription": "Search for exact Tag name order by",
- "searchParameters": {
- "entityFilters":null,
- "tagFilters":null,
- "attributes":null,
- "query":null,
- "excludeDeletedEntities":true,
- "limit":25,
- "typeName":null,
- "sortBy": "name",
- "sortOrder": "DESCENDING",
- "classification":"fooTag"
- },
- "expectedCount": 1
}
-
]
\ No newline at end of file