You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/07/24 15:57:27 UTC
[22/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/377fe19f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/377fe19f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/377fe19f
Branch: refs/heads/feature-odf
Commit: 377fe19f023a4036f9fa1503dec6ea698fdb214b
Parents: 71f9c06
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Jul 18 00:25:36 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 18 08:05:53 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntitySearchProcessor.java | 73 ++++++++++----------
.../apache/atlas/discovery/SearchContext.java | 2 +-
.../apache/atlas/discovery/SearchProcessor.java | 14 +++-
3 files changed, 50 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 605cb15..3204ecf 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,7 +20,6 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
-import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -34,60 +33,70 @@ public class EntitySearchProcessor extends SearchProcessor {
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("EntitySearchProcessor");
private final AtlasIndexQuery indexQuery;
- private final AtlasGraphQuery partialGraphQuery;
- private final AtlasGraphQuery allGraphQuery;
+ private final AtlasGraphQuery graphQuery;
+ private final AtlasGraphQuery filterGraphQuery;
public EntitySearchProcessor(SearchContext context) {
super(context);
- AtlasEntityType entityType = context.getEntityType();
- AtlasClassificationType classificationType = context.getClassificationType();
- FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- Set<String> solrAttributes = new HashSet<>();
- Set<String> gremlinAttributes = new HashSet<>();
- Set<String> allAttributes = new HashSet<>();
+ final AtlasEntityType entityType = context.getEntityType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
- boolean useSolrSearch = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY && canApplySolrFilter(entityType, filterCriteria, false);
+ final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+ final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
- if (useSolrSearch) {
- StringBuilder solrQuery = new StringBuilder();
+ StringBuilder solrQuery = new StringBuilder();
+ if (typeSearchBySolr) {
constructTypeTestQuery(solrQuery, typeAndSubTypes);
+ }
+
+ if (attrSearchBySolr) {
constructFilterQuery(solrQuery, entityType, filterCriteria, solrAttributes);
+ } else {
+ gremlinAttributes.addAll(solrAttributes);
+ }
+ if (solrQuery.length() > 0) {
String solrQueryString = STRAY_AND_PATTERN.matcher(solrQuery).replaceAll(")");
solrQueryString = STRAY_OR_PATTERN.matcher(solrQueryString).replaceAll(")");
solrQueryString = STRAY_ELIPSIS_PATTERN.matcher(solrQueryString).replaceAll("");
indexQuery = context.getGraph().indexQuery(Constants.VERTEX_INDEX, solrQueryString);
+ } else {
+ indexQuery = null;
+ }
- if (CollectionUtils.isNotEmpty(gremlinAttributes) || classificationType != null) {
- AtlasGraphQuery query = context.getGraph().query();
+ if (CollectionUtils.isNotEmpty(gremlinAttributes) || !typeSearchBySolr) {
+ AtlasGraphQuery query = context.getGraph().query();
- addClassificationNameConditionIfNecessary(query);
+ if (!typeSearchBySolr) {
+ query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+ }
+
+ graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
- partialGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
- } else {
- partialGraphQuery = null;
+ if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
+ graphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
}
} else {
- indexQuery = null;
- partialGraphQuery = null;
+ graphQuery = null;
}
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
- addClassificationNameConditionIfNecessary(query);
-
- allGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
+ filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
- allGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
+ filterGraphQuery.has(Constants.STATE_PROPERTY_KEY, "ACTIVE");
}
}
@@ -128,15 +137,15 @@ public class EntitySearchProcessor extends SearchProcessor {
vertices = getVerticesFromIndexQueryResult(queryResult);
- if (partialGraphQuery != null) {
+ if (graphQuery != null) {
AtlasGraphQuery guidQuery = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(vertices));
- guidQuery.addConditionsFrom(partialGraphQuery);
+ guidQuery.addConditionsFrom(graphQuery);
vertices = getVertices(guidQuery.vertices().iterator());
}
} else {
- Iterator<AtlasVertex> queryResult = allGraphQuery.vertices(qryOffset, limit).iterator();
+ Iterator<AtlasVertex> queryResult = graphQuery.vertices(qryOffset, limit).iterator();
if (!queryResult.hasNext()) { // no more results from query - end of search
break;
@@ -182,7 +191,7 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery query = context.getGraph().query().in(Constants.GUID_PROPERTY_KEY, getGuids(entityVertices));
- query.addConditionsFrom(allGraphQuery);
+ query.addConditionsFrom(filterGraphQuery);
List<AtlasVertex> ret = getVertices(query.vertices().iterator());
@@ -194,10 +203,4 @@ public class EntitySearchProcessor extends SearchProcessor {
return ret;
}
-
- private void addClassificationNameConditionIfNecessary(AtlasGraphQuery query) {
- if (context.getClassificationType() != null && !context.needClassificationProcessor()) {
- query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, context.getClassificationType().getTypeAndAllSubTypes());
- }
- }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 55a07f3..8dd7667 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -109,7 +109,7 @@ public class SearchContext {
}
public boolean needClassificationProcessor() {
- return classificationType != null && (hasAttributeFilter(searchParameters.getTagFilters()) || entityType == null);
+ return classificationType != null;
}
public boolean needEntityProcessor() {
http://git-wip-us.apache.org/repos/asf/atlas/blob/377fe19f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 1481809..ff0bd2e 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -88,7 +88,7 @@ public abstract class SearchProcessor {
public abstract List<AtlasVertex> execute();
public List<AtlasVertex> filter(List<AtlasVertex> entityVertices) {
- return nextProcessor == null ? entityVertices : nextProcessor.filter(entityVertices);
+ return nextProcessor == null || CollectionUtils.isEmpty(entityVertices) ? entityVertices : nextProcessor.filter(entityVertices);
}
@@ -193,12 +193,20 @@ public abstract class SearchProcessor {
String filterQuery = toSolrQuery(type, filterCriteria, solrAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
- solrQuery.append(AND_STR).append(filterQuery);
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append(filterQuery);
}
}
if (type instanceof AtlasEntityType && context.getSearchParameters().getExcludeDeletedEntities()) {
- solrQuery.append(AND_STR).append("v.\"__state\":").append("ACTIVE");
+ if (solrQuery.length() > 0) {
+ solrQuery.append(AND_STR);
+ }
+
+ solrQuery.append("v.\"__state\":").append("ACTIVE");
}
}