You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/07/24 15:57:28 UTC
[23/39] atlas git commit: ATLAS-1961: Basic search improvement in use
of Solr index for attribute filtering (# 2)
ATLAS-1961: Basic search improvement in use of Solr index for attribute filtering (# 2)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/cfb6b84f
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/cfb6b84f
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/cfb6b84f
Branch: refs/heads/feature-odf
Commit: cfb6b84f41b05275db826bbd43e4c39145b6d2d5
Parents: 377fe19
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Tue Jul 18 13:27:46 2017 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue Jul 18 16:59:02 2017 -0700
----------------------------------------------------------------------
.../atlas/discovery/EntitySearchProcessor.java | 30 +++++++++++-----
.../discovery/FullTextSearchProcessor.java | 36 ++++++++++++++++++--
.../apache/atlas/discovery/SearchContext.java | 9 +++--
.../apache/atlas/discovery/SearchProcessor.java | 4 +--
4 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 3204ecf..50376ef 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -20,6 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
+import org.apache.atlas.type.AtlasClassificationType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -39,18 +40,21 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final AtlasEntityType entityType = context.getEntityType();
- final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
- final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- final Set<String> solrAttributes = new HashSet<>();
- final Set<String> gremlinAttributes = new HashSet<>();
- final Set<String> allAttributes = new HashSet<>();
+ final AtlasEntityType entityType = context.getEntityType();
+ final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
+ final Set<String> typeAndSubTypes = entityType.getTypeAndAllSubTypes();
+ final Set<String> solrAttributes = new HashSet<>();
+ final Set<String> gremlinAttributes = new HashSet<>();
+ final Set<String> allAttributes = new HashSet<>();
+
+ final AtlasClassificationType classificationType = context.getClassificationType();
+ final boolean filterClassification = classificationType != null && !context.needClassificationProcessor();
processSearchAttributes(entityType, filterCriteria, solrAttributes, gremlinAttributes, allAttributes);
- final boolean typeSearchBySolr = typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
- final boolean attrSearchBySolr = canApplySolrFilter(entityType, filterCriteria, false);
+ final boolean typeSearchBySolr = !filterClassification && typeAndSubTypes.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY;
+ final boolean attrSearchBySolr = !filterClassification && CollectionUtils.isNotEmpty(solrAttributes) && canApplySolrFilter(entityType, filterCriteria, false);
StringBuilder solrQuery = new StringBuilder();
@@ -82,6 +86,10 @@ public class EntitySearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
+ if (filterClassification) {
+ query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+ }
+
graphQuery = toGremlinFilterQuery(entityType, filterCriteria, gremlinAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities() && indexQuery == null) {
@@ -93,6 +101,10 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery query = context.getGraph().query().in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+ if (filterClassification) {
+ query.in(Constants.TRAIT_NAMES_PROPERTY_KEY, classificationType.getTypeAndAllSubTypes());
+ }
+
filterGraphQuery = toGremlinFilterQuery(entityType, filterCriteria, allAttributes, query);
if (context.getSearchParameters().getExcludeDeletedEntities()) {
@@ -115,7 +127,7 @@ public class EntitySearchProcessor extends SearchProcessor {
}
try {
- int qryOffset = (nextProcessor == null) ? context.getSearchParameters().getOffset() : 0;
+ int qryOffset = (nextProcessor == null && (graphQuery == null || indexQuery == null)) ? context.getSearchParameters().getOffset() : 0;
int limit = context.getSearchParameters().getLimit();
int resultIdx = qryOffset;
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 4ddd642..83368c2 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -22,12 +22,14 @@ import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
public class FullTextSearchProcessor extends SearchProcessor {
@@ -40,9 +42,39 @@ public class FullTextSearchProcessor extends SearchProcessor {
super(context);
SearchParameters searchParameters = context.getSearchParameters();
- String queryString = String.format("v.\"%s\":(%s)", Constants.ENTITY_TEXT_PROPERTY_KEY, searchParameters.getQuery());
+ StringBuilder queryString = new StringBuilder();
- indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString);
+ queryString.append("v.\"").append(Constants.ENTITY_TEXT_PROPERTY_KEY).append("\":(").append(searchParameters.getQuery());
+
+ // if search includes entity-type criteria, adding a filter here can help avoid unnecessary
+ // processing (and rejection) by subsequent EntitySearchProcessor
+ if (context.getEntityType() != null) {
+ Set<String> typeAndSubTypeNames = context.getEntityType().getTypeAndAllSubTypes();
+
+ if (typeAndSubTypeNames.size() <= MAX_ENTITY_TYPES_IN_INDEX_QUERY) {
+ queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ } else {
+ LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
+ context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ }
+ }
+
+ // if search includes classification criteria, adding a filter here can help avoid unnecessary
+ // processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
+ if (context.getClassificationType() != null) {
+ Set<String> typeAndSubTypeNames = context.getClassificationType().getTypeAndAllSubTypes();
+
+ if (typeAndSubTypeNames.size() <= MAX_CLASSIFICATION_TYPES_IN_INDEX_QUERY) {
+ queryString.append(AND_STR).append("(").append(StringUtils.join(typeAndSubTypeNames, SPACE_STRING)).append(")");
+ } else {
+ LOG.warn("'{}' has too many subtypes ({}) to include in index-query; might cause poor performance",
+ context.getEntityType().getTypeName(), typeAndSubTypeNames.size());
+ }
+ }
+
+ queryString.append(")");
+
+ indexQuery = context.getGraph().indexQuery(Constants.FULLTEXT_INDEX, queryString.toString());
}
@Override
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 8dd7667..929f8d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -61,7 +61,6 @@ public class SearchContext {
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
-
}
}
@@ -104,15 +103,15 @@ public class SearchContext {
return toString(new StringBuilder()).toString();
}
- public boolean needFullTextrocessor() {
+ boolean needFullTextrocessor() {
return StringUtils.isNotEmpty(searchParameters.getQuery());
}
- public boolean needClassificationProcessor() {
- return classificationType != null;
+ boolean needClassificationProcessor() {
+ return classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()));
}
- public boolean needEntityProcessor() {
+ boolean needEntityProcessor() {
return entityType != null;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/cfb6b84f/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index ff0bd2e..596b43b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -181,7 +181,7 @@ public abstract class SearchProcessor {
protected void constructTypeTestQuery(StringBuilder solrQuery, Set<String> typeAndAllSubTypes) {
String typeAndSubtypesString = StringUtils.join(typeAndAllSubTypes, SPACE_STRING);
- solrQuery.append("v.\"__typeName\": (")
+ solrQuery.append("v.\"").append(Constants.TYPE_NAME_PROPERTY_KEY).append("\": (")
.append(typeAndSubtypesString)
.append(")");
}
@@ -206,7 +206,7 @@ public abstract class SearchProcessor {
solrQuery.append(AND_STR);
}
- solrQuery.append("v.\"__state\":").append("ACTIVE");
+ solrQuery.append("v.\"").append(Constants.STATE_PROPERTY_KEY).append("\":ACTIVE");
}
}