You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ni...@apache.org on 2020/08/27 14:12:07 UTC
[atlas] branch branch-2.0 updated (dfec483 -> 897c2fc)
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a change to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from dfec483 ATLAS-3925: UI: Properties with value 0 or false are displayed as N/A for
new 11b73ee ATLAS-3848 : Quick Search : Fixed incorrect aggregation metrics
new 01d8c3a ATLAS-3838: Support multiple tag/classification in basic/quick search API ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types
new 2a7d321 ATLAS-3880 : BasicSearch: Multiple type/tag: Log invalid and allow searching valid type/tag names
new e97fb30 ATLAS-3888 : BasicSearch: Multiple type/tag: [Regression], Restrict when tag filters and regex in tag
new 897c2fc ATLAS-3782 : Support NOT_CONTAINS operator in basic search
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../repository/graphdb/AggregationContext.java | 38 +--
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 2 +-
.../graphdb/janus/AtlasSolrQueryBuilder.java | 174 ++++++----
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 54 +++-
...arameters0.json => searchparameters2Types.json} | 7 +-
.../org/apache/atlas/type/AtlasTypeRegistry.java | 18 ++
.../discovery/ClassificationSearchProcessor.java | 134 +++-----
.../atlas/discovery/EntityDiscoveryService.java | 8 +-
.../atlas/discovery/EntitySearchProcessor.java | 53 ++--
.../atlas/discovery/FreeTextSearchProcessor.java | 7 +-
.../atlas/discovery/FullTextSearchProcessor.java | 19 +-
.../atlas/discovery/GraphIndexQueryBuilder.java | 25 +-
.../atlas/discovery/SearchAggregatorImpl.java | 7 +-
.../org/apache/atlas/discovery/SearchContext.java | 266 ++++++++++++----
.../apache/atlas/discovery/SearchProcessor.java | 350 +++++++++++++--------
.../test/java/org/apache/atlas/BasicTestSetup.java | 43 ++-
.../atlas/discovery/AtlasDiscoveryServiceTest.java | 338 ++++++++++++++++++++
...java => ClassificationSearchProcessorTest.java} | 175 ++++++++---
.../atlas/discovery/EntitySearchProcessorTest.java | 173 +++++++++-
.../org/apache/atlas/web/rest/DiscoveryREST.java | 11 +-
20 files changed, 1399 insertions(+), 503 deletions(-)
copy graphdb/janus/src/test/resources/{searchparameters0.json => searchparameters2Types.json} (74%)
create mode 100644 repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
rename repository/src/test/java/org/apache/atlas/discovery/{BasicSearchClassificationTest.java => ClassificationSearchProcessorTest.java} (52%)
[atlas] 02/05: ATLAS-3838: Support multiple tag/classification in
basic/quick search API ATLAS-3652: Quick Search: API requirement for GET
request on multiple entity types
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 01d8c3abffd053024ed530a4686812fe2de5777f
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Fri Jun 26 11:24:46 2020 +0530
ATLAS-3838: Support multiple tag/classification in basic/quick search API
ATLAS-3652: Quick Search: API requirement for GET request on multiple entity types
Signed-off-by: Sarath Subramanian <sa...@apache.org>
both JIRA's addressed in the same commit.
(cherry picked from commit 8b5cb9d9aa9f977ba5dfc455dfe440dd55eace06)
---
.../repository/graphdb/AggregationContext.java | 38 +--
.../graphdb/janus/AtlasJanusGraphIndexClient.java | 2 +-
.../graphdb/janus/AtlasSolrQueryBuilder.java | 151 +++++----
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 54 +++-
.../src/test/resources/searchparameters2Types.json | 19 ++
.../discovery/ClassificationSearchProcessor.java | 34 +-
.../atlas/discovery/EntityDiscoveryService.java | 8 +-
.../atlas/discovery/EntitySearchProcessor.java | 53 ++--
.../atlas/discovery/FreeTextSearchProcessor.java | 7 +-
.../atlas/discovery/FullTextSearchProcessor.java | 19 +-
.../atlas/discovery/GraphIndexQueryBuilder.java | 25 +-
.../atlas/discovery/SearchAggregatorImpl.java | 7 +-
.../org/apache/atlas/discovery/SearchContext.java | 253 +++++++++++----
.../apache/atlas/discovery/SearchProcessor.java | 343 +++++++++++++--------
.../atlas/discovery/EntitySearchProcessorTest.java | 114 ++++++-
15 files changed, 750 insertions(+), 377 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
index 6006fef..efde79b 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AggregationContext.java
@@ -25,14 +25,14 @@ import java.util.Map;
import java.util.Set;
public class AggregationContext {
- private final String queryString;
- private final FilterCriteria filterCriteria;
- private final AtlasEntityType searchForEntityType;
- private final Set<String> aggregationFieldNames;
- private final Set<AtlasAttribute> aggregationAttributes;
- private final Map<String, String> indexFieldNameCache;
- private final boolean excludeDeletedEntities;
- private final boolean includeSubTypes;
+ private final String queryString;
+ private final FilterCriteria filterCriteria;
+ private final Set<AtlasEntityType> searchForEntityTypes;
+ private final Set<String> aggregationFieldNames;
+ private final Set<AtlasAttribute> aggregationAttributes;
+ private final Map<String, String> indexFieldNameCache;
+ private final boolean excludeDeletedEntities;
+ private final boolean includeSubTypes;
/**
* @param queryString the query string whose aggregation metrics need to be retrieved.
@@ -41,17 +41,17 @@ public class AggregationContext {
* @param indexFieldNameCache
* @param excludeDeletedEntities a boolean flag to indicate if the deleted entities need to be excluded in search
*/
- public AggregationContext(String queryString,
- FilterCriteria filterCriteria,
- AtlasEntityType searchForEntityType,
- Set<String> aggregationFieldNames,
- Set<AtlasAttribute> aggregationAttributes,
- Map<String, String> indexFieldNameCache,
- boolean excludeDeletedEntities,
- boolean includeSubTypes) {
+ public AggregationContext(String queryString,
+ FilterCriteria filterCriteria,
+ Set<AtlasEntityType> searchForEntityType,
+ Set<String> aggregationFieldNames,
+ Set<AtlasAttribute> aggregationAttributes,
+ Map<String, String> indexFieldNameCache,
+ boolean excludeDeletedEntities,
+ boolean includeSubTypes) {
this.queryString = queryString;
this.filterCriteria = filterCriteria;
- this.searchForEntityType = searchForEntityType;
+ this.searchForEntityTypes = searchForEntityType;
this.aggregationFieldNames = aggregationFieldNames;
this.aggregationAttributes = aggregationAttributes;
this.indexFieldNameCache = indexFieldNameCache;
@@ -67,8 +67,8 @@ public class AggregationContext {
return filterCriteria;
}
- public AtlasEntityType getSearchForEntityType() {
- return searchForEntityType;
+ public Set<AtlasEntityType> getSearchForEntityTypes() {
+ return searchForEntityTypes;
}
public Set<String> getAggregationFieldNames() {
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
index 29bd2a4..8142514 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphIndexClient.java
@@ -175,7 +175,7 @@ public class AtlasJanusGraphIndexClient implements AtlasGraphIndexClient {
Map<String, String> indexFieldName2PropertyKeyNameMap = new HashMap<>();
AtlasSolrQueryBuilder solrQueryBuilder = new AtlasSolrQueryBuilder();
- solrQueryBuilder.withEntityType(aggregationContext.getSearchForEntityType())
+ solrQueryBuilder.withEntityTypes(aggregationContext.getSearchForEntityTypes())
.withQueryString(aggregationContext.getQueryString())
.withCriteria(aggregationContext.getFilterCriteria())
.withExcludedDeletedEntities(aggregationContext.isExcludeDeletedEntities())
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index b8fd4a0..6c06a3c 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -29,31 +29,28 @@ import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
+import java.util.*;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
public class AtlasSolrQueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
- private AtlasEntityType entityType;
- private String queryString;
- private FilterCriteria criteria;
- private boolean excludeDeletedEntities;
- private boolean includeSubtypes;
- private Map<String, String> indexFieldNameCache;
- public static final char CUSTOM_ATTR_SEPARATOR = '=';
- public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
+ private Set<AtlasEntityType> entityTypes;
+ private String queryString;
+ private FilterCriteria criteria;
+ private boolean excludeDeletedEntities;
+ private boolean includeSubtypes;
+ private Map<String, String> indexFieldNameCache;
+ public static final char CUSTOM_ATTR_SEPARATOR = '=';
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() {
}
- public AtlasSolrQueryBuilder withEntityType(AtlasEntityType searchForEntityType) {
- this.entityType = searchForEntityType;
+ public AtlasSolrQueryBuilder withEntityTypes(Set<AtlasEntityType> searchForEntityTypes) {
+ this.entityTypes = searchForEntityTypes;
return this;
}
@@ -112,7 +109,7 @@ public class AtlasSolrQueryBuilder {
isAndNeeded = true;
}
- if (entityType != null) {
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
if (isAndNeeded) {
queryBuilder.append(" AND ");
}
@@ -140,29 +137,25 @@ public class AtlasSolrQueryBuilder {
}
private void buildForEntityType(StringBuilder queryBuilder) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Search is being done for entities of type {}", entityType.getTypeName());
- }
String typeIndexFieldName = indexFieldNameCache.get(Constants.ENTITY_TYPE_PROPERTY_KEY);
queryBuilder.append(" +")
.append(typeIndexFieldName)
- .append(":(")
- .append(entityType.getTypeName())
- .append(" ");
+ .append(":(");
- if (includeSubtypes) {
- Set<String> allSubTypes = entityType.getAllSubTypes();
+ Set<String> typesToSearch = new HashSet<>();
+ for (AtlasEntityType type : entityTypes) {
- if(allSubTypes.size() != 0 ) {
- for(String subTypeName: allSubTypes) {
- queryBuilder.append(subTypeName).append(" ");
- }
+ if (includeSubtypes) {
+ typesToSearch.addAll(type.getTypeAndAllSubTypes());
+ } else {
+ typesToSearch.add(type.getTypeName());
}
}
- queryBuilder.append(" ) ");
+ queryBuilder.append(StringUtils.join(typesToSearch, " ")).append(" ) ");
+
}
private void dropDeletedEntities(StringBuilder queryBuilder) throws AtlasBaseException {
@@ -173,9 +166,8 @@ public class AtlasSolrQueryBuilder {
String indexFieldName = indexFieldNameCache.get(Constants.STATE_PROPERTY_KEY);
if (indexFieldName == null) {
- String msg = String.format("There is no index field name defined for attribute '%s' for entity '%s'",
- Constants.STATE_PROPERTY_KEY,
- entityType.getTypeName());
+ String msg = String.format("There is no index field name defined for attribute '%s'",
+ Constants.STATE_PROPERTY_KEY);
LOG.error(msg);
@@ -187,10 +179,46 @@ public class AtlasSolrQueryBuilder {
private AtlasSolrQueryBuilder withCriteria(StringBuilder queryBuilder, FilterCriteria criteria) throws AtlasBaseException {
List<FilterCriteria> criterion = criteria.getCriterion();
+ Set<String> indexAttributes = new HashSet<>();
+ if (StringUtils.isNotEmpty(criteria.getAttributeName()) && CollectionUtils.isEmpty(criterion)) { // no child criterion
+
+ String attributeName = criteria.getAttributeName();
+ String attributeValue = criteria.getAttributeValue();
+ Operator operator = criteria.getOperator();
+
+ ArrayList<StringBuilder> orExpQuery = new ArrayList<>();
+
+ for (AtlasEntityType type : entityTypes) {
+ String indexAttributeName = getIndexAttributeName(type, attributeName);
+
+ //check to remove duplicate attribute query (for eg. name)
+ if (!indexAttributes.contains(indexAttributeName)) {
+ StringBuilder sb = new StringBuilder();
+
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
+ // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
+ // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
+ // surrounding inverted commas to attributeValue works
+ operator = Operator.EQ;
+ attributeValue = getIndexQueryAttributeValue(attributeValue);
+ }
+
+ withPropertyCondition(sb, indexAttributeName, operator, attributeValue);
+ indexAttributes.add(indexAttributeName);
+ orExpQuery.add(sb);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(orExpQuery)) {
+ if (orExpQuery.size() > 1) {
+ String orExpStr = StringUtils.join(orExpQuery, FilterCriteria.Condition.OR.name());
+ queryBuilder.append(" ( ").append(orExpStr).append(" ) ");
+ } else {
+ queryBuilder.append(orExpQuery.iterator().next());
+ }
+ }
- if(criterion == null || CollectionUtils.isEmpty(criteria.getCriterion())) { // no child criterion
- withPropertyCondition(queryBuilder, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
- } else {
+ } else if (CollectionUtils.isNotEmpty(criterion)) {
beginCriteria(queryBuilder);
for (Iterator<FilterCriteria> iterator = criterion.iterator(); iterator.hasNext(); ) {
@@ -209,40 +237,12 @@ public class AtlasSolrQueryBuilder {
return this;
}
- private void withPropertyCondition(StringBuilder queryBuilder, String attributeName, Operator operator, String attributeValue) throws AtlasBaseException {
- if (StringUtils.isNotEmpty(attributeName) && operator != null) {
+ private void withPropertyCondition(StringBuilder queryBuilder, String indexFieldName, Operator operator, String attributeValue) throws AtlasBaseException {
+ if (StringUtils.isNotEmpty(indexFieldName) && operator != null) {
if (attributeValue != null) {
attributeValue = attributeValue.trim();
}
- if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
- // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
- // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
- // surrounding inverted commas to attributeValue works
- operator = Operator.EQ;
- attributeValue = getIndexQueryAttributeValue(attributeValue);
- }
-
- AtlasAttribute attribute = entityType.getAttribute(attributeName);
-
- if (attribute == null) {
- String msg = String.format("Received unknown attribute '%s' for type '%s'.", attributeName, entityType.getTypeName());
-
- LOG.error(msg);
-
- throw new AtlasBaseException(msg);
- }
-
- String indexFieldName = attribute.getIndexFieldName();
-
- if (indexFieldName == null) {
- String msg = String.format("Received non-index attribute %s for type %s.", attributeName, entityType.getTypeName());
-
- LOG.error(msg);
-
- throw new AtlasBaseException(msg);
- }
-
beginCriteria(queryBuilder);
switch (operator) {
@@ -308,6 +308,29 @@ public class AtlasSolrQueryBuilder {
return attributeValue;
}
+ private String getIndexAttributeName(AtlasEntityType type, String attrName) throws AtlasBaseException {
+ AtlasAttribute ret = type.getAttribute(attrName);
+
+ if (ret == null) {
+ String msg = String.format("Received unknown attribute '%s' for type '%s'.", attrName, type.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ String indexFieldName = ret.getIndexFieldName();
+
+ if (indexFieldName == null) {
+ String msg = String.format("Received non-index attribute %s for type %s.", attrName, type.getTypeName());
+
+ LOG.error(msg);
+
+ throw new AtlasBaseException(msg);
+ }
+
+ return indexFieldName;
+ }
private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( ");
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
index 60d8610..06c7221 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
@@ -33,8 +33,10 @@ import org.testng.annotations.Test;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.mockito.Mockito.when;
@@ -45,6 +47,9 @@ public class AtlasSolrQueryBuilderTest {
private AtlasEntityType hiveTableEntityTypeMock;
@Mock
+ private AtlasEntityType hiveTableEntityTypeMock2;
+
+ @Mock
private AtlasStructType.AtlasAttribute nameAttributeMock;
@Mock
@@ -95,6 +100,7 @@ public class AtlasSolrQueryBuilderTest {
when(hiveTableEntityTypeMock.getTypeName()).thenReturn("hive_table");
+ when(hiveTableEntityTypeMock2.getTypeName()).thenReturn("hive_db");
when(nameAttributeMock.getIndexFieldName()).thenReturn("name_index");
when(commentAttributeMock.getIndexFieldName()).thenReturn("comment_index");
@@ -114,7 +120,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
}
@Test
@@ -124,7 +130,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
@@ -134,7 +140,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
}
@Test
@@ -144,7 +150,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
@@ -154,7 +160,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
}
@Test
@@ -164,7 +170,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
}
@Test
@@ -174,7 +180,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
}
@Test
@@ -184,7 +190,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
}
@Test
@@ -194,7 +200,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
}
@Test
@@ -204,7 +210,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
}
@Test
@@ -214,9 +220,18 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
+ Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
}
+ @Test
+ public void testGenerateSolrQueryString2TypeNames() throws IOException, AtlasBaseException {
+ final String fileName = "src/test/resources/searchparameters2Types.json";
+ AtlasSolrQueryBuilder underTest = new AtlasSolrQueryBuilder();
+
+ processSearchParametersForMultipleTypeNames(fileName, underTest);
+
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table hive_db ) ");
+ }
@@ -243,8 +258,23 @@ public class AtlasSolrQueryBuilderTest {
ObjectMapper mapper = new ObjectMapper();
SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
+ Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
+ underTest.withEntityTypes(hiveTableEntityTypeMocks)
+ .withQueryString(searchParameters.getQuery())
+ .withCriteria(searchParameters.getEntityFilters())
+ .withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
+ .withCommonIndexFieldNames(indexFieldNamesMap);
+ }
+
+ private void processSearchParametersForMultipleTypeNames(String fileName, AtlasSolrQueryBuilder underTest) throws IOException, AtlasBaseException {
+ ObjectMapper mapper = new ObjectMapper();
+ SearchParameters searchParameters = mapper.readValue(new FileInputStream(fileName), SearchParameters.class);
- underTest.withEntityType(hiveTableEntityTypeMock)
+ Set<AtlasEntityType> hiveTableEntityTypeMocks = new HashSet<>();
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock);
+ hiveTableEntityTypeMocks.add(hiveTableEntityTypeMock2);
+ underTest.withEntityTypes(hiveTableEntityTypeMocks)
.withQueryString(searchParameters.getQuery())
.withCriteria(searchParameters.getEntityFilters())
.withExcludedDeletedEntities(searchParameters.getExcludeDeletedEntities())
diff --git a/graphdb/janus/src/test/resources/searchparameters2Types.json b/graphdb/janus/src/test/resources/searchparameters2Types.json
new file mode 100644
index 0000000..77fc70f
--- /dev/null
+++ b/graphdb/janus/src/test/resources/searchparameters2Types.json
@@ -0,0 +1,19 @@
+
+{
+ "excludeDeletedEntities":true,
+ "includeSubClassifications":true,
+ "includeSubTypes":true,
+ "includeClassificationAttributes":true,
+ "entityFilters":{
+ },
+ "tagFilters":null,
+ "attributes":[
+ "comment"
+ ],
+ "query":"t",
+ "limit":25,
+ "offset":0,
+ "typeName":"hive_table,hive_db",
+ "classification":null,
+ "termName":null
+}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 5dd0d7f..9c72cd4 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -67,17 +67,16 @@ public class ClassificationSearchProcessor extends SearchProcessor {
public ClassificationSearchProcessor(SearchContext context) {
super(context);
- final AtlasClassificationType classificationType = context.getClassificationType();
final FilterCriteria filterCriteria = context.getSearchParameters().getTagFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes = context.getClassificationTypes();
+ final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
final String typeAndSubTypesQryStr = context.getClassificationTypesQryStr();
- final boolean isBuiltInType = context.isBuiltInClassificationType();
final boolean isWildcardSearch = context.isWildCardSearch();
+ final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
- processSearchAttributes(classificationType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
+ processSearchAttributes(classificationTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
/* for classification search, if any attribute can't be handled by index query - switch to all filter by Graph query
There are four cases in the classification type :
@@ -87,19 +86,25 @@ public class ClassificationSearchProcessor extends SearchProcessor {
4. classification is not present in the search parameter
each of above cases with either has empty/or not tagFilters
*/
- final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) &&
+ final boolean useIndexSearchForEntity = (CollectionUtils.isNotEmpty(classificationTypes) || isWildcardSearch) &&
!context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index
* to query classification index as well.
*/
- final boolean useIndexSearchForClassification = (!isBuiltInType && !isWildcardSearch) &&
+ final boolean useIndexSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
+ classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
+ !isWildcardSearch) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) &&
CollectionUtils.isNotEmpty(indexAttributes) &&
- canApplyIndexFilter(classificationType, filterCriteria, false);
+ canApplyIndexFilter(classificationTypes, filterCriteria, false);
- traitPredicate = buildTraitPredict(classificationType);
+ final boolean useGraphSearchForClassification = (CollectionUtils.isNotEmpty(classificationTypes) &&
+ classificationTypes.iterator().next() != SearchContext.MATCH_ALL_NOT_CLASSIFIED &&
+ !isWildcardSearch && CollectionUtils.isNotEmpty(graphAttributes));
+
+ traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
AtlasGraph graph = context.getGraph();
@@ -115,8 +120,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// tagFilters is not allowed in wildcard search
graphIndexQueryBuilder.addClassificationTypeFilter(queryString);
} else {
- if (isBuiltInType) {
-
+ if (classificationTypes.iterator().next() == SearchContext.MATCH_ALL_NOT_CLASSIFIED) {
// tagFilters is not allowed in unique classificationType search
graphIndexQueryBuilder.addClassificationFilterForBuiltInTypes(queryString);
@@ -146,7 +150,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(queryString, typeAndSubTypesQryStr);
- constructFilterQuery(queryString, classificationType, filterCriteria, indexAttributes);
+ constructFilterQuery(queryString, classificationTypes, filterCriteria, indexAttributes);
String indexQueryString = STRAY_AND_PATTERN.matcher(queryString).replaceAll(")");
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
@@ -158,7 +162,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
}
- Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
+ Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
@@ -170,7 +174,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
// only registered classification will search with tag filters
- if (!isWildcardSearch && !isBuiltInType && !graphAttributes.isEmpty()) {
+ if (useGraphSearchForClassification) {
AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
@@ -179,7 +183,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
- tagGraphQueryWithAttributes = toGraphFilterQuery(classificationType, filterCriteria, allAttributes, query);
+ tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
gremlinQueryBindings = new HashMap<>();
StringBuilder gremlinQuery = new StringBuilder();
@@ -188,7 +192,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
gremlinQuery.append(".as('e').filter(out()");
gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
- constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
+ // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
// After filtering on tags go back to e and output the list of entity vertices
gremlinQuery.append(").toList()");
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index 37eef90..f814345 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -491,9 +491,11 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
resultAttributes.addAll(searchContext.getEntityAttributes());
}
- AtlasEntityType entityType = searchContext.getEntityType();
- if (entityType != null) {
- for (String resultAttribute : resultAttributes) {
+ if (CollectionUtils.isNotEmpty(searchContext.getEntityTypes())) {
+
+ AtlasEntityType entityType = searchContext.getEntityTypes().iterator().next();
+
+ for (String resultAttribute : resultAttributes) {
AtlasAttribute attribute = entityType.getAttribute(resultAttribute);
if (attribute == null) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 56956e6..5dcff3b 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -31,27 +31,15 @@ import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
import org.apache.commons.collections.PredicateUtils;
-import org.apache.commons.lang.StringUtils;
-import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.util.stream.StreamSupport;
import static org.apache.atlas.SortOrder.ASCENDING;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
-import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
+import static org.apache.atlas.discovery.SearchContext.*;
+import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
@@ -69,28 +57,28 @@ public class EntitySearchProcessor extends SearchProcessor {
public EntitySearchProcessor(SearchContext context) {
super(context);
- final AtlasEntityType entityType = context.getEntityType();
+ final Set<AtlasEntityType> entityTypes = context.getEntityTypes();
final FilterCriteria filterCriteria = context.getSearchParameters().getEntityFilters();
final Set<String> indexAttributes = new HashSet<>();
final Set<String> graphAttributes = new HashSet<>();
final Set<String> allAttributes = new HashSet<>();
- final Set<String> typeAndSubTypes = context.getEntityTypes();
+ final Set<String> typeAndSubTypes = context.getEntityTypeNames();
final String typeAndSubTypesQryStr = context.getEntityTypesQryStr();
final String sortBy = context.getSearchParameters().getSortBy();
final SortOrder sortOrder = context.getSearchParameters().getSortOrder();
- final AtlasClassificationType classificationType = context.getClassificationType();
- final Set<String> classificationTypeAndSubTypes = context.getClassificationTypes();
- final boolean filterClassification;
+ final Set<AtlasClassificationType> classificationTypes = context.getClassificationTypes();
+ final Set<String> classificationTypeAndSubTypes = context.getClassificationTypeNames();
+ final boolean filterClassification;
- if (classificationType != null) {
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
filterClassification = !context.needClassificationProcessor();
} else {
filterClassification = false;
}
final Predicate typeNamePredicate;
- final Predicate traitPredicate = buildTraitPredict(classificationType);
+ final Predicate traitPredicate = buildTraitPredict(classificationTypes);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
@@ -101,10 +89,10 @@ public class EntitySearchProcessor extends SearchProcessor {
typeNamePredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
}
- processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
+ processSearchAttributes(entityTypes, filterCriteria, indexAttributes, graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
- final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityType, filterCriteria, false);
+ final boolean attrSearchByIndex = !filterClassification && CollectionUtils.isNotEmpty(indexAttributes) && canApplyIndexFilter(entityTypes, filterCriteria, false);
StringBuilder indexQuery = new StringBuilder();
@@ -116,9 +104,9 @@ public class EntitySearchProcessor extends SearchProcessor {
}
if (attrSearchByIndex) {
- constructFilterQuery(indexQuery, entityType, filterCriteria, indexAttributes);
+ constructFilterQuery(indexQuery, entityTypes, filterCriteria, indexAttributes);
- Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, indexAttributes);
+ Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, indexAttributes);
if (attributePredicate != null) {
inMemoryPredicate = PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
@@ -149,6 +137,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
if (filterClassification) {
+ AtlasClassificationType classificationType = classificationTypes.iterator().next();
List<AtlasGraphQuery> orConditions = new LinkedList<>();
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
@@ -176,10 +165,10 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
- graphQuery = toGraphFilterQuery(entityType, filterCriteria, graphAttributes, query);
+ graphQuery = toGraphFilterQuery(entityTypes, filterCriteria, graphAttributes, query);
// Prepare in-memory predicate for attribute filtering
- Predicate attributePredicate = constructInMemoryPredicate(entityType, filterCriteria, graphAttributes);
+ Predicate attributePredicate = constructInMemoryPredicate(entityTypes, filterCriteria, graphAttributes);
if (attributePredicate != null) {
if (graphQueryPredicate != null) {
@@ -199,7 +188,8 @@ public class EntitySearchProcessor extends SearchProcessor {
}
}
if (sortBy != null && !sortBy.isEmpty()) {
- AtlasAttribute sortByAttribute = context.getEntityType().getAttribute(sortBy);
+ final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
+ AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute != null) {
AtlasGraphQuery.SortOrder qrySortOrder = sortOrder == SortOrder.ASCENDING ? ASC : DESC;
@@ -215,8 +205,7 @@ public class EntitySearchProcessor extends SearchProcessor {
// Prepare the graph query and in-memory filter for the filtering phase
filterGraphQueryPredicate = typeNamePredicate;
-
- Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes);
+ Predicate attributesPredicate = constructInMemoryPredicate(entityTypes, filterCriteria, allAttributes);
if (attributesPredicate != null) {
filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
@@ -265,7 +254,7 @@ public class EntitySearchProcessor extends SearchProcessor {
SortOrder sortOrder = context.getSearchParameters().getSortOrder();
String sortBy = context.getSearchParameters().getSortBy();
- final AtlasEntityType entityType = context.getEntityType();
+ final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
if (sortByAttribute == null) {
sortBy = null;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
index 6e3a760..d9981e6 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FreeTextSearchProcessor.java
@@ -18,7 +18,6 @@
package org.apache.atlas.discovery;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.*;
@@ -53,13 +52,13 @@ public class FreeTextSearchProcessor extends SearchProcessor {
queryString.append(searchParameters.getQuery());
- if (CollectionUtils.isNotEmpty(context.getEntityTypes()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ if (CollectionUtils.isNotEmpty(context.getEntityTypeNames()) && context.getEntityTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getEntityTypesQryStr());
}
graphIndexQueryBuilder.addActiveStateQueryFilter(queryString);
- if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypeNames()) && context.getClassificationTypesQryStr().length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(context.getClassificationTypesQryStr());
}
@@ -138,7 +137,7 @@ public class FreeTextSearchProcessor extends SearchProcessor {
continue;
}
- if (context.getClassificationType() != null) {
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
List<String> entityClassifications = GraphHelper.getAllTraitNames(vertex);
if (!context.includeClassificationTypes(entityClassifications)) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 99cb1d0..b37d93a 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -24,6 +24,7 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.commons.collections.CollectionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,10 +32,7 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
public class FullTextSearchProcessor extends SearchProcessor {
@@ -53,29 +51,28 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
- if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) {
- String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ String typeAndSubTypeNamesQryStr = context.getEntityTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
queryString.append(AND_STR).append(typeAndSubTypeNamesQryStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
- context.getEntityType().getTypeName(), typeAndSubTypeNamesQryStr.length());
+ searchParameters.getTypeName(), typeAndSubTypeNamesQryStr.length());
}
}
// if search includes classification criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent ClassificationSearchProcessor or EntitySearchProcessor
- if (context.getClassificationType() != null && context.getClassificationType() != MATCH_ALL_WILDCARD_CLASSIFICATION &&
- context.getClassificationType() != MATCH_ALL_CLASSIFIED &&
- context.getClassificationType() != MATCH_ALL_NOT_CLASSIFIED) {
- String typeAndSubTypeNamesStr = context.getClassificationType().getTypeAndAllSubTypesQryStr();
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes()) &&
+ context.getClassificationTypes().iterator().next() != MATCH_ALL_NOT_CLASSIFIED) {
+ String typeAndSubTypeNamesStr = context.getClassificationTypesQryStr();
if (typeAndSubTypeNamesStr.length() <= MAX_QUERY_STR_LENGTH_TAGS) {
queryString.append(AND_STR).append(typeAndSubTypeNamesStr);
} else {
LOG.warn("'{}' has too many subtypes (query-string-length={}) to include in index-query; might cause poor performance",
- context.getClassificationType().getTypeName(), typeAndSubTypeNamesStr.length());
+ searchParameters.getClassification(), typeAndSubTypeNamesStr.length());
}
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
index 28d2086..35d64b7 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/GraphIndexQueryBuilder.java
@@ -26,6 +26,8 @@ import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NA
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
public class GraphIndexQueryBuilder {
@@ -36,20 +38,20 @@ public class GraphIndexQueryBuilder {
}
void addClassificationTypeFilter(StringBuilder indexQuery) {
- if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
- String classificationName = context.getSearchParameters().getClassification();
+ if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationNames())) {
+ String classificationNames = AtlasStructType.AtlasAttribute.escapeIndexQueryValue(context.getClassificationNames());
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationName)
+ indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append('\"').append(CLASSIFICATION_NAMES_KEY).append('\"').append(':').append(classificationNames)
.append(" OR ").append(INDEX_SEARCH_PREFIX).append('\"').append(PROPAGATED_CLASSIFICATION_NAMES_KEY)
- .append('\"').append(':').append(classificationName).append(")");
+ .append('\"').append(':').append(classificationNames).append(")");
}
}
void addClassificationAndSubTypesQueryFilter(StringBuilder indexQuery) {
- if (indexQuery != null && StringUtils.isNotEmpty(context.getSearchParameters().getClassification())) {
+ if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
String classificationTypesQryStr = context.getClassificationTypesQryStr();
if (indexQuery.length() != 0) {
@@ -63,17 +65,8 @@ public class GraphIndexQueryBuilder {
}
void addClassificationFilterForBuiltInTypes(StringBuilder indexQuery) {
- if (indexQuery != null && context.getClassificationType() != null) {
- if (context.getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION || context.getClassificationType() == MATCH_ALL_CLASSIFIED) {
- if (indexQuery.length() != 0) {
- indexQuery.append(" AND ");
- }
- indexQuery.append("(").append(INDEX_SEARCH_PREFIX).append("\"")
- .append(CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]")
- .append(" OR ").append(INDEX_SEARCH_PREFIX).append("\"")
- .append(PROPAGATED_CLASSIFICATION_NAMES_KEY).append("\"").append(":" + "[* TO *]").append(")");
-
- } else if (context.getClassificationType() == MATCH_ALL_NOT_CLASSIFIED) {
+ if (indexQuery != null && CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
+ if (context.getClassificationTypes().iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
if (indexQuery.length() != 0) {
indexQuery.append(" AND ");
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
index e8f7dbc..2d095be 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchAggregatorImpl.java
@@ -56,12 +56,7 @@ public class SearchAggregatorImpl implements SearchAggregator {
try {
AtlasGraphIndexClient graphIndexClient = graph.getGraphIndexClient();
- String searchedOnTypeName = searchParameters.getTypeName();
- AtlasEntityType searchForEntityType = null;
-
- if (searchedOnTypeName != null) {
- searchForEntityType = typeRegistry.getEntityTypeByName(searchedOnTypeName);
- }
+ Set<AtlasEntityType> searchForEntityType = searchContext.getEntityTypes();
Map<String, String> indexFieldNameCache = new HashMap<>();
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 2c2888e..04e8218 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -21,7 +21,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.discovery.SearchParameters.*;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.repository.Constants;
@@ -44,13 +44,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.*;
+import java.util.stream.Collectors;
import static org.apache.atlas.discovery.SearchProcessor.ALL_TYPE_QUERY;
-import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
-import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATION_TYPES;
-import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
-import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
-import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.*;
/*
* Search context captures elements required for performing a basic search
@@ -62,12 +59,12 @@ public class SearchContext {
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
- private final AtlasEntityType entityType;
+ private final Set<AtlasEntityType> entityTypes;
private final Set<String> indexedKeys;
private final Set<String> entityAttributes;
private final SearchParameters searchParameters;
- private final AtlasClassificationType classificationType;
- private final String classificationName;
+ private final Set<AtlasClassificationType> classificationTypes;
+ private final Set<String> classificationNames;
private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes;
private final String typeAndSubTypesQryStr;
@@ -80,27 +77,18 @@ public class SearchContext {
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFICATION_TYPES = AtlasClassificationType.getClassificationRoot();
public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
+ public final static String TYPENAME_DELIMITER = ",";
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
- this.classificationName = searchParameters.getClassification();
this.searchParameters = searchParameters;
this.typeRegistry = typeRegistry;
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>();
- this.entityType = getEntityType(searchParameters.getTypeName());
- this.classificationType = getClassificationType(classificationName);
-
- // Validate if the type name exists
- if (StringUtils.isNotEmpty(searchParameters.getTypeName()) && entityType == null) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, searchParameters.getTypeName());
- }
-
- // Validate if the classification exists
- if ((StringUtils.isNotEmpty(classificationName) && classificationType == null && !classificationName.contains(WILDCARD_CLASSIFICATIONS))) {
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classificationName);
- }
+ this.entityTypes = getEntityTypes(searchParameters.getTypeName());
+ this.classificationNames = getClassificationNames(searchParameters.getClassification());
+ this.classificationTypes = getClassificationTypes(this.classificationNames);
AtlasVertex glossaryTermVertex = getGlossaryTermVertex(searchParameters.getTermName());
@@ -109,40 +97,77 @@ public class SearchContext {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_GLOSSARY_TERM, searchParameters.getTermName());
}
- // Invalid attributes will raise an exception with 400 error code
- validateAttributes(entityType, searchParameters.getEntityFilters());
+ // Invalid attributes or unsupported attribute in a type, will raise an exception with 400 error code
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ for (AtlasEntityType entityType : entityTypes) {
+ validateAttributes(entityType, searchParameters.getEntityFilters());
- // Invalid attribute will raise an exception with 400 error code
- validateAttributes(entityType, searchParameters.getSortBy());
+ validateAttributes(entityType, searchParameters.getSortBy());
+ }
+ }
// Invalid attributes will raise an exception with 400 error code
- validateAttributes(classificationType, searchParameters.getTagFilters());
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ for (AtlasClassificationType classificationType : classificationTypes) {
+ validateAttributes(classificationType, searchParameters.getTagFilters());
+ }
+ }
- if (classificationType != null && !isBuiltInClassificationType()) {
- if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
- classificationTypeAndSubTypes = Collections.emptySet();
- classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
- } else {
- classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
- classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr();
+ //remove other types if builtin type is present
+ filterStructTypes();
+
+ //gather all classifications and its corresponding subtypes
+ Set<String> classificationTypeAndSubTypes = new HashSet<>();
+ String classificationTypeAndSubTypesQryStr = null;
+
+ if (CollectionUtils.isNotEmpty(classificationTypes) && classificationTypes.iterator().next() != MATCH_ALL_NOT_CLASSIFIED ) {
+ for (AtlasClassificationType classificationType : classificationTypes) {
+
+ if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
+ classificationTypeAndSubTypes = Collections.emptySet();
+ classificationTypeAndSubTypesQryStr = ALL_TYPE_QUERY;
+ break;
+ } else {
+ Set<String> allTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
+ classificationTypeAndSubTypes.addAll(allTypes); }
+ }
+
+ if (CollectionUtils.isNotEmpty(classificationTypeAndSubTypes)) {
+ classificationTypeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(classificationTypeAndSubTypes);
}
} else {
classificationTypeAndSubTypes = Collections.emptySet();
classificationTypeAndSubTypesQryStr = "";
}
+ this.classificationTypeAndSubTypes = classificationTypeAndSubTypes;
+ this.classificationTypeAndSubTypesQryStr = classificationTypeAndSubTypesQryStr;
+
+ //gather all types and its corresponding subtypes
+ Set<String> typeAndSubTypes = new HashSet<>();
+ String typeAndSubTypesQryStr = null;
+
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ for (AtlasEntityType entityType : entityTypes) {
+
+ if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
+ typeAndSubTypes = Collections.emptySet();
+ typeAndSubTypesQryStr = ALL_TYPE_QUERY;
+ break;
+ } else {
+ Set<String> allTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
+ typeAndSubTypes.addAll(allTypes);
+ }
+ }
- if (entityType != null) {
- if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
- typeAndSubTypes = Collections.emptySet();
- typeAndSubTypesQryStr = ALL_TYPE_QUERY;
- } else {
- typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
- typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
+ if (CollectionUtils.isNotEmpty(typeAndSubTypes)) {
+ typeAndSubTypesQryStr = AtlasAttribute.escapeIndexQueryValue(typeAndSubTypes);
}
} else {
typeAndSubTypes = Collections.emptySet();
typeAndSubTypesQryStr = "";
}
+ this.typeAndSubTypes = typeAndSubTypes;
+ this.typeAndSubTypesQryStr = typeAndSubTypesQryStr;
if (glossaryTermVertex != null) {
addProcessor(new TermSearchProcessor(this, getAssignedEntities(glossaryTermVertex)));
@@ -179,37 +204,37 @@ public class SearchContext {
public Set<String> getEntityAttributes() { return entityAttributes; }
- public AtlasEntityType getEntityType() { return entityType; }
+ public Set<AtlasClassificationType> getClassificationTypes() { return classificationTypes; }
- public AtlasClassificationType getClassificationType() { return classificationType; }
+ public Set<String> getEntityTypeNames() { return typeAndSubTypes; }
- public Set<String> getEntityTypes() { return typeAndSubTypes; }
-
- public Set<String> getClassificationTypes() { return classificationTypeAndSubTypes; }
+ public Set<String> getClassificationTypeNames() { return classificationTypeAndSubTypes; }
public String getEntityTypesQryStr() { return typeAndSubTypesQryStr; }
public String getClassificationTypesQryStr() { return classificationTypeAndSubTypesQryStr; }
+ public Set<AtlasEntityType> getEntityTypes() { return entityTypes; }
+
public SearchProcessor getSearchProcessor() { return searchProcessor; }
- public String getClassificationName() {return classificationName;}
+ public Set<String> getClassificationNames() {return classificationNames;}
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
}
- public boolean includeClassificationTypes(Collection<String> classificationTypes) {
+ public boolean includeClassificationTypes(Collection<String> traitNames) {
final boolean ret;
- if (classificationType == null || classificationTypeAndSubTypes.isEmpty()) {
+ if (CollectionUtils.isEmpty(classificationTypes) || classificationTypeAndSubTypes.isEmpty()) {
ret = true;
- } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
- ret = CollectionUtils.isEmpty(classificationTypes);
- } else if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
- ret = CollectionUtils.isNotEmpty(classificationTypes);
+ } else if (classificationTypes.iterator().next() == MATCH_ALL_NOT_CLASSIFIED) {
+ ret = CollectionUtils.isEmpty(traitNames);
+ } else if (classificationTypes.iterator().next() == MATCH_ALL_CLASSIFICATION_TYPES) {
+ ret = CollectionUtils.isNotEmpty(traitNames);
} else {
- ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, classificationTypes);
+ ret = CollectionUtils.containsAny(classificationTypeAndSubTypes, traitNames);
}
return ret;
@@ -243,25 +268,18 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
- return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
- }
-
- boolean isBuiltInClassificationType() {
- return getClassificationType() == MATCH_ALL_WILDCARD_CLASSIFICATION
- || getClassificationType() == MATCH_ALL_CLASSIFIED
- || getClassificationType() == MATCH_ALL_NOT_CLASSIFIED;
+ return (CollectionUtils.isNotEmpty(classificationTypes) && (CollectionUtils.isEmpty(entityTypes) || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isWildCardSearch () {
- String classification = getSearchParameters().getClassification();
- if (StringUtils.isNotEmpty(classification) && getClassificationType() == null) {
- return classification.contains("*");
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ return classificationNames.stream().anyMatch(classification -> classification.contains(WILDCARD_CLASSIFICATIONS));
}
return false;
}
boolean needEntityProcessor() {
- return entityType != null;
+ return CollectionUtils.isNotEmpty(entityTypes);
}
private void validateAttributes(final AtlasStructType structType, final FilterCriteria filterCriteria) throws AtlasBaseException {
@@ -285,7 +303,14 @@ public class SearchContext {
if (structType == null) {
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, "NULL");
}
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, structType.getTypeName());
+
+ String name = structType.getTypeName();
+ if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
+ name = ALL_ENTITY_TYPES;
+ } else if (name.equals(MATCH_ALL_CLASSIFICATION_TYPES.getTypeName())) {
+ name = ALL_CLASSIFICATION_TYPES;
+ }
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attributeName, name);
}
}
}
@@ -321,11 +346,107 @@ public class SearchContext {
return ret;
}
+ private Set<AtlasClassificationType> getClassificationTypes(Set<String> classificationNames) {
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ return classificationNames.stream().map(n ->
+ getClassificationType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+ }
+
+ return null;
+ }
+
+ private Set<String> getClassificationNames(String classification) throws AtlasBaseException {
+ Set<String> classificationNames = new HashSet<>();
+
+ if (StringUtils.isNotEmpty(classification)) {
+ String[] types = classification.split(TYPENAME_DELIMITER);
+ Set<String> names = new HashSet<>(Arrays.asList(types));
+
+ names.forEach(name -> {
+ AtlasClassificationType type = getClassificationType(name);
+ if (type != null || name.contains(WILDCARD_CLASSIFICATIONS)) {
+ classificationNames.add(name);
+ }
+ });
+
+ // Validate if the classification exists
+ if (CollectionUtils.isEmpty(classificationNames) || classificationNames.size() != names.size()) {
+ if (CollectionUtils.isNotEmpty(classificationNames)) {
+ names.removeAll(classificationNames);
+ }
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, String.join(TYPENAME_DELIMITER, names));
+ }
+ }
+
+ return classificationNames;
+ }
+
private AtlasEntityType getEntityType(String entityName) {
return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES :
typeRegistry.getEntityTypeByName(entityName);
}
+ private Set<AtlasEntityType> getEntityTypes(String typeName) throws AtlasBaseException {
+
+ Set<AtlasEntityType> entityTypes = null;
+ //split multiple typeNames by comma
+ if (StringUtils.isNotEmpty(typeName)) {
+
+ String[] types = typeName.split(TYPENAME_DELIMITER);
+ Set<String> typeNames = new HashSet<>(Arrays.asList(types));
+ entityTypes = typeNames.stream().map(n ->
+ getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
+
+ // Validate if the type name is incorrect
+ if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) {
+ if (CollectionUtils.isNotEmpty(entityTypes)) {
+ Set<String> validEntityTypes = new HashSet<>();
+ for (AtlasEntityType entityType : entityTypes) {
+ String name = entityType.getTypeName();
+ if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
+ validEntityTypes.add(ALL_ENTITY_TYPES);
+ continue;
+ }
+ validEntityTypes.add(entityType.getTypeName());
+ }
+
+ typeNames.removeAll(validEntityTypes);
+ }
+
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, String.join(TYPENAME_DELIMITER, typeNames));
+ }
+
+ }
+
+ return entityTypes;
+ }
+
+ private void filterStructTypes(){
+ //if typeName contains ALL_ENTITY_TYPES, remove others as OR condition will not effect any other
+ if (CollectionUtils.isNotEmpty(entityTypes) && entityTypes.contains(MATCH_ALL_ENTITY_TYPES)) {
+ entityTypes.clear();
+ entityTypes.add(MATCH_ALL_ENTITY_TYPES);
+ }
+
+ //No Builtin Classification can be together
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ if (classificationTypes.contains(MATCH_ALL_NOT_CLASSIFIED)) {
+ classificationTypes.clear();
+ classificationTypes.add(MATCH_ALL_NOT_CLASSIFIED);
+
+ classificationNames.clear();
+ classificationNames.add(MATCH_ALL_NOT_CLASSIFIED.getTypeName());
+ } else if (classificationTypes.contains(MATCH_ALL_WILDCARD_CLASSIFICATION) || classificationTypes.contains(MATCH_ALL_CLASSIFICATION_TYPES) || classificationTypes.contains(MATCH_ALL_CLASSIFIED)) {
+ classificationTypes.clear();
+ classificationTypes.add(MATCH_ALL_CLASSIFICATION_TYPES);
+
+ classificationNames.clear();
+ classificationNames.add(ALL_CLASSIFICATION_TYPES);
+ }
+ }
+ }
+
+
private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index 981ba6a..c62f805 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -51,16 +51,8 @@ import java.util.regex.Pattern;
import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
-import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAME_DELIMITER;
-import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
-import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
@@ -153,11 +145,19 @@ public abstract class SearchProcessor {
public abstract long getResultCount();
protected boolean isEntityRootType() {
- return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
+ //always size will be one if in case of _ALL_ENTITY_TYPES
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ return context.getEntityTypes().iterator().next() == SearchContext.MATCH_ALL_ENTITY_TYPES;
+ }
+ return false;
}
protected boolean isClassificationRootType() {
- return context.getClassificationType() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
+ //always size will be one if in case of _ALL_CLASSIFICATION_TYPES
+ if (CollectionUtils.isNotEmpty(context.getClassificationTypes())) {
+ return context.getClassificationTypes().iterator().next() == SearchContext.MATCH_ALL_CLASSIFICATION_TYPES;
+ }
+ return false;
}
protected boolean isSystemAttribute(String attrName) {
@@ -195,9 +195,14 @@ public abstract class SearchProcessor {
}
}
- protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
+ protected Predicate buildTraitPredict(Set<AtlasClassificationType> classificationTypes) {
Predicate traitPredicate;
- if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
+ AtlasClassificationType classificationType = null;
+ if (CollectionUtils.isNotEmpty(classificationTypes)) {
+ classificationType = classificationTypes.iterator().next();
+ }
+
+ if (classificationType == MATCH_ALL_CLASSIFICATION_TYPES) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
@@ -205,20 +210,27 @@ public abstract class SearchProcessor {
SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
} else if (context.isWildCardSearch()) {
//For wildcard search __classificationNames which of String type is taken instead of _traitNames which is of Array type
- //No need to escape, as classification Names only support letters,numbers,space and underscore
- String regexString = getRegexString("\\|" + context.getClassificationName() + "\\|");
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class),
- SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
+ Set<String> classificationNames = context.getClassificationNames();
+ List<Predicate> predicates = new ArrayList<>();
+
+ classificationNames.forEach(classificationName -> {
+ //No need to escape, as classification Names only support letters,numbers,space and underscore
+ String regexString = getRegexString("\\|" + classificationName + "\\|");
+ predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(CLASSIFICATION_NAMES_KEY, regexString, String.class));
+ predicates.add(SearchPredicateUtil.getRegexPredicateGenerator().generatePredicate(PROPAGATED_CLASSIFICATION_NAMES_KEY, regexString, String.class));
+ });
+
+ traitPredicate = PredicateUtils.anyPredicate(predicates);
} else {
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class),
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypeNames(), List.class));
}
return traitPredicate;
}
- protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
- if (structType == null || filterCriteria == null) {
+ protected void processSearchAttributes(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
+ if (CollectionUtils.isEmpty(structTypes) || filterCriteria == null) {
return;
}
@@ -227,7 +239,7 @@ public abstract class SearchProcessor {
if (filterCondition != null && CollectionUtils.isNotEmpty(criterion)) {
for (SearchParameters.FilterCriteria criteria : criterion) {
- processSearchAttributes(structType, criteria, indexFiltered, graphFiltered, allAttributes);
+ processSearchAttributes(structTypes, criteria, indexFiltered, graphFiltered, allAttributes);
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
String attributeName = filterCriteria.getAttributeName();
@@ -267,20 +279,23 @@ public abstract class SearchProcessor {
}
try {
- if (isIndexSearchable(filterCriteria, structType)) {
- indexFiltered.add(attributeName);
- } else {
- LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName));
+ for (AtlasStructType structType : structTypes) {
+ String qualifiedName = structType.getVertexPropertyName(attributeName);
+ if (isIndexSearchable(filterCriteria, structType)) {
+ indexFiltered.add(qualifiedName);
+ } else {
+ LOG.warn("not using index-search for attribute '{}'; might cause poor performance", structType.getVertexPropertyName(attributeName));
- graphFiltered.add(attributeName);
- }
+ graphFiltered.add(qualifiedName);
+ }
- if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
- // Capture the entity attributes
- context.getEntityAttributes().add(attributeName);
- }
+ if (structType instanceof AtlasEntityType && !isSystemAttribute(attributeName)) {
+ // Capture the entity attributes
+ context.getEntityAttributes().add(attributeName);
+ }
- allAttributes.add(attributeName);
+ allAttributes.add(qualifiedName);
+ }
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
}
@@ -298,7 +313,7 @@ public abstract class SearchProcessor {
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z)
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
//
- protected boolean canApplyIndexFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) {
+ protected boolean canApplyIndexFilter(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, boolean insideOrCondition) {
if (!context.hasAttributeFilter(filterCriteria)) {
return true;
}
@@ -314,7 +329,7 @@ public abstract class SearchProcessor {
// If we have nested criterion let's find any nested ORs with non-indexed attr
for (FilterCriteria criteria : criterion) {
- ret = canApplyIndexFilter(structType, criteria, insideOrCondition);
+ ret = canApplyIndexFilter(structTypes, criteria, insideOrCondition);
if (!ret) {
break;
@@ -322,10 +337,11 @@ public abstract class SearchProcessor {
}
} else if (StringUtils.isNotEmpty(filterCriteria.getAttributeName())) {
try {
-
-
- if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) {
- ret = false;
+ for (AtlasStructType structType : structTypes) {
+ if (insideOrCondition && !isIndexSearchable(filterCriteria, structType)) {
+ ret = false;
+ break;
+ }
}
} catch (AtlasBaseException e) {
LOG.warn(e.getMessage());
@@ -338,7 +354,7 @@ public abstract class SearchProcessor {
protected void filterWhiteSpaceClassification(List<AtlasVertex> entityVertices) {
if (CollectionUtils.isNotEmpty(entityVertices)) {
final Iterator<AtlasVertex> it = entityVertices.iterator();
- final Set<String> typeAndSubTypes = context.getClassificationTypes();
+ final Set<String> typeAndSubTypes = context.getClassificationTypeNames();
while (it.hasNext()) {
AtlasVertex entityVertex = it.next();
@@ -363,13 +379,13 @@ public abstract class SearchProcessor {
}
}
- protected void constructFilterQuery(StringBuilder indexQuery, AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) {
+ protected void constructFilterQuery(StringBuilder indexQuery, Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
- String filterQuery = toIndexQuery(type, filterCriteria, indexAttributes, 0);
+ String filterQuery = toIndexQuery(structTypes, filterCriteria, indexAttributes, 0);
if (StringUtils.isNotEmpty(filterQuery)) {
if (indexQuery.length() > 0) {
@@ -381,14 +397,14 @@ public abstract class SearchProcessor {
}
}
- protected Predicate constructInMemoryPredicate(AtlasStructType type, FilterCriteria filterCriteria, Set<String> indexAttributes) {
+ protected Predicate constructInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria filterCriteria, Set<String> indexAttributes) {
Predicate ret = null;
if (filterCriteria != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Processing Filters");
}
- ret = toInMemoryPredicate(type, filterCriteria, indexAttributes);
+ ret = toInMemoryPredicate(structTypes, filterCriteria, indexAttributes);
}
return ret;
}
@@ -473,17 +489,20 @@ public abstract class SearchProcessor {
return ret;
}
- private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, int level) {
- return toIndexQuery(type, criteria, indexAttributes, new StringBuilder(), level);
+ private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, int level) {
+ return toIndexQuery(structTypes, criteria, indexAttributes, new StringBuilder(), level);
}
- private String toIndexQuery(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
+ private String toIndexQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes, StringBuilder sb, int level) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(indexAttributes);
+
Condition condition = criteria.getCondition();
if (condition != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
StringBuilder nestedExpression = new StringBuilder();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- String nestedQuery = toIndexQuery(type, filterCriteria, indexAttributes, level + 1);
+ String nestedQuery = toIndexQuery(structTypes, filterCriteria, filterAttributes, level + 1);
if (StringUtils.isNotEmpty(nestedQuery)) {
if (nestedExpression.length() > 0) {
@@ -502,19 +521,45 @@ public abstract class SearchProcessor {
} else {
return EMPTY_STRING;
}
- } else if (indexAttributes.contains(criteria.getAttributeName())){
- return toIndexExpression(type, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
- } else {
- return EMPTY_STRING;
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<String> orExpQuery = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String name = structType.getVertexPropertyName(criteria.getAttributeName());
+
+ if (filterAttributes.contains(name)) {
+ String nestedQuery = toIndexExpression(structType, criteria.getAttributeName(), criteria.getOperator(), criteria.getAttributeValue());
+ orExpQuery.add(nestedQuery);
+ filterAttributes.remove(name);
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(orExpQuery)) {
+ if (orExpQuery.size() > 1) {
+ String orExpStr = StringUtils.join(orExpQuery, " "+Condition.OR.name()+" ");
+ return BRACE_OPEN_STR + " " + orExpStr + " " + BRACE_CLOSE_STR;
+ } else {
+ return orExpQuery.iterator().next();
+ }
+ } else {
+ return EMPTY_STRING;
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
}
+ return EMPTY_STRING;
}
- private Predicate toInMemoryPredicate(AtlasStructType type, FilterCriteria criteria, Set<String> indexAttributes) {
+ private Predicate toInMemoryPredicate(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> indexAttributes) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(indexAttributes);
+
if (criteria.getCondition() != null && CollectionUtils.isNotEmpty(criteria.getCriterion())) {
List<Predicate> predicates = new ArrayList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- Predicate predicate = toInMemoryPredicate(type, filterCriteria, indexAttributes);
+ Predicate predicate = toInMemoryPredicate(structTypes, filterCriteria, filterAttributes);
if (predicate != null) {
predicates.add(predicate);
@@ -528,21 +573,40 @@ public abstract class SearchProcessor {
return PredicateUtils.anyPredicate(predicates);
}
}
- } else if (indexAttributes.contains(criteria.getAttributeName())) {
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<Predicate> predicates = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String name = structType.getVertexPropertyName(criteria.getAttributeName());
+
+ if (filterAttributes.contains(name)) {
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
+
+ //process attribute value and attribute operator for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
- //process attribute value and attribute operator for pipeSeperated fields
- if (isPipeSeparatedSystemAttribute(attrName)) {
- FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue = processedCriteria.getAttributeValue();
- operator = processedCriteria.getOperator();
- }
+ predicates.add(toInMemoryPredicate(structType, attrName, operator, attrValue));
+ filterAttributes.remove(name);
+ }
+ }
- return toInMemoryPredicate(type, attrName, operator, attrValue);
+ if (CollectionUtils.isNotEmpty(predicates)) {
+ if (predicates.size() > 1) {
+ return PredicateUtils.anyPredicate(predicates);
+ } else {
+ return predicates.iterator().next();
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
+ }
}
-
return null;
}
@@ -704,12 +768,15 @@ public abstract class SearchProcessor {
return ret;
}
- protected AtlasGraphQuery toGraphFilterQuery(AtlasStructType type, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
+ protected AtlasGraphQuery toGraphFilterQuery(Set<? extends AtlasStructType> structTypes, FilterCriteria criteria, Set<String> graphAttributes, AtlasGraphQuery query) {
+ Set<String> filterAttributes = new HashSet<>();
+ filterAttributes.addAll(graphAttributes);
+
if (criteria != null) {
if (criteria.getCondition() != null) {
if (criteria.getCondition() == Condition.AND) {
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query());
+ AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
query.addConditionsFrom(nestedQuery);
}
@@ -717,7 +784,7 @@ public abstract class SearchProcessor {
List<AtlasGraphQuery> orConditions = new LinkedList<>();
for (FilterCriteria filterCriteria : criteria.getCriterion()) {
- AtlasGraphQuery nestedQuery = toGraphFilterQuery(type, filterCriteria, graphAttributes, context.getGraph().query());
+ AtlasGraphQuery nestedQuery = toGraphFilterQuery(structTypes, filterCriteria, filterAttributes, context.getGraph().query());
orConditions.add(context.getGraph().query().createChildQuery().addConditionsFrom(nestedQuery));
}
@@ -726,60 +793,80 @@ public abstract class SearchProcessor {
query.or(orConditions);
}
}
- } else if (graphAttributes.contains(criteria.getAttributeName())) {
- String attrName = criteria.getAttributeName();
- String attrValue = criteria.getAttributeValue();
- SearchParameters.Operator operator = criteria.getOperator();
-
- //process attribute value and attribute operator for pipeSeperated fields
- if (isPipeSeparatedSystemAttribute(attrName)) {
- FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
- attrValue = processedCriteria.getAttributeValue();
- operator = processedCriteria.getOperator();
- }
-
- final String qualifiedName = type.getAttribute(attrName).getVertexPropertyName();
+ } else if (StringUtils.isNotEmpty(criteria.getAttributeName())) {
+ try {
+ ArrayList<AtlasGraphQuery> queries = new ArrayList<>();
+ for (AtlasStructType structType : structTypes) {
+ String qualifiedName = structType.getVertexPropertyName(criteria.getAttributeName());
+ if (filterAttributes.contains(qualifiedName)) {
+
+ String attrName = criteria.getAttributeName();
+ String attrValue = criteria.getAttributeValue();
+ SearchParameters.Operator operator = criteria.getOperator();
+
+ //process attribute value and attribute operator for pipeSeperated fields
+ if (isPipeSeparatedSystemAttribute(attrName)) {
+ FilterCriteria processedCriteria = processPipeSeperatedSystemAttribute(attrName, operator, attrValue);
+ attrValue = processedCriteria.getAttributeValue();
+ operator = processedCriteria.getOperator();
+ }
+
+ AtlasGraphQuery innerQry = context.getGraph().query().createChildQuery();
+ switch (operator) {
+ case LT:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
+ break;
+ case LTE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
+ break;
+ case GT:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
+ break;
+ case GTE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
+ break;
+ case EQ:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
+ break;
+ case NEQ:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
+ break;
+ case LIKE:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
+ break;
+ case CONTAINS:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
+ break;
+ case STARTS_WITH:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
+ break;
+ case ENDS_WITH:
+ innerQry.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
+ break;
+ case IS_NULL:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
+ break;
+ case NOT_NULL:
+ innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
+ break;
+ default:
+ LOG.warn("{}: unsupported operator. Ignored", operator);
+ break;
+ }
+ queries.add(context.getGraph().query().createChildQuery().addConditionsFrom(innerQry));
+ filterAttributes.remove(qualifiedName);
+ }
+ }
- switch (operator) {
- case LT:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN, attrValue);
- break;
- case LTE:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.LESS_THAN_EQUAL, attrValue);
- break;
- case GT:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN, attrValue);
- break;
- case GTE:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.GREATER_THAN_EQUAL, attrValue);
- break;
- case EQ:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, attrValue);
- break;
- case NEQ:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, attrValue);
- break;
- case LIKE:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, attrValue);
- break;
- case CONTAINS:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getContainsRegex(attrValue));
- break;
- case STARTS_WITH:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.PREFIX, attrValue);
- break;
- case ENDS_WITH:
- query.has(qualifiedName, AtlasGraphQuery.MatchingOperator.REGEX, getSuffixRegex(attrValue));
- break;
- case IS_NULL:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.EQUAL, null);
- break;
- case NOT_NULL:
- query.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
- break;
- default:
- LOG.warn("{}: unsupported operator. Ignored", operator);
- break;
+ if (CollectionUtils.isNotEmpty(queries)) {
+ if (queries.size() > 1) {
+ return context.getGraph().query().createChildQuery().or(queries);
+ } else {
+ return queries.iterator().next();
+ }
+ }
+ } catch (AtlasBaseException e) {
+ LOG.warn(e.getMessage());
}
}
}
@@ -987,11 +1074,13 @@ public abstract class SearchProcessor {
}
private static String getSortByAttribute(SearchContext context) {
- final AtlasEntityType entityType = context.getEntityType();
- String sortBy = context.getSearchParameters().getSortBy();
- AtlasStructType.AtlasAttribute sortByAttribute = entityType != null ? entityType.getAttribute(sortBy) : null;
- if (sortByAttribute != null) {
- return sortByAttribute.getVertexPropertyName();
+ if (CollectionUtils.isNotEmpty(context.getEntityTypes())) {
+ final AtlasEntityType entityType = context.getEntityTypes().iterator().next();
+ String sortBy = context.getSearchParameters().getSortBy();
+ AtlasStructType.AtlasAttribute sortByAttribute = entityType.getAttribute(sortBy);
+ if (sortByAttribute != null) {
+ return sortByAttribute.getVertexPropertyName();
+ }
}
return null;
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index 43f11d1..b7ce978 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -130,7 +130,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
new EntitySearchProcessor(context);
}
- @Test
+ @Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
@@ -197,4 +197,116 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains("hive_Table_Null_tableType"));
}
+
+ @Test
+ public void ALLEntityType() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 20);
+ }
+
+ @Test
+ public void ALLEntityTypeWithTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(SearchParameters.ALL_ENTITY_TYPES);
+ params.setClassification(FACT_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 5);
+ }
+
+ @Test
+ public void entityType() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 3);
+ }
+
+ @Test
+ public void entityTypes() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 14);
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
+ public void entityTypesNotAllowed() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+",Not_Exists");
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+ }
+
+ @Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Attribute tableType not found for type "+DATABASE_TYPE)
+ public void entityFiltersNotAllowed() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.CONTAINS, "ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+ }
+
+ @Test
+ public void entityTypesAndTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ params.setClassification(FACT_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 3);
+ }
+
+ @Test
+ public void searchWithEntityTypesAndEntityFilters() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 4);
+ }
+
+ @Test
+ public void searchWithEntityTypesAndEntityFiltersAndTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(DATABASE_TYPE+","+HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("owner", SearchParameters.Operator.CONTAINS, "ETL");
+ params.setEntityFilters(filterCriteria);
+ params.setClassification(LOGDATA_CLASSIFICATION);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
+
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ assertEquals(processor.execute().size(), 2);
+ }
+
}
[atlas] 05/05: ATLAS-3782 : Support NOT_CONTAINS operator in basic
search
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 897c2fc19f342d6fb8832734ac698e442bf85b58
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Tue Jun 30 14:53:45 2020 +0530
ATLAS-3782 : Support NOT_CONTAINS operator in basic search
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit 9a067d38e27fc2255647133373b952a6a64ed1a4)
---
.../graphdb/janus/AtlasSolrQueryBuilder.java | 15 +-
.../discovery/ClassificationSearchProcessor.java | 104 +++----
.../apache/atlas/discovery/SearchProcessor.java | 7 +-
.../test/java/org/apache/atlas/BasicTestSetup.java | 43 ++-
.../atlas/discovery/AtlasDiscoveryServiceTest.java | 338 +++++++++++++++++++++
...java => ClassificationSearchProcessorTest.java} | 175 ++++++++---
.../atlas/discovery/EntitySearchProcessorTest.java | 59 +++-
7 files changed, 608 insertions(+), 133 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index 6c06a3c..43114e9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -195,11 +195,15 @@ public class AtlasSolrQueryBuilder {
if (!indexAttributes.contains(indexAttributeName)) {
StringBuilder sb = new StringBuilder();
- if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
// CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
// for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
// surrounding inverted commas to attributeValue works
- operator = Operator.EQ;
+ if (operator.equals(Operator.CONTAINS)) {
+ operator = Operator.EQ;
+ } else if (operator.equals(Operator.NOT_CONTAINS)) {
+ operator = Operator.NEQ;
+ }
attributeValue = getIndexQueryAttributeValue(attributeValue);
}
@@ -261,6 +265,9 @@ public class AtlasSolrQueryBuilder {
case CONTAINS:
withContains(queryBuilder, indexFieldName, attributeValue);
break;
+ case NOT_CONTAINS:
+ withNotContains(queryBuilder, indexFieldName, attributeValue);
+ break;
case IS_NULL:
withIsNull(queryBuilder, indexFieldName);
break;
@@ -388,6 +395,10 @@ public class AtlasSolrQueryBuilder {
queryBuilder.append("+").append(indexFieldName).append(":*").append(attributeValue).append("* ");
}
+ private void withNotContains(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
+ queryBuilder.append("*:* -").append(indexFieldName).append(":*").append(attributeValue).append("* ");
+ }
+
private void withIsNull(StringBuilder queryBuilder, String indexFieldName) {
queryBuilder.append("-").append(indexFieldName).append(":*").append(" ");
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 9c72cd4..647ff9c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -18,13 +18,12 @@
package org.apache.atlas.discovery;
import org.apache.atlas.SortOrder;
-import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.*;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.AtlasClassificationType;
-import org.apache.atlas.util.AtlasGremlinQueryProvider;
import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.collections.CollectionUtils;
@@ -35,8 +34,6 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.script.ScriptEngine;
-import javax.script.ScriptException;
import java.util.*;
/**
@@ -52,10 +49,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasIndexQuery indexQuery;
private final AtlasIndexQuery classificationIndexQuery;
private final AtlasGraphQuery tagGraphQueryWithAttributes;
- private final Map<String, Object> gremlinQueryBindings;
- private final String gremlinTagFilterQuery;
private final Predicate traitPredicate;
private final Predicate isEntityPredicate;
+ private Predicate activePredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
@@ -107,6 +103,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
traitPredicate = buildTraitPredict(classificationTypes);
isEntityPredicate = SearchPredicateUtil.generateIsEntityVertexPredicate(context.getTypeRegistry());
+ if (context.getSearchParameters().getExcludeDeletedEntities()) {
+ activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
+ .generatePredicate(Constants.STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name(), String.class);
+ }
+
+ Predicate attributePredicate = null;
+ Predicate typeNamePredicate = null;
+
AtlasGraph graph = context.getGraph();
// index query directly on entity
@@ -156,19 +160,12 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_OR_PATTERN.matcher(indexQueryString).replaceAll(")");
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
- Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
-
- if (typeNamePredicate != null) {
- inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
- }
-
- Predicate attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
+ this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
- if (attributePredicate != null) {
- inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
- }
+ typeNamePredicate = isClassificationRootType() ? null :
+ SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, indexAttributes);
- this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
} else {
classificationIndexQuery = null;
}
@@ -176,7 +173,6 @@ public class ClassificationSearchProcessor extends SearchProcessor {
// only registered classification will search with tag filters
if (useGraphSearchForClassification) {
- AtlasGremlinQueryProvider queryProvider = AtlasGremlinQueryProvider.INSTANCE;
AtlasGraphQuery query = graph.query();
if (!isClassificationRootType()) {
@@ -184,31 +180,20 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
tagGraphQueryWithAttributes = toGraphFilterQuery(classificationTypes, filterCriteria, allAttributes, query);
- gremlinQueryBindings = new HashMap<>();
- StringBuilder gremlinQuery = new StringBuilder();
- gremlinQuery.append("g.V().has('__guid', within(guids))");
- gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_CLASSIFICATION_FILTER));
- gremlinQuery.append(".as('e').filter(out()");
- gremlinQuery.append(queryProvider.getQuery(AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_TYPE_FILTER));
+ typeNamePredicate = isClassificationRootType() ? null :
+ SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ attributePredicate = constructInMemoryPredicate(classificationTypes, filterCriteria, allAttributes);
- // constructGremlinFilterQuery(gremlinQuery, gremlinQueryBindings, context.getClassificationType(), context.getSearchParameters().getTagFilters());
-
- // After filtering on tags go back to e and output the list of entity vertices
- gremlinQuery.append(").toList()");
-
- gremlinQueryBindings.put("traitNames", typeAndSubTypes);
- gremlinQueryBindings.put("typeNames", typeAndSubTypes); // classification typeName
-
- gremlinTagFilterQuery = gremlinQuery.toString();
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("gremlinTagFilterQuery={}", gremlinTagFilterQuery);
- }
} else {
tagGraphQueryWithAttributes = null;
- gremlinTagFilterQuery = null;
- gremlinQueryBindings = null;
+ }
+
+ if (typeNamePredicate != null) {
+ inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
+ }
+ if (attributePredicate != null) {
+ inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
}
@@ -274,23 +259,23 @@ public class ClassificationSearchProcessor extends SearchProcessor {
CollectionUtils.filter(entityVertices, isEntityPredicate);
} else {
- if (tagGraphQueryWithAttributes != null) {
+ if (classificationIndexQuery != null) {
- Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
+ Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
- getVertices(queryResult, classificationVertices);
+ getVerticesFromIndexQueryResult(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
- } else if (classificationIndexQuery != null){
+ CollectionUtils.filter(classificationVertices, inMemoryPredicate);
+ } else if (tagGraphQueryWithAttributes != null) {
- Iterator<AtlasIndexQuery.Result> queryResult = classificationIndexQuery.vertices(qryOffset, limit);
+ Iterator<AtlasVertex> queryResult = tagGraphQueryWithAttributes.vertices(qryOffset, limit).iterator();
- getVerticesFromIndexQueryResult(queryResult, classificationVertices);
+ getVertices(queryResult, classificationVertices);
isLastResultPage = classificationVertices.size() < limit;
- // Do in-memory filtering before the graph query
CollectionUtils.filter(classificationVertices, inMemoryPredicate);
}
}
@@ -322,6 +307,9 @@ public class ClassificationSearchProcessor extends SearchProcessor {
}
// Do in-memory filtering
CollectionUtils.filter(entityVertices, isEntityPredicate);
+ if (activePredicate != null) {
+ CollectionUtils.filter(entityVertices, activePredicate);
+ }
super.filter(entityVertices);
@@ -347,30 +335,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
if (LOG.isDebugEnabled()) {
LOG.debug("==> ClassificationSearchProcessor.filter({})", entityVertices.size());
}
- //in case of classification type + graph attributes
- if (gremlinTagFilterQuery != null && gremlinQueryBindings != null) {
- // Now filter on the tag attributes
- Set<String> guids = getGuids(entityVertices);
-
- // Clear prior results
- entityVertices.clear();
-
- if (CollectionUtils.isNotEmpty(guids)) {
- gremlinQueryBindings.put("guids", guids);
- try {
- AtlasGraph graph = context.getGraph();
- ScriptEngine gremlinScriptEngine = graph.getGremlinScriptEngine();
- List<AtlasVertex> atlasVertices = (List<AtlasVertex>) graph.executeGremlinScript(gremlinScriptEngine, gremlinQueryBindings, gremlinTagFilterQuery, false);
-
- if (CollectionUtils.isNotEmpty(atlasVertices)) {
- entityVertices.addAll(atlasVertices);
- }
- } catch (AtlasBaseException | ScriptException e) {
- LOG.warn(e.getMessage(), e);
- }
- }
- } else if (inMemoryPredicate != null) {
+ if (inMemoryPredicate != null) {
//in case of classification type + index attributes
CollectionUtils.filter(entityVertices, traitPredicate);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index c62f805..c5e3d6f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -464,9 +464,9 @@ public abstract class SearchProcessor {
AtlasType attributeType = structType.getAttributeType(filterCriteria.getAttributeName());
if (AtlasBaseTypeDef.ATLAS_TYPE_STRING.equals(attributeType.getTypeName())) {
- if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ) {
+ if (filterCriteria.getOperator() == SearchParameters.Operator.NEQ || filterCriteria.getOperator() == SearchParameters.Operator.NOT_CONTAINS) {
if (LOG.isDebugEnabled()) {
- LOG.debug("NEQ operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", qualifiedName);
+ LOG.debug("{} operator found for string attribute {}, deferring to in-memory or graph query (might cause poor performance)", filterCriteria.getOperator(), qualifiedName);
}
ret = false;
@@ -633,6 +633,7 @@ public abstract class SearchProcessor {
op = SearchParameters.Operator.NOT_CONTAINS;
break;
case CONTAINS:
+ case NOT_CONTAINS:
if (attrName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY)) {
attrVal = getCustomAttributeIndexQueryValue(attrVal, true);
}
@@ -849,6 +850,8 @@ public abstract class SearchProcessor {
case NOT_NULL:
innerQry.has(qualifiedName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, null);
break;
+ case NOT_CONTAINS:
+ break;
default:
LOG.warn("{}: unsupported operator. Ignored", operator);
break;
diff --git a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
index 8b98b39..270051f 100644
--- a/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
+++ b/repository/src/test/java/org/apache/atlas/BasicTestSetup.java
@@ -19,10 +19,12 @@ package org.apache.atlas;
import com.google.common.collect.ImmutableList;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.SearchParameters;
-import org.apache.atlas.model.instance.AtlasClassification;
-import org.apache.atlas.model.instance.AtlasEntity;
-import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.glossary.AtlasGlossary;
+import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
+import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
+import org.apache.atlas.model.instance.*;
import org.apache.atlas.model.typedef.*;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
@@ -42,6 +44,7 @@ import static org.testng.Assert.fail;
public abstract class BasicTestSetup {
+ // Entity type //
protected static final String DATABASE_TYPE = "hive_db";
protected static final String HIVE_TABLE_TYPE = "hive_table";
private static final String COLUMN_TYPE = "hive_column";
@@ -50,6 +53,7 @@ public abstract class BasicTestSetup {
private static final String VIEW_TYPE = "hive_process";
protected static final String DATASET_SUBTYPE = "Asset";
+ //Classification type //
public static final String DIMENSION_CLASSIFICATION = "Dimension";
public static final String FACT_CLASSIFICATION = "Fact";
public static final String PII_CLASSIFICATION = "PII";
@@ -59,14 +63,21 @@ public abstract class BasicTestSetup {
public static final String LOGDATA_CLASSIFICATION = "Log Data";
public static final String DIMENSIONAL_CLASSIFICATION = "Dimensional";
+ // Glossary type //
+ public static final String SALES_GLOSSARY = "salesGlossary";
+ public static final String SALES_TERM = "salesTerm";
+
@Inject
protected AtlasTypeRegistry typeRegistry;
@Inject
protected AtlasTypeDefStore typeDefStore;
@Inject
protected AtlasEntityStore entityStore;
+ @Inject
+ protected GlossaryService glossaryService;
private boolean baseLoaded = false;
+ private EntityMutationResponse hiveEntities;
protected void setupTestData() {
loadBaseModels();
@@ -77,6 +88,7 @@ public abstract class BasicTestSetup {
private void loadBaseModels() {
try {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("0000-Area0/0011-glossary_model.json", typeDefStore, typeRegistry);
baseLoaded = true;
} catch (IOException | AtlasBaseException e) {
fail("Base model setup is required for test to run!");
@@ -97,7 +109,7 @@ public abstract class BasicTestSetup {
AtlasEntity.AtlasEntitiesWithExtInfo hiveTestEntities = hiveTestEntities();
try {
- entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
+ hiveEntities = entityStore.createOrUpdate(new AtlasEntityStream(hiveTestEntities), false);
} catch (AtlasBaseException e) {
fail("Hive entities need to be created for test to run!");
}
@@ -450,12 +462,13 @@ public abstract class BasicTestSetup {
return datasetSubType;
}
- public void createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
+ public EntityMutationResponse createDummyEntity(String name, String type, String... traitNames) throws AtlasBaseException {
AtlasEntity entity = new AtlasEntity(type);
entity.setAttribute("name", name);
entity.setAttribute(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
entity.setClassifications(Stream.of(traitNames).map(AtlasClassification::new).collect(Collectors.toList()));
- entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
+ EntityMutationResponse resp = entityStore.createOrUpdate(new AtlasEntityStream(new AtlasEntity.AtlasEntitiesWithExtInfo(entity)), false);
+ return resp;
}
public SearchParameters.FilterCriteria getSingleFilterCondition(String attName, SearchParameters.Operator op, String attrValue) {
@@ -472,4 +485,22 @@ public abstract class BasicTestSetup {
return filterCriteria;
}
+ public void assignGlossary() throws AtlasBaseException {
+ AtlasGlossary glossary = new AtlasGlossary();
+ glossary.setName(SALES_GLOSSARY);
+ glossary = glossaryService.createGlossary(glossary);
+
+ AtlasGlossaryTerm term = new AtlasGlossaryTerm();
+ term.setAnchor(new AtlasGlossaryHeader(glossary.getGuid()));
+ term.setName(SALES_TERM);
+ term = glossaryService.createTerm(term);
+
+ List<AtlasRelatedObjectId> guids = hiveEntities.getCreatedEntities().stream().filter(e -> e.getTypeName().equals(HIVE_TABLE_TYPE))
+ .map(p -> {AtlasRelatedObjectId obj = new AtlasRelatedObjectId();
+ obj.setGuid(p.getGuid());
+ obj.setTypeName(p.getTypeName()); return obj;}).collect(Collectors.toList());
+
+ glossaryService.assignTermToEntities(term.getGuid(), guids);
+ }
+
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
new file mode 100644
index 0000000..d8c1546
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/discovery/AtlasDiscoveryServiceTest.java
@@ -0,0 +1,338 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.discovery;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.commons.collections.CollectionUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import javax.inject.Inject;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+
+import static org.apache.atlas.model.discovery.SearchParameters.*;
+import static org.testng.Assert.assertEquals;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class AtlasDiscoveryServiceTest extends BasicTestSetup {
+
+ @Inject
+ private AtlasDiscoveryService discoveryService;
+
+ @BeforeClass
+ public void setup() throws AtlasException, AtlasBaseException {
+ ApplicationProperties.get().setProperty(ApplicationProperties.ENABLE_FREETEXT_SEARCH_CONF, true);
+ setupTestData();
+ createDimensionalTaggedEntity("sales");
+ assignGlossary();
+ }
+
+ /* TermSearchProcessor(TSP),
+ FreeTextSearchProcessor(FSP),
+ ClassificationSearchProcessor(CSP),
+ EntitySearchProcessor(ESP) */
+
+ @Test
+ public void term() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 10);
+ }
+
+ // TSP execute and CSP,ESP filter
+ @Test
+ public void term_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setClassification(METRIC_CLASSIFICATION);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ for(AtlasEntityHeader e : entityHeaders){
+ System.out.println(e.toString());
+ }
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void term_entity() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setTypeName(HIVE_TABLE_TYPE);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 10);
+ }
+
+ @Test
+ public void term_entity_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTermName(SALES_TERM+"@"+SALES_GLOSSARY);
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isEmpty(entityHeaders));
+ }
+
+ //FSP execute and CSP,ESP filter
+ @Test
+ public void query_ALLTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 5);
+ }
+
+ @Test
+ public void query_ALLTag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(ALL_CLASSIFICATION_TYPES);
+ //typeName will check for only classification name not propogated classification
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__typeName", Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void query_NOTCLASSIFIEDTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(NO_CLASSIFICATIONS);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+
+ @Test
+ public void query_ALLWildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("*");
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 5);
+ }
+
+ @Test
+ public void query_wildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("Dimen*on");
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+ }
+
+ @Test
+ public void query_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(METRIC_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_entity() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ @Test
+ public void query_entity_entityFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.NOT_NULL, "null");
+ params.setEntityFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 3);
+ }
+
+ @Test
+ public void query_entity_entityFilter_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
+ params.setEntityFilters(fc);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+ @Test
+ public void query_entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.IS_NULL, "null");
+ params.setEntityFilters(fcE);
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ params.setQuery("sales");
+ SearchParameters.FilterCriteria fcC = getSingleFilterCondition("attr1", Operator.EQ, "value1");
+ params.setTagFilters(fcC);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 1);
+ }
+
+ @Test
+ public void query_entity_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+
+ }
+
+ @Test
+ public void query_entity_tag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ params.setQuery("sales");
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 2);
+ }
+
+ // CSP Execute and ESP filter
+ @Test
+ public void entity_entityFilter_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria fcE = getSingleFilterCondition("tableType", Operator.EQ, "Managed");
+ params.setEntityFilters(fcE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fcC = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fcC);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+
+ }
+
+ @Test
+ public void entity_tag_tagFilter() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ params.setClassification(METRIC_CLASSIFICATION);
+ SearchParameters.FilterCriteria fc = getSingleFilterCondition("__timestamp", SearchParameters.Operator.LT, String.valueOf(System.currentTimeMillis()));
+ params.setTagFilters(fc);
+
+ List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ assertEquals(entityHeaders.size(), 4);
+ }
+
+ private void createDimensionalTaggedEntity(String name) throws AtlasBaseException {
+ EntityMutationResponse resp = createDummyEntity(name, HIVE_TABLE_TYPE);
+ AtlasEntityHeader entityHeader = resp.getCreatedEntities().get(0);
+ String guid = entityHeader.getGuid();
+ HashMap<String,Object> attr = new HashMap<>();
+ attr.put("attr1","value1");
+ entityStore.addClassification(Arrays.asList(guid), new AtlasClassification(DIMENSIONAL_CLASSIFICATION, attr));
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java b/repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
similarity index 52%
rename from repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
rename to repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
index 9b16e91..8693459 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/BasicSearchClassificationTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/ClassificationSearchProcessorTest.java
@@ -26,9 +26,15 @@ import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream;
+import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.commons.collections.CollectionUtils;
import org.testng.Assert;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -37,19 +43,23 @@ import javax.inject.Inject;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
+import java.util.stream.Collectors;
import static org.apache.atlas.model.discovery.SearchParameters.*;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.fail;
@Guice(modules = TestModules.TestOnlyModule.class)
-public class BasicSearchClassificationTest extends BasicTestSetup {
+public class ClassificationSearchProcessorTest extends BasicTestSetup {
@Inject
- private AtlasDiscoveryService discoveryService;
+ private AtlasGraph graph;
+ @Inject
+ public GraphBackedSearchIndexer indexer;
+ @Inject
+ private EntityGraphRetriever entityRetriever;
- private int totalEntities = 0;
private int totalClassifiedEntities = 0;
- private int getTotalClassifiedEntitiesHistorical = 0;
private int dimensionTagEntities = 10;
private String dimensionTagDeleteGuid;
private String dimensionalTagGuid;
@@ -65,14 +75,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
public void searchByALLTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(ALL_CLASSIFICATION_TYPES);
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- Assert.assertTrue(CollectionUtils.isNotEmpty(entityHeaders));
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- totalEntities = getEntityCount();
- totalClassifiedEntities = entityHeaders.size();
- getTotalClassifiedEntitiesHistorical = getEntityWithTagCountHistorical();
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ totalClassifiedEntities = vertices.size();
}
@Test
@@ -81,10 +91,14 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(ALL_CLASSIFICATION_TYPES);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria);
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), totalClassifiedEntities);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), totalClassifiedEntities);
}
@Test
@@ -95,29 +109,40 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setTagFilters(filterCriteria);
params.setLimit(totalClassifiedEntities - 2);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), totalClassifiedEntities - 2);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), totalClassifiedEntities - 2);
}
- @Test
+ //@Test
public void searchByNOTCLASSIFIED() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(NO_CLASSIFICATIONS);
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), totalEntities - totalClassifiedEntities);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 20);
}
@Test
public void searchByTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION);
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), dimensionTagEntities);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities);
}
@Test
@@ -126,11 +151,23 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSIONAL_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.EQ, "Test");
params.setTagFilters(filterCriteria);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), 1);
- assertEquals(entityHeaders.get(0).getGuid(), dimensionalTagGuid);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 1);
+ List<String> guids = vertices.stream().map(g -> {
+ try {
+ return entityRetriever.toAtlasEntityHeader(g).getGuid();
+ } catch (AtlasBaseException e) {
+ fail("Failure in mapping vertex to AtlasEntityHeader");
+ }
+ return "";
+ }).collect(Collectors.toList());
+ Assert.assertTrue(guids.contains(dimensionalTagGuid));
}
@@ -141,35 +178,86 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__timestamp", Operator.LT, String.valueOf(System.currentTimeMillis()));
params.setTagFilters(filterCriteria);
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), dimensionTagEntities);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities);
}
@Test
public void searchByWildcardTag() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification("Dimension*");
+ params.setLimit(20);
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
- assertEquals(entityHeaders.size(), dimensionTagEntities + 1);
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), dimensionTagEntities + 1);
}
- //@Test
+ @Test
+ public void searchByALLWildcardTag() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification("*");
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(),20);
+
+ }
+
+ @Test
+ public void searchWithNotContains() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setClassification(DIMENSIONAL_CLASSIFICATION);
+ FilterCriteria filterCriteria = getSingleFilterCondition("attr1", Operator.NOT_CONTAINS, "Test");
+ params.setTagFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isEmpty(vertices));
+ }
+
+
+ @Test
public void searchByTagAndGraphSysFilters() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setClassification(DIMENSION_CLASSIFICATION);
FilterCriteria filterCriteria = getSingleFilterCondition("__entityStatus", Operator.EQ, "DELETED");
params.setTagFilters(filterCriteria);
params.setExcludeDeletedEntities(false);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
-
- assertEquals(entityHeaders.size(), 1);
- assertEquals(entityHeaders.get(0).getGuid(), dimensionTagDeleteGuid);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ ClassificationSearchProcessor processor = new ClassificationSearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ Assert.assertTrue(CollectionUtils.isNotEmpty(vertices));
+ assertEquals(vertices.size(), 1);
+ List<String> guids = vertices.stream().map(g -> {
+ try {
+ return entityRetriever.toAtlasEntityHeader(g).getGuid();
+ } catch (AtlasBaseException e) {
+ fail("Failure in mapping vertex to AtlasEntityHeader");
+ }
+ return "";
+ }).collect(Collectors.toList());
+ Assert.assertTrue(guids.contains(dimensionTagDeleteGuid));
}
@@ -209,21 +297,8 @@ public class BasicSearchClassificationTest extends BasicTestSetup {
}
- private int getEntityCount() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setTypeName(ALL_ENTITY_TYPES);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
- return entityHeaders.size();
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
}
-
- private int getEntityWithTagCountHistorical() throws AtlasBaseException {
- SearchParameters params = new SearchParameters();
- params.setClassification(ALL_CLASSIFICATION_TYPES);
- params.setExcludeDeletedEntities(false);
-
- List<AtlasEntityHeader> entityHeaders = discoveryService.searchWithParameters(params).getEntities();
- return entityHeaders.size();
- }
-
}
diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index 8e42d17..7b40c21 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -23,11 +23,13 @@ import org.apache.atlas.SortOrder;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever;
import org.apache.atlas.type.AtlasTypeRegistry;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -60,6 +62,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Inject
public GraphBackedSearchIndexer indexer;
+ private String expectedEntityName = "hive_Table_Null_tableType";
@Test
public void searchTablesByClassification() throws AtlasBaseException {
@@ -132,7 +135,6 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Test(priority = -1)
public void searchWithNEQ_stringAttr() throws AtlasBaseException {
- String expectedEntityName = "hive_Table_Null_tableType";
createDummyEntity(expectedEntityName,HIVE_TABLE_TYPE);
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -154,7 +156,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertTrue(nameList.contains(expectedEntityName));
}
- @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ @Test
public void searchWithNEQ_pipeSeperatedAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -173,10 +175,10 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
}
- assertTrue(nameList.contains("hive_Table_Null_tableType"));
+ assertTrue(nameList.contains(expectedEntityName));
}
- @Test(dependsOnMethods = "searchWithNEQ_stringAttr")
+ @Test
public void searchWithNEQ_doubleAttr() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
params.setTypeName(HIVE_TABLE_TYPE);
@@ -309,4 +311,53 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
assertEquals(processor.execute().size(), 2);
}
+ @Test
+ public void searchWithNotContains_stringAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("tableType", SearchParameters.Operator.NOT_CONTAINS, "Managed");
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 3);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains(expectedEntityName));
+ }
+
+ @Test
+ public void searchWithNotContains_pipeSeperatedAttr() throws AtlasBaseException {
+ SearchParameters params = new SearchParameters();
+ params.setTypeName(HIVE_TABLE_TYPE);
+ SearchParameters.FilterCriteria filterCriteria = getSingleFilterCondition("__classificationNames", SearchParameters.Operator.NOT_CONTAINS, METRIC_CLASSIFICATION);
+ params.setEntityFilters(filterCriteria);
+ params.setLimit(20);
+
+ SearchContext context = new SearchContext(params, typeRegistry, graph, indexer.getVertexIndexKeys());
+ EntitySearchProcessor processor = new EntitySearchProcessor(context);
+ List<AtlasVertex> vertices = processor.execute();
+
+ assertEquals(vertices.size(), 7);
+
+ List<String> nameList = new ArrayList<>();
+ for (AtlasVertex vertex : vertices) {
+ nameList.add((String) entityRetriever.toAtlasEntityHeader(vertex, Collections.singleton("name")).getAttribute("name"));
+ }
+
+ assertTrue(nameList.contains(expectedEntityName));
+ }
+
+ @AfterClass
+ public void teardown() {
+ AtlasGraphProvider.cleanup();
+ }
+
}
[atlas] 04/05: ATLAS-3888 : BasicSearch: Multiple type/tag:
[Regression], Restrict when tag filters and regex in tag
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit e97fb302231a3b452d99cd892a75a09b9499d24a
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Fri Jul 10 15:28:30 2020 +0530
ATLAS-3888 : BasicSearch: Multiple type/tag: [Regression], Restrict when tag filters and regex in tag
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit d9d0e8c2abe293a520f89b85015d9c52985e28cb)
---
.../main/java/org/apache/atlas/discovery/SearchContext.java | 11 +++++++++++
1 file changed, 11 insertions(+)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index dd24a8c..2cb287f 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -106,6 +106,17 @@ public class SearchContext {
}
}
+ //Wildcard tag with filter will raise an exception with 400 error code
+ if (CollectionUtils.isNotEmpty(classificationNames) && hasAttributeFilter(searchParameters.getTagFilters())) {
+ for (String classificationName : classificationNames){
+ //in case of '*' , filters are allowed, but
+ //in case of regex 'PI*', filters are not allowed ( if present in any of the requested tag)
+ if (classificationName.contains(WILDCARD_CLASSIFICATIONS) && !classificationName.equals(WILDCARD_CLASSIFICATIONS)) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified with wildcard tag name");
+ }
+ }
+ }
+
// Invalid attributes will raise an exception with 400 error code
if (CollectionUtils.isNotEmpty(classificationTypes)) {
for (AtlasClassificationType classificationType : classificationTypes) {
[atlas] 03/05: ATLAS-3880 : BasicSearch: Multiple type/tag: Log
invalid and allow searching valid type/tag names
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 2a7d321119cd60f5adb08ea9fa10d0baa3bd662b
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Tue Jul 7 19:03:40 2020 +0530
ATLAS-3880 : BasicSearch: Multiple type/tag: Log invalid and allow searching valid type/tag names
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit f690755f79f9696e3214b9a3f158977b3bd5f4a7)
---
.../org/apache/atlas/discovery/SearchContext.java | 42 +++++++++++-----------
.../atlas/discovery/EntitySearchProcessorTest.java | 2 +-
.../org/apache/atlas/web/rest/DiscoveryREST.java | 11 ++++--
3 files changed, 32 insertions(+), 23 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 04e8218..dd24a8c 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -359,7 +359,7 @@ public class SearchContext {
Set<String> classificationNames = new HashSet<>();
if (StringUtils.isNotEmpty(classification)) {
- String[] types = classification.split(TYPENAME_DELIMITER);
+ String[] types = classification.split(TYPENAME_DELIMITER);
Set<String> names = new HashSet<>(Arrays.asList(types));
names.forEach(name -> {
@@ -370,11 +370,13 @@ public class SearchContext {
});
// Validate if the classification exists
- if (CollectionUtils.isEmpty(classificationNames) || classificationNames.size() != names.size()) {
- if (CollectionUtils.isNotEmpty(classificationNames)) {
- names.removeAll(classificationNames);
- }
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, String.join(TYPENAME_DELIMITER, names));
+ if (CollectionUtils.isEmpty(classificationNames)) {
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_CLASSIFICATION, classification);
+
+ } else if (classificationNames.size() != names.size()) {
+ names.removeAll(classificationNames);
+
+ LOG.info("Could not search for {} , invalid classifications", String.join(TYPENAME_DELIMITER, names));
}
}
@@ -398,24 +400,24 @@ public class SearchContext {
getEntityType(n)).filter(Objects::nonNull).collect(Collectors.toSet());
// Validate if the type name is incorrect
- if (CollectionUtils.isEmpty(entityTypes) || entityTypes.size() != typeNames.size()) {
- if (CollectionUtils.isNotEmpty(entityTypes)) {
- Set<String> validEntityTypes = new HashSet<>();
- for (AtlasEntityType entityType : entityTypes) {
- String name = entityType.getTypeName();
- if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
- validEntityTypes.add(ALL_ENTITY_TYPES);
- continue;
- }
- validEntityTypes.add(entityType.getTypeName());
+ if (CollectionUtils.isEmpty(entityTypes)) {
+ throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME,typeName);
+
+ } else if (entityTypes.size() != typeNames.size()) {
+ Set<String> validEntityTypes = new HashSet<>();
+ for (AtlasEntityType entityType : entityTypes) {
+ String name = entityType.getTypeName();
+ if (name.equals(MATCH_ALL_ENTITY_TYPES.getTypeName())) {
+ validEntityTypes.add(ALL_ENTITY_TYPES);
+ continue;
}
-
- typeNames.removeAll(validEntityTypes);
+ validEntityTypes.add(entityType.getTypeName());
}
- throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_TYPENAME, String.join(TYPENAME_DELIMITER, typeNames));
- }
+ typeNames.removeAll(validEntityTypes);
+ LOG.info("Could not search for {} , invalid typeNames", String.join(TYPENAME_DELIMITER, typeNames));
+ }
}
return entityTypes;
diff --git a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
index b7ce978..8e42d17 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/EntitySearchProcessorTest.java
@@ -250,7 +250,7 @@ public class EntitySearchProcessorTest extends BasicTestSetup {
@Test(expectedExceptions = AtlasBaseException.class, expectedExceptionsMessageRegExp = "Not_Exists: Unknown/invalid typename")
public void entityTypesNotAllowed() throws AtlasBaseException {
SearchParameters params = new SearchParameters();
- params.setTypeName(DATABASE_TYPE+",Not_Exists");
+ params.setTypeName("Not_Exists");
params.setLimit(20);
SearchContext context = new SearchContext(params, typeRegistry, graph, Collections.<String>emptySet());
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 3a45e1c..dd9980a 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -326,9 +326,16 @@ public class DiscoveryREST {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Limit/offset should be non-negative");
}
+ if (StringUtils.isEmpty(parameters.getTypeName()) && !isEmpty(parameters.getEntityFilters())) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "EntityFilters specified without Type name");
+ }
+
+ if (StringUtils.isEmpty(parameters.getClassification()) && !isEmpty(parameters.getTagFilters())) {
+ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name");
+ }
+
if (StringUtils.isEmpty(parameters.getTypeName()) && StringUtils.isEmpty(parameters.getClassification()) &&
- StringUtils.isEmpty(parameters.getQuery()) && StringUtils.isEmpty(parameters.getTermName()) &&
- isEmpty(parameters.getEntityFilters()) && isEmpty(parameters.getTagFilters())) {
+ StringUtils.isEmpty(parameters.getQuery()) && StringUtils.isEmpty(parameters.getTermName())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_SEARCH_PARAMS);
}
[atlas] 01/05: ATLAS-3848 : Quick Search : Fixed incorrect
aggregation metrics
Posted by ni...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
nixon pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 11b73ee0a58e93bb2df78d15dac7b850a60ac799
Author: Pinal Shah <pi...@freestoneinfotech.com>
AuthorDate: Thu Jun 18 17:49:31 2020 +0530
ATLAS-3848 : Quick Search : Fixed incorrect aggregation metrics
Signed-off-by: nixonrodrigues <ni...@apache.org>
(cherry picked from commit aa86d62c057d56f858dc7962aa158d23c82f972b)
---
.../graphdb/janus/AtlasSolrQueryBuilder.java | 34 +++++++++++++++++++++-
.../graphdb/janus/AtlasSolrQueryBuilderTest.java | 22 +++++++-------
.../org/apache/atlas/type/AtlasTypeRegistry.java | 18 ++++++++++++
3 files changed, 62 insertions(+), 12 deletions(-)
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
index 05db148..b8fd4a0 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilder.java
@@ -34,6 +34,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
+
public class AtlasSolrQueryBuilder {
private static final Logger LOG = LoggerFactory.getLogger(AtlasSolrQueryBuilder.class);
@@ -43,6 +45,8 @@ public class AtlasSolrQueryBuilder {
private boolean excludeDeletedEntities;
private boolean includeSubtypes;
private Map<String, String> indexFieldNameCache;
+ public static final char CUSTOM_ATTR_SEPARATOR = '=';
+ public static final String CUSTOM_ATTR_SEARCH_FORMAT = "\"\\\"%s\\\":\\\"%s\\\"\"";
public AtlasSolrQueryBuilder() {
@@ -211,6 +215,14 @@ public class AtlasSolrQueryBuilder {
attributeValue = attributeValue.trim();
}
+ if (attributeName.equals(CUSTOM_ATTRIBUTES_PROPERTY_KEY) && operator.equals(Operator.CONTAINS)) {
+ // CustomAttributes stores key value pairs in String format, so ideally it should be 'contains' operator to search for one pair,
+ // for use-case, E1 having key1=value1 and E2 having key1=value2, searching key1=value1 results both E1,E2
+ // surrounding inverted commas to attributeValue works
+ operator = Operator.EQ;
+ attributeValue = getIndexQueryAttributeValue(attributeValue);
+ }
+
AtlasAttribute attribute = entityType.getAttribute(attributeName);
if (attribute == null) {
@@ -231,6 +243,8 @@ public class AtlasSolrQueryBuilder {
throw new AtlasBaseException(msg);
}
+ beginCriteria(queryBuilder);
+
switch (operator) {
case EQ:
withEqual(queryBuilder, indexFieldName, attributeValue);
@@ -274,9 +288,27 @@ public class AtlasSolrQueryBuilder {
LOG.error(msg);
throw new AtlasBaseException(msg);
}
+
+ endCriteria(queryBuilder);
}
}
+ private String getIndexQueryAttributeValue(String attributeValue) {
+
+ if (StringUtils.isNotEmpty(attributeValue)) {
+ int separatorIdx = attributeValue.indexOf(CUSTOM_ATTR_SEPARATOR);
+ String key = separatorIdx != -1 ? attributeValue.substring(0, separatorIdx) : null;
+ String value = key != null ? attributeValue.substring(separatorIdx + 1) : null;
+
+ if (key != null && value != null) {
+ return String.format(CUSTOM_ATTR_SEARCH_FORMAT, key, value);
+ }
+ }
+
+ return attributeValue;
+ }
+
+
private void beginCriteria(StringBuilder queryBuilder) {
queryBuilder.append("( ");
}
@@ -294,7 +326,7 @@ public class AtlasSolrQueryBuilder {
}
private void withNotEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
- queryBuilder.append("-").append(indexFieldName).append(":").append(attributeValue).append(" ");
+ queryBuilder.append("*:* -").append(indexFieldName).append(":").append(attributeValue).append(" ");
}
private void withEqual(StringBuilder queryBuilder, String indexFieldName, String attributeValue) {
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
index 2af1818..60d8610 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/AtlasSolrQueryBuilderTest.java
@@ -114,7 +114,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 OR +comment_index:*t10* )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) OR ( +comment_index:*t10* ) )");
}
@Test
@@ -124,7 +124,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
@@ -134,7 +134,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 AND +comment_index:*t10* )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) AND ( +comment_index:*t10* ) )");
}
@Test
@@ -144,7 +144,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +name_index:t10 ) )");
}
@Test
@@ -154,7 +154,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND +name_index:t10 ");
+ Assert.assertEquals(underTest.build(), "+t AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +name_index:t10 )");
}
@Test
@@ -164,7 +164,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +comment_index:*United States* AND +descrption__index:*nothing* AND +name_index:*t100* )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +comment_index:*United States* ) AND ( +descrption__index:*nothing* ) AND ( +name_index:*t100* ) )");
}
@Test
@@ -174,7 +174,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:{ 100 TO * ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:{ 100 TO * ] ) )");
}
@Test
@@ -184,7 +184,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ 100 TO * ] AND +started__index:[ 100 TO * ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ 100 TO * ] ) AND ( +started__index:[ 100 TO * ] ) )");
}
@Test
@@ -194,7 +194,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO100} )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO100} ) )");
}
@Test
@@ -204,7 +204,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +created__index:[ * TO 100 ] AND +started__index:[ * TO 100 ] )");
+ Assert.assertEquals(underTest.build(), "+t10 AND -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +created__index:[ * TO 100 ] ) AND ( +started__index:[ * TO 100 ] ) )");
}
@Test
@@ -214,7 +214,7 @@ public class AtlasSolrQueryBuilderTest {
processSearchParameters(fileName, underTest);
- Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( +qualifiedName__index:testdb.t1* )");
+ Assert.assertEquals(underTest.build(), " -__state_index:DELETED AND +__typeName__index:(hive_table ) AND ( ( +qualifiedName__index:testdb.t1* ) )");
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 2c7cb2b..4a79b6f 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -62,6 +62,7 @@ public class AtlasTypeRegistry {
commonIndexFieldNameCache = new HashMap<>();
resolveReferencesForRootTypes();
+ resolveIndexFieldNamesForRootTypes();
}
// used only by AtlasTransientTypeRegistry
@@ -72,6 +73,7 @@ public class AtlasTypeRegistry {
commonIndexFieldNameCache = other.commonIndexFieldNameCache;
resolveReferencesForRootTypes();
+ resolveIndexFieldNamesForRootTypes();
}
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
@@ -275,6 +277,22 @@ public class AtlasTypeRegistry {
}
}
+ private void resolveIndexFieldNamesForRootTypes() {
+ for (AtlasStructType structType : Arrays.asList(AtlasEntityType.ENTITY_ROOT, AtlasClassificationType.CLASSIFICATION_ROOT)) {
+ for (AtlasAttribute attribute : structType.getAllAttributes().values()) {
+ String indexFieldName = getIndexFieldName(attribute.getVertexPropertyName());
+
+ if (StringUtils.isNotEmpty(indexFieldName)) {
+ attribute.setIndexFieldName(indexFieldName);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Attribute {} with index name {} is added", attribute.getVertexPropertyName(), attribute.getIndexFieldName());
+ }
+ }
+ }
+ }
+
/**
* retrieves the index field name for the common field passed in.
* @param propertyName the name of the common field.