You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2019/11/28 00:56:17 UTC
[atlas] 01/02: ATLAS-3533: search with term and tag doesn't return
right results
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit d0c1b30c05a587839015db1c3ff1dafc959294fd
Author: Le Ma <lm...@cloudera.com>
AuthorDate: Mon Nov 25 11:54:29 2019 -0800
ATLAS-3533: search with term and tag doesn't return right results
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
---
.../test/java/org/apache/atlas/TestUtilsV2.java | 2 +-
.../discovery/ClassificationSearchProcessor.java | 21 +++-
.../atlas/discovery/EntitySearchProcessor.java | 12 +--
.../org/apache/atlas/discovery/SearchContext.java | 2 +-
.../apache/atlas/discovery/SearchProcessor.java | 21 ++++
.../impexp/ZipFileResourceTestUtils.java | 2 +-
.../atlas/web/adapters/TestEntitiesREST.java | 110 ++++++++++++++++++++-
7 files changed, 150 insertions(+), 20 deletions(-)
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index ff79994..6fe5063 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -548,7 +548,7 @@ public final class TestUtilsV2 {
public static final String PII = "PII";
public static final String PHI = "PHI";
- public static final String SUPER_TYPE_NAME = "Base";
+ public static final String SUPER_TYPE_NAME = "Referenceable";
public static final String STORAGE_DESC_TYPE = "hive_storagedesc";
public static final String PARTITION_STRUCT_TYPE = "partition_struct_type";
public static final String PARTITION_CLASS_TYPE = "partition_class_type";
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 63880f9..c0a5a46 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -69,13 +69,14 @@ public class ClassificationSearchProcessor extends SearchProcessor {
private final AtlasGraphQuery tagGraphQueryWithAttributes;
private final Map<String, Object> gremlinQueryBindings;
private final String gremlinTagFilterQuery;
+ private final Predicate traitPredicate;
// Some index engines may take space as a delimiter, when basic search is
// executed, unsatisfying results may be returned.
// eg, an entity A has classification "cls" and B has "cls 1"
// when user execute a exact search for "cls", only A should be returned
// but both A and B are returned. To avoid this, we should filter the res.
- private boolean whiteSpaceFilter = false;
+ private boolean whiteSpaceFilter = false;
public ClassificationSearchProcessor(SearchContext context) {
super(context);
@@ -146,8 +147,13 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
LOG.debug("Using query string '{}'.", indexQuery);
+
+ traitPredicate = buildTraitPredict(classificationType);
+ inMemoryPredicate = inMemoryPredicate == null ? traitPredicate : PredicateUtils.andPredicate(inMemoryPredicate, traitPredicate);
+
} else {
- indexQuery = null;
+ indexQuery = null;
+ traitPredicate = null;
}
// index query directly on classification
@@ -165,12 +171,15 @@ public class ClassificationSearchProcessor extends SearchProcessor {
indexQueryString = STRAY_ELIPSIS_PATTERN.matcher(indexQueryString).replaceAll("");
Predicate typeNamePredicate = isClassificationRootType() ? null : SearchPredicateUtil.getINPredicateGenerator().generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+
+ if (typeNamePredicate != null) {
+ inMemoryPredicate = inMemoryPredicate == null ? typeNamePredicate : PredicateUtils.andPredicate(inMemoryPredicate, typeNamePredicate);
+ }
+
Predicate attributePredicate = constructInMemoryPredicate(classificationType, filterCriteria, indexAttributes);
if (attributePredicate != null) {
- inMemoryPredicate = typeNamePredicate == null ? attributePredicate : PredicateUtils.andPredicate(typeNamePredicate, attributePredicate);
- } else {
- inMemoryPredicate = typeNamePredicate;
+ inMemoryPredicate = inMemoryPredicate == null ? attributePredicate : PredicateUtils.andPredicate(inMemoryPredicate, attributePredicate);
}
this.classificationIndexQuery = graph.indexQuery(Constants.VERTEX_INDEX, indexQueryString);
@@ -360,6 +369,8 @@ public class ClassificationSearchProcessor extends SearchProcessor {
LOG.warn(e.getMessage(), e);
}
}
+ } else if (traitPredicate != null) {
+ CollectionUtils.filter(entityVertices, traitPredicate);
}
super.filter(entityVertices);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 93dbe87..b5606d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -88,20 +88,10 @@ public class EntitySearchProcessor extends SearchProcessor {
}
final Predicate typeNamePredicate;
- final Predicate traitPredicate;
+ final Predicate traitPredicate = buildTraitPredict(classificationType);
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
- if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
- } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
- traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
- SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
- } else {
- traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class),
- SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
- }
if (!isEntityRootType()) {
typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index ef50d8d..3534113 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -243,7 +243,7 @@ public class SearchContext {
}
boolean needClassificationProcessor() {
- return (classificationType != null || isWildCardSearch());
+ return (classificationType != null && (entityType == null || hasAttributeFilter(searchParameters.getTagFilters()))) || isWildCardSearch() ;
}
boolean isBuiltInClassificationType() {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index b04021f..76d5a01 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -32,6 +32,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
import org.apache.atlas.type.*;
import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
import org.apache.atlas.util.AtlasGremlinQueryProvider;
+import org.apache.atlas.util.SearchPredicateUtil;
import org.apache.atlas.util.SearchPredicateUtil.*;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.Predicate;
@@ -45,10 +46,15 @@ import java.math.BigInteger;
import java.util.*;
import java.util.regex.Pattern;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_NAMES_KEY;
import static org.apache.atlas.repository.Constants.CUSTOM_ATTRIBUTES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.LABELS_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.PROPAGATED_CLASSIFICATION_NAMES_KEY;
+import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.util.SearchPredicateUtil.*;
public abstract class SearchProcessor {
@@ -179,6 +185,21 @@ public abstract class SearchProcessor {
}
}
+ protected Predicate buildTraitPredict(AtlasClassificationType classificationType) {
+ Predicate traitPredicate;
+ if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED || context.isWildCardSearch()) {
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ } else if (classificationType == MATCH_ALL_NOT_CLASSIFIED) {
+ traitPredicate = PredicateUtils.andPredicate(SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
+ SearchPredicateUtil.getIsNullOrEmptyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, null, List.class));
+ } else {
+ traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class),
+ SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, context.getClassificationTypes(), List.class));
+ }
+ return traitPredicate;
+ }
+
protected void processSearchAttributes(AtlasStructType structType, FilterCriteria filterCriteria, Set<String> indexFiltered, Set<String> graphFiltered, Set<String> allAttributes) {
if (structType == null || filterCriteria == null) {
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
index f4e84b2..0ffc3d5 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/ZipFileResourceTestUtils.java
@@ -294,7 +294,7 @@ public class ZipFileResourceTestUtils {
createTypesAsNeeded(typesFromJson, typeDefStore, typeRegistry);
}
- private static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ public static void createTypesAsNeeded(AtlasTypesDef typesFromJson, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
if(typesFromJson == null) {
return;
}
diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
index e2906a2..afc4c50 100644
--- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
+++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
@@ -30,19 +30,26 @@ import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPE
import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.createTypesAsNeeded;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.glossary.GlossaryService;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
+import org.apache.atlas.model.glossary.AtlasGlossary;
+import org.apache.atlas.model.glossary.AtlasGlossaryTerm;
+import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntityHeader;
import org.apache.atlas.model.instance.AtlasObjectId;
+import org.apache.atlas.model.instance.AtlasRelatedObjectId;
import org.apache.atlas.model.instance.AtlasStruct;
import org.apache.atlas.model.instance.ClassificationAssociateRequest;
import org.apache.atlas.model.instance.EntityMutationResponse;
@@ -55,6 +62,7 @@ import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
import org.apache.atlas.web.rest.DiscoveryREST;
import org.apache.atlas.web.rest.EntityREST;
+import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testng.Assert;
@@ -64,6 +72,8 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
+
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -81,12 +91,16 @@ public class TestEntitiesREST {
@Inject
private AtlasTypeRegistry typeRegistry;
@Inject
+ private GlossaryService glossaryService;
+ @Inject
private AtlasTypeDefStore typeStore;
@Inject
private DiscoveryREST discoveryREST;
@Inject
private EntityREST entityREST;
+ private AtlasGlossary glossary;
+ private AtlasGlossaryTerm term1;
private AtlasEntity dbEntity;
private AtlasEntity tableEntity;
private AtlasEntity tableEntity2;
@@ -108,8 +122,14 @@ public class TestEntitiesREST {
}
}
+ loadGlossaryType();
+
createEntities();
+ createGlossary();
+
+ createTerms();
+
initTagMap();
registerEntities();
@@ -372,6 +392,33 @@ public class TestEntitiesREST {
}
@Test(dependsOnMethods = "testSearchByMultiTags")
+ public void testSearchByTerms() throws Exception {
+
+ // database - phi, felt_classification
+ // table1 - phi, classification, term: term1 | table2 - classification, term:term2
+ // column - phi
+ assignTermTo(term1, tableEntity);
+ assignTermTo(term1, tableEntity2);
+
+ searchParameters = new SearchParameters();
+ searchParameters.setTermName(term1.getName() + "@testSearchGlossary");
+ searchParameters.setClassification(CLASSIFICATION);
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters.setClassification(PII);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
+
+ searchParameters.setClassification(PHI);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 1);
+ }
+
+ @Test(dependsOnMethods = "testSearchByMultiTags")
public void testSearchByOtherSystemAttributes() throws Exception {
// database - phi, felt_classification
@@ -506,6 +553,14 @@ public class TestEntitiesREST {
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
+
+ searchParameters = new SearchParameters();
+ searchParameters.setQuery("classification");
+ searchParameters.setClassification(PHI);
+
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 1);
}
@Test(dependsOnMethods = "testSearchBySystemAttributesWithQuery")
@@ -536,7 +591,7 @@ public class TestEntitiesREST {
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 5);
+ Assert.assertEquals(res.getEntities().size(), 7);
}
@Test(dependsOnMethods = "testTagSearchBySystemAttributes")
@@ -582,6 +637,59 @@ public class TestEntitiesREST {
*
*/
+ private void loadGlossaryType() throws IOException, AtlasBaseException {
+
+ registerAtlasTypesDef("/addons/models/0000-Area0/0010-base_model.json");
+ registerAtlasTypesDef("/addons/models/0000-Area0/0011-glossary_model.json");
+ }
+
+ private void registerAtlasTypesDef (String path) throws IOException, AtlasBaseException {
+ String projectBaseDirectory = System.getProperty("projectBaseDir");
+
+ String baseModel = projectBaseDirectory + path;
+ File f = new File(baseModel);
+ String s = FileUtils.readFileToString(f);
+ createTypesAsNeeded(AtlasType.fromJson(s, AtlasTypesDef.class), typeStore, typeRegistry);
+ }
+
+ private void createGlossary() throws AtlasBaseException {
+ glossary = new AtlasGlossary();
+ glossary.setQualifiedName("testSearchGlossary");
+ glossary.setName("Search glossary");
+ glossary.setShortDescription("Short description");
+ glossary.setLongDescription("Long description");
+ glossary.setUsage("N/A");
+ glossary.setLanguage("en-US");
+
+ AtlasGlossary created = glossaryService.createGlossary(glossary);
+ glossary.setGuid(created.getGuid());
+ }
+
+ private void assignTermTo(AtlasGlossaryTerm term, AtlasEntity entity) throws AtlasBaseException {
+ AtlasRelatedObjectId relatedObjectId = new AtlasRelatedObjectId();
+ relatedObjectId.setGuid(entity.getGuid());
+ relatedObjectId.setTypeName(entity.getTypeName());
+
+ glossaryService.assignTermToEntities(term.getGuid(), Arrays.asList(relatedObjectId));
+ }
+
+ private void createTerms() throws AtlasBaseException {
+ term1 = new AtlasGlossaryTerm();
+ // Glossary anchor
+ AtlasGlossaryHeader glossaryId = new AtlasGlossaryHeader();
+ glossaryId.setGlossaryGuid(glossary.getGuid());
+
+ term1.setName("term1");
+ term1.setShortDescription("Short description");
+ term1.setLongDescription("Long description");
+ term1.setAbbreviation("CHK");
+ term1.setExamples(Arrays.asList("Personal", "Joint"));
+ term1.setUsage("N/A");
+ term1.setAnchor(glossaryId);
+ AtlasGlossaryTerm created1 = glossaryService.createTerm(term1);
+ term1.setGuid(created1.getGuid());
+ }
+
private void createEntities() {
dbEntity = TestUtilsV2.createDBEntity();
tableEntity = TestUtilsV2.createTableEntity(dbEntity);