You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2019/11/19 21:13:18 UTC
[atlas] branch branch-2.0 updated: ATLAS-3482 Enhance basic search
to enable search on entity system attributes
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a350b9f ATLAS-3482 Enhance basic search to enable search on entity system attributes
a350b9f is described below
commit a350b9fe783abb3995ca355d4f41a0e18d88c68d
Author: Le Ma <lm...@cloudera.com>
AuthorDate: Fri Nov 8 12:32:11 2019 -0800
ATLAS-3482 Enhance basic search to enable search on entity system attributes
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 5932b1af358288b07f382b40939b48150f844fd6)
---
.../atlas/model/discovery/SearchParameters.java | 1 +
.../apache/atlas/model/typedef/AtlasStructDef.java | 10 +
.../apache/atlas/type/AtlasClassificationType.java | 31 +-
.../org/apache/atlas/type/AtlasEntityType.java | 63 +++-
.../org/apache/atlas/type/AtlasStructType.java | 49 ++-
.../org/apache/atlas/type/AtlasTypeRegistry.java | 15 +-
.../main/java/org/apache/atlas/type/Constants.java | 56 +++
.../test/java/org/apache/atlas/TestUtilsV2.java | 2 +-
.../atlas/type/TestAtlasRelationshipType.java | 46 +--
.../apache/atlas/type/TestAtlasTypeRegistry.java | 18 +-
.../discovery/ClassificationSearchProcessor.java | 2 +-
.../atlas/discovery/EntitySearchProcessor.java | 32 +-
.../atlas/discovery/FullTextSearchProcessor.java | 3 +-
.../org/apache/atlas/discovery/SearchContext.java | 58 +--
.../apache/atlas/discovery/SearchProcessor.java | 7 +-
.../org/apache/atlas/web/rest/DiscoveryREST.java | 12 +-
.../atlas/web/adapters/TestEntitiesREST.java | 407 +++++++++++++--------
17 files changed, 542 insertions(+), 270 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
index 8f0e591..15a19e9 100644
--- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
+++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParameters.java
@@ -61,6 +61,7 @@ public class SearchParameters implements Serializable {
public static final String WILDCARD_CLASSIFICATIONS = "*";
public static final String ALL_CLASSIFICATIONS = "_CLASSIFIED";
public static final String NO_CLASSIFICATIONS = "_NOT_CLASSIFIED";
+ public static final String ALL_ENTITY_TYPES = "_ALL_ENTITY_TYPES";
/**
* @return The type of query
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
index e10965b..bb7ead0 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
@@ -300,6 +300,16 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this(name, typeName, DEFAULT_SEARCHWEIGHT);
}
+ public AtlasAttributeDef(String name, String typeName, boolean isUnique, boolean isIndexable) {
+ this(name, typeName, false, Cardinality.SINGLE, COUNT_NOT_SET, COUNT_NOT_SET, isUnique, isIndexable,
+ false, null,null, null, null, DEFAULT_SEARCHWEIGHT, null);
+ }
+
+ public AtlasAttributeDef(String name, String typeName, Cardinality cardinality, boolean isUnique, boolean isIndexable) {
+ this(name, typeName, false, cardinality, COUNT_NOT_SET, COUNT_NOT_SET, isUnique, isIndexable,
+ false, null,null, null, null, DEFAULT_SEARCHWEIGHT, null);
+ }
+
public AtlasAttributeDef(String name, String typeName, int searchWeight) {
this(name, typeName, false, Cardinality.SINGLE, searchWeight, null);
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
index 4171942..c9b7521 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
@@ -17,27 +17,35 @@
*/
package org.apache.atlas.type;
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
+import static org.apache.atlas.type.Constants.CREATED_BY_KEY;
+import static org.apache.atlas.type.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
+import static org.apache.atlas.type.Constants.MODIFIED_BY_KEY;
+import static org.apache.atlas.type.Constants.STATE_PROPERTY_KEY;
+import static org.apache.atlas.type.Constants.TIMESTAMP_PROPERTY_KEY;
+import static org.apache.atlas.type.Constants.TYPE_NAME_PROPERTY_KEY;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TimeBoundary;
import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.validator.routines.DateValidator;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.util.*;
-
/**
* class that implements behaviour of a classification-type.
*/
public class AtlasClassificationType extends AtlasStructType {
- private static final Logger LOG = LoggerFactory.getLogger(AtlasClassificationType.class);
+
+ public static final AtlasClassificationType CLASSIFICATION_ROOT = initRootClassificationType();
+ private static final String CLASSIFICATION_ROOT_NAME = "__CLASSIFICATION_ROOT";
private final AtlasClassificationDef classificationDef;
private final String typeQryStr;
@@ -488,6 +496,21 @@ public class AtlasClassificationType extends AtlasStructType {
return CollectionUtils.isEmpty(this.entityTypes) || this.entityTypes.contains(entityType.getTypeName());
}
+ private static AtlasClassificationType initRootClassificationType() {
+ List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{
+ add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, AtlasBaseTypeDef.ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_LONG, false, true));
+ add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_LONG, false, true));
+ add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ }};
+
+ AtlasClassificationDef classificationDef = new AtlasClassificationDef(CLASSIFICATION_ROOT_NAME, "", "", attributeDefs);
+
+ return new AtlasClassificationType(classificationDef);
+ }
+
private void getTypeHierarchyInfo(AtlasTypeRegistry typeRegistry,
Set<String> allSuperTypeNames,
Map<String, AtlasAttribute> allAttributes) throws AtlasBaseException {
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 884447f..557ef74 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -17,6 +17,9 @@
*/
package org.apache.atlas.type;
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_LONG;
+import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.ATLAS_TYPE_STRING;
+import static org.apache.atlas.type.Constants.*;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
@@ -42,31 +45,31 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-
-
/**
* class that implements behaviour of an entity-type.
*/
public class AtlasEntityType extends AtlasStructType {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityType.class);
+ public static final AtlasEntityType ENTITY_ROOT = initRootEntityType();
private static final String NAME = "name";
private static final String DESCRIPTION = "description";
private static final String OWNER = "owner";
private static final String CREATE_TIME = "createTime";
private static final String DYN_ATTRIBUTE_PREFIX = "dynAttribute:";
- private static final char DYN_ATTRIBUTE_NAME_SEPARATOR = '.';
- private static final char DYN_ATTRIBUTE_OPEN_DELIM = '{';
- private static final char DYN_ATTRIBUTE_CLOSE_DELIM = '}';
+ private static final String OPTION_SCHEMA_ATTRIBUTES = "schemaAttributes";
+ private static final String INTERNAL_TYPENAME = "__internal";
+
+ private static final char DYN_ATTRIBUTE_NAME_SEPARATOR = '.';
+ private static final char DYN_ATTRIBUTE_OPEN_DELIM = '{';
+ private static final char DYN_ATTRIBUTE_CLOSE_DELIM = '}';
private static final String[] ENTITY_HEADER_ATTRIBUTES = new String[]{NAME, DESCRIPTION, OWNER, CREATE_TIME};
- private static final String OPTION_SCHEMA_ATTRIBUTES = "schemaAttributes";
+ private static final String ENTITY_ROOT_NAME = "__ENTITY_ROOT";
private final AtlasEntityDef entityDef;
private final String typeQryStr;
- private static final String INTERNAL_TYPENAME = "__internal";
-
private List<AtlasEntityType> superTypes = Collections.emptyList();
private Set<String> allSuperTypes = Collections.emptySet();
private Set<String> subTypes = Collections.emptySet();
@@ -83,7 +86,6 @@ public class AtlasEntityType extends AtlasStructType {
private List<AtlasAttribute> dynEvalTriggerAttributes = Collections.emptyList();
private Map<String,List<TemplateToken>> parsedTemplates = Collections.emptyMap();
-
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
@@ -104,6 +106,8 @@ public class AtlasEntityType extends AtlasStructType {
return entityDef;
}
+ public static AtlasEntityType getEntityRoot() {return ENTITY_ROOT; }
+
@Override
void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
super.resolveReferences(typeRegistry);
@@ -400,10 +404,14 @@ public class AtlasEntityType extends AtlasStructType {
}
public String getQualifiedAttributeName(String attrName) throws AtlasBaseException {
- if (allAttributes.containsKey(attrName)) {
- return allAttributes.get(attrName).getQualifiedName();
- } else if (relationshipAttributes.containsKey(attrName)) {
- return relationshipAttributes.get(attrName).values().iterator().next().getQualifiedName();
+ AtlasAttribute ret = getAttribute(attrName);
+
+ if (ret == null) {
+ ret = relationshipAttributes.get(attrName).values().iterator().next();
+ }
+
+ if (ret != null) {
+ return ret.getQualifiedName();
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, entityDef.getName());
@@ -506,11 +514,6 @@ public class AtlasEntityType extends AtlasStructType {
}
@Override
- public AtlasAttribute getAttribute(String attributeName) {
- return allAttributes.get(attributeName);
- }
-
- @Override
public boolean validateValue(Object obj, String objName, List<String> messages) {
boolean ret = true;
@@ -621,6 +624,30 @@ public class AtlasEntityType extends AtlasStructType {
}
}
+ private static AtlasEntityType initRootEntityType() {
+ List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{
+ add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_LONG, false, true));
+ add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_LONG, false, true));
+ add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+
+ add(new AtlasAttributeDef(GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
+ add(new AtlasAttributeDef(HISTORICAL_GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
+ add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ }};
+
+ AtlasEntityDef entityDef = new AtlasEntityDef(ENTITY_ROOT_NAME, "", "", attributeDefs);
+
+ return new AtlasEntityType(entityDef);
+ }
+
private void addSubType(AtlasEntityType subType) {
subTypes.add(subType.getTypeName());
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index e8bf7f9..3475ce6 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -216,7 +216,23 @@ public class AtlasStructType extends AtlasType {
}
public AtlasAttribute getAttribute(String attributeName) {
- return allAttributes.get(attributeName);
+ AtlasAttribute ret = allAttributes.get(attributeName);
+
+ if (ret == null) {
+ ret = getSystemAttribute(attributeName);
+ }
+
+ return ret;
+ }
+
+ public AtlasAttribute getSystemAttribute(String attributeName) {
+ AtlasAttribute ret = null;
+ if (this instanceof AtlasEntityType) {
+ ret = AtlasEntityType.ENTITY_ROOT.allAttributes.get(attributeName);
+ } else if (this instanceof AtlasClassificationType) {
+ ret = AtlasClassificationType.CLASSIFICATION_ROOT.allAttributes.get(attributeName);
+ }
+ return ret;
}
@Override
@@ -631,8 +647,10 @@ public class AtlasStructType extends AtlasType {
}
public String getQualifiedAttributeName(String attrName) throws AtlasBaseException {
- if ( allAttributes.containsKey(attrName)) {
- return allAttributes.get(attrName).getQualifiedName();
+ AtlasAttribute attribute = getAttribute(attrName);
+
+ if (attribute != null) {
+ return attribute.getQualifiedName();
}
throw new AtlasBaseException(AtlasErrorCode.UNKNOWN_ATTRIBUTE, attrName, structDef.getName());
@@ -916,20 +934,33 @@ public class AtlasStructType extends AtlasType {
}
public static String getQualifiedAttributeName(AtlasStructDef structDef, String attrName) {
- return attrName.contains(".") ? attrName : String.format("%s.%s", structDef.getName(), attrName);
+ if (isRootType(structDef)) {
+ return attrName;
+ } else {
+ return attrName.contains(".") ? attrName : String.format("%s.%s", structDef.getName(), attrName);
+ }
}
public static String generateVertexPropertyName(AtlasStructDef structDef, AtlasAttributeDef attrDef, String qualifiedName) {
String vertexPropertyName = qualifiedName;
-
- if(!attrDef.getName().contains(".") &&
- AtlasAttributeDef.IndexType.STRING.equals(attrDef.getIndexType()) &&
- ATLAS_TYPE_STRING.equalsIgnoreCase(attrDef.getTypeName())) {
- vertexPropertyName = String.format("%s.%s%s", structDef.getName(), VERTEX_PROPERTY_PREFIX_STRING_INDEX_TYPE, attrDef.getName());
+ String attrName = attrDef.getName();
+ if (isRootType(structDef)) {
+ return attrName;
+ } else {
+ if(!attrDef.getName().contains(".") &&
+ AtlasAttributeDef.IndexType.STRING.equals(attrDef.getIndexType()) &&
+ ATLAS_TYPE_STRING.equalsIgnoreCase(attrDef.getTypeName())) {
+ vertexPropertyName = String.format("%s.%s%s", structDef.getName(), VERTEX_PROPERTY_PREFIX_STRING_INDEX_TYPE, attrDef.getName());
+ }
}
return encodePropertyKey(vertexPropertyName);
}
+ private static boolean isRootType(AtlasStructDef structDef) {
+ return StringUtils.equals(structDef.getName(), AtlasEntityType.ENTITY_ROOT.getTypeName()) ||
+ StringUtils.equals(structDef.getName(), AtlasClassificationType.CLASSIFICATION_ROOT.getTypeName());
+ }
+
// Keys copied from org.janusgraph.graphdb.types.system.SystemTypeManager.RESERVED_CHARS
// JanusGraph checks that these chars are not part of any keys hence encoding
// also including Titan reserved characters to support migrated property keys
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 8b4fd1c..b071dc9 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -54,12 +54,13 @@ public class AtlasTypeRegistry {
private final Set<String> missingRelationshipDefs;
private final Map<String, String> commonIndexFieldNameCache;
-
public AtlasTypeRegistry() {
registryData = new RegistryData();
updateSynchronizer = new TypeRegistryUpdateSynchronizer(this);
missingRelationshipDefs = new HashSet<>();
commonIndexFieldNameCache = new HashMap<>();
+
+ resolveReferencesForRootTypes();
}
// used only by AtlasTransientTypeRegistry
@@ -68,6 +69,8 @@ public class AtlasTypeRegistry {
updateSynchronizer = other.updateSynchronizer;
missingRelationshipDefs = other.missingRelationshipDefs;
commonIndexFieldNameCache = other.commonIndexFieldNameCache;
+
+ resolveReferencesForRootTypes();
}
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
@@ -242,6 +245,16 @@ public class AtlasTypeRegistry {
commonIndexFieldNameCache.put(propertyName, indexFieldName);
}
+ private void resolveReferencesForRootTypes() {
+ try {
+ AtlasEntityType.ENTITY_ROOT.resolveReferences(this);
+ AtlasClassificationType.CLASSIFICATION_ROOT.resolveReferences(this);
+ } catch (AtlasBaseException e) {
+ LOG.error("Failed to initialize root types", e);
+ throw new RuntimeException(e);
+ }
+ }
+
/**
* retrieves the index field name for the common field passed in.
* @param propertyName the name of the common field.
diff --git a/intg/src/main/java/org/apache/atlas/type/Constants.java b/intg/src/main/java/org/apache/atlas/type/Constants.java
new file mode 100644
index 0000000..5b824fb
--- /dev/null
+++ b/intg/src/main/java/org/apache/atlas/type/Constants.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.type;
+
+import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.encodePropertyKey;
+
+/**
+ * Intg Constants.
+ */
+public final class Constants {
+
+ public static final String INTERNAL_PROPERTY_KEY_PREFIX = "__";
+
+ /**
+ * Shared System Attributes
+ */
+ public static final String TYPE_NAME_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typeName");
+ public static final String STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "state");
+ public static final String CREATED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "createdBy");
+ public static final String SUPER_TYPES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "superTypeNames");
+ public static final String MODIFIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modifiedBy");
+ public static final String TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "timestamp");
+ public static final String MODIFICATION_TIMESTAMP_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "modificationTimestamp");
+
+ /**
+ * Entity-Only System Attributes
+ */
+ public static final String GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "guid");
+ public static final String HISTORICAL_GUID_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "historicalGuids");
+ public static final String LABELS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "labels");
+ public static final String CUSTOM_ATTRIBUTES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "customAttributes");
+ public static final String TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "traitNames");
+ public static final String PROPAGATED_TRAIT_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedTraitNames");
+ public static final String CLASSIFICATION_TEXT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationsText");
+ public static final String CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "classificationNames");
+ public static final String PROPAGATED_CLASSIFICATION_NAMES_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagatedClassificationNames");
+ public static final String IS_INCOMPLETE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isIncomplete");
+
+ private Constants() {}
+}
+
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 530d5cd..ff79994 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -786,7 +786,7 @@ public final class TestUtilsV2 {
createUniqueRequiredAttrDef("name", "string"),
createOptionalAttrDef("description", "string"),
createRequiredAttrDef("type", "string"),
- createOptionalAttrDef("created", "date"),
+ createOptionalAttrDef("created", "string"),
// enum
new AtlasAttributeDef("tableType", "tableType", false,
SINGLE, 1, 1,
diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java
index 1075395..eaaae4d 100644
--- a/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java
+++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasRelationshipType.java
@@ -27,9 +27,9 @@ import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
import org.apache.atlas.model.typedef.AtlasRelationshipDef.RelationshipCategory;
import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
-import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
import org.testng.Assert;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import java.util.*;
@@ -38,7 +38,7 @@ import static org.testng.Assert.fail;
public class TestAtlasRelationshipType {
- private AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
+ private AtlasTypeRegistry typeRegistry;
private static final String EMPLOYEE_TYPE = "employee";
private static final String DEPARTMENT_TYPE = "department";
@@ -48,35 +48,35 @@ public class TestAtlasRelationshipType {
private static final String EMPLOYEE_ADDRESS_RELATION_TYPE = "employeeAddress";
private static final String EMPLOYEE_PHONE_RELATION_TYPE = "employeePhone";
- @Test
- public void createTypesAndRelationships() throws AtlasBaseException {
+ @BeforeMethod
+ public void setUp() throws AtlasBaseException {
+ typeRegistry = new AtlasTypeRegistry();
createEmployeeTypes();
-
createRelationshipTypes();
}
@Test
public void testvalidateAtlasRelationshipDef() throws AtlasBaseException {
- AtlasRelationshipEndDef ep_single = new AtlasRelationshipEndDef("typeA", "attr1", AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE);
- AtlasRelationshipEndDef ep_single_container = new AtlasRelationshipEndDef("typeB", "attr2", AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE);
- AtlasRelationshipEndDef ep_single_container_2 = new AtlasRelationshipEndDef("typeC", "attr3", AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, true);
- AtlasRelationshipEndDef ep_single_container_3 = new AtlasRelationshipEndDef("typeD", "attr4", AtlasStructDef.AtlasAttributeDef.Cardinality.SINGLE, true);
- AtlasRelationshipEndDef ep_SET = new AtlasRelationshipEndDef("typeD", "attr4", AtlasStructDef.AtlasAttributeDef.Cardinality.SET,false);
- AtlasRelationshipEndDef ep_LIST = new AtlasRelationshipEndDef("typeE", "attr5", AtlasStructDef.AtlasAttributeDef.Cardinality.LIST,true);
- AtlasRelationshipEndDef ep_SET_container = new AtlasRelationshipEndDef("typeF", "attr6", AtlasStructDef.AtlasAttributeDef.Cardinality.SET,true);
+ AtlasRelationshipEndDef ep_single = new AtlasRelationshipEndDef("typeA", "attr1", Cardinality.SINGLE);
+ AtlasRelationshipEndDef ep_single_container = new AtlasRelationshipEndDef("typeB", "attr2", Cardinality.SINGLE);
+ AtlasRelationshipEndDef ep_single_container_2 = new AtlasRelationshipEndDef("typeC", "attr3", Cardinality.SINGLE, true);
+ AtlasRelationshipEndDef ep_single_container_3 = new AtlasRelationshipEndDef("typeD", "attr4", Cardinality.SINGLE, true);
+ AtlasRelationshipEndDef ep_SET = new AtlasRelationshipEndDef("typeD", "attr4", Cardinality.SET,false);
+ AtlasRelationshipEndDef ep_LIST = new AtlasRelationshipEndDef("typeE", "attr5", Cardinality.LIST,true);
+ AtlasRelationshipEndDef ep_SET_container = new AtlasRelationshipEndDef("typeF", "attr6", Cardinality.SET,true);
AtlasRelationshipDef relationshipDef1 = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.ASSOCIATION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_single, ep_SET);
+ RelationshipCategory.ASSOCIATION, PropagateTags.ONE_TO_TWO, ep_single, ep_SET);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef1);
AtlasRelationshipDef relationshipDef2 = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.COMPOSITION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_SET_container, ep_single);
+ RelationshipCategory.COMPOSITION, PropagateTags.ONE_TO_TWO, ep_SET_container, ep_single);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef2);
AtlasRelationshipDef relationshipDef3 = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.AGGREGATION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_SET_container, ep_single);
+ RelationshipCategory.AGGREGATION, PropagateTags.ONE_TO_TWO, ep_SET_container, ep_single);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef3);
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.ASSOCIATION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_single_container_2, ep_single_container);
+ RelationshipCategory.ASSOCIATION, PropagateTags.ONE_TO_TWO, ep_single_container_2, ep_single_container);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -86,7 +86,7 @@ public class TestAtlasRelationshipType {
}
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.COMPOSITION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_single, ep_single_container);
+ RelationshipCategory.COMPOSITION, PropagateTags.ONE_TO_TWO, ep_single, ep_single_container);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -96,7 +96,7 @@ public class TestAtlasRelationshipType {
}
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.AGGREGATION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_single, ep_single_container);
+ RelationshipCategory.AGGREGATION, PropagateTags.ONE_TO_TWO, ep_single, ep_single_container);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -107,7 +107,7 @@ public class TestAtlasRelationshipType {
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.COMPOSITION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_SET_container, ep_SET);
+ RelationshipCategory.COMPOSITION, PropagateTags.ONE_TO_TWO, ep_SET_container, ep_SET);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -117,7 +117,7 @@ public class TestAtlasRelationshipType {
}
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.COMPOSITION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_single, ep_LIST);
+ RelationshipCategory.COMPOSITION, PropagateTags.ONE_TO_TWO, ep_single, ep_LIST);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -127,7 +127,7 @@ public class TestAtlasRelationshipType {
}
try {
AtlasRelationshipDef relationshipDef = new AtlasRelationshipDef("emptyRelationshipDef", "desc 1", "version1",
- AtlasRelationshipDef.RelationshipCategory.COMPOSITION, AtlasRelationshipDef.PropagateTags.ONE_TO_TWO, ep_LIST, ep_single);
+ RelationshipCategory.COMPOSITION, PropagateTags.ONE_TO_TWO, ep_LIST, ep_single);
AtlasRelationshipType.validateAtlasRelationshipDef(relationshipDef);
fail("This call is expected to fail");
} catch (AtlasBaseException abe) {
@@ -138,8 +138,8 @@ public class TestAtlasRelationshipType {
}
- @Test(dependsOnMethods = "createTypesAndRelationships")
- public void testRelationshipAttributes() throws Exception {
+ @Test
+ public void testRelationshipAttributes() {
Map<String, Map<String, AtlasAttribute>> employeeRelationAttrs = getRelationAttrsForType(EMPLOYEE_TYPE);
Assert.assertNotNull(employeeRelationAttrs);
diff --git a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java
index 476bc33..945c06b 100644
--- a/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java
+++ b/intg/src/test/java/org/apache/atlas/type/TestAtlasTypeRegistry.java
@@ -47,7 +47,7 @@ public class TestAtlasTypeRegistry {
* L2_1 L2_2 L2_3 L2_4
*/
@Test
- public void testClassificationDefValidHierarchy() {
+ public void testClassificationDefValidHierarchy() throws AtlasBaseException {
AtlasClassificationDef classifiL0 = new AtlasClassificationDef("L0");
AtlasClassificationDef classifiL1_1 = new AtlasClassificationDef("L1-1");
AtlasClassificationDef classifiL1_2 = new AtlasClassificationDef("L1-2");
@@ -135,7 +135,7 @@ public class TestAtlasTypeRegistry {
}
@Test
- public void testClassificationDefInvalidHierarchy_Self() {
+ public void testClassificationDefInvalidHierarchy_Self() throws AtlasBaseException {
AtlasClassificationDef classifiDef1 = new AtlasClassificationDef("classifiDef-1");
classifiDef1.addSuperType(classifiDef1.getName());
@@ -171,7 +171,7 @@ public class TestAtlasTypeRegistry {
* L2_1 L2_2 L2_3 L2_4
*/
@Test
- public void testClassificationDefInvalidHierarchy_CircularRef() {
+ public void testClassificationDefInvalidHierarchy_CircularRef() throws AtlasBaseException {
AtlasClassificationDef classifiL0 = new AtlasClassificationDef("L0");
AtlasClassificationDef classifiL1_1 = new AtlasClassificationDef("L1-1");
AtlasClassificationDef classifiL1_2 = new AtlasClassificationDef("L1-2");
@@ -228,7 +228,7 @@ public class TestAtlasTypeRegistry {
* L2_1 L2_2 L2_3 L2_4
*/
@Test
- public void testEntityDefValidHierarchy() {
+ public void testEntityDefValidHierarchy() throws AtlasBaseException {
AtlasEntityDef entL0 = new AtlasEntityDef("L0");
AtlasEntityDef entL1_1 = new AtlasEntityDef("L1-1");
AtlasEntityDef entL1_2 = new AtlasEntityDef("L1-2");
@@ -315,7 +315,7 @@ public class TestAtlasTypeRegistry {
}
@Test
- public void testEntityDefInvalidHierarchy_Self() {
+ public void testEntityDefInvalidHierarchy_Self() throws AtlasBaseException {
AtlasEntityDef entDef1 = new AtlasEntityDef("entDef-1");
entDef1.addSuperType(entDef1.getName());
@@ -351,7 +351,7 @@ public class TestAtlasTypeRegistry {
* L2_1 L2_2 L2_3 L2_4
*/
@Test
- public void testEntityDefInvalidHierarchy_CircularRef() {
+ public void testEntityDefInvalidHierarchy_CircularRef() throws AtlasBaseException {
AtlasEntityDef entL0 = new AtlasEntityDef("L0");
AtlasEntityDef entL1_1 = new AtlasEntityDef("L1-1");
AtlasEntityDef entL1_2 = new AtlasEntityDef("L1-2");
@@ -399,7 +399,7 @@ public class TestAtlasTypeRegistry {
}
@Test
- public void testNestedUpdates() {
+ public void testNestedUpdates() throws AtlasBaseException {
AtlasTypeRegistry typeRegistry = new AtlasTypeRegistry();
AtlasTransientTypeRegistry ttr = null;
boolean commit = false;
@@ -436,7 +436,7 @@ public class TestAtlasTypeRegistry {
}
@Test
- public void testParallelUpdates() {
+ public void testParallelUpdates() throws AtlasBaseException {
final int numOfThreads = 3;
final int numOfTypesPerKind = 30;
final String enumTypePrefix = "testEnum-";
@@ -503,7 +503,7 @@ public class TestAtlasTypeRegistry {
* verify that after the update failure, the registry still has correct super-type/sub-type information for L0 and L1
*/
@Test
- public void testRegistryValidityOnInvalidUpdate() {
+ public void testRegistryValidityOnInvalidUpdate() throws AtlasBaseException {
AtlasEntityDef entL0 = new AtlasEntityDef("L0");
AtlasEntityDef entL1 = new AtlasEntityDef("L1");
diff --git a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
index 672f381..be59862 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/ClassificationSearchProcessor.java
@@ -101,7 +101,7 @@ public class ClassificationSearchProcessor extends SearchProcessor {
each of above cases with either has empty/or not tagFilters
*/
final boolean useIndexSearchForEntity = (classificationType != null || isWildcardSearch) &&
- filterCriteria == null &&
+ !context.hasAttributeFilter(filterCriteria) &&
(typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TAGS);
/* If classification's attributes can be applied index filter, we can use direct index
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
index 03eb92b..ad0c9cb 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntitySearchProcessor.java
@@ -49,10 +49,12 @@ import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.ComparisionOperator.NOT_EQUAL;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.ASC;
import static org.apache.atlas.repository.graphdb.AtlasGraphQuery.SortOrder.DESC;
+import static org.apache.atlas.type.AtlasEntityType.ENTITY_ROOT;
public class EntitySearchProcessor extends SearchProcessor {
private static final Logger LOG = LoggerFactory.getLogger(EntitySearchProcessor.class);
@@ -86,11 +88,10 @@ public class EntitySearchProcessor extends SearchProcessor {
filterClassification = false;
}
- final Predicate typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator()
- .generatePredicate(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ final Predicate typeNamePredicate;
+ final Predicate traitPredicate;
final Predicate activePredicate = SearchPredicateUtil.getEQPredicateGenerator()
.generatePredicate(Constants.STATE_PROPERTY_KEY, "ACTIVE", String.class);
- final Predicate traitPredicate;
if (classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION || classificationType == MATCH_ALL_CLASSIFIED) {
traitPredicate = PredicateUtils.orPredicate(SearchPredicateUtil.getNotEmptyPredicateGenerator().generatePredicate(TRAIT_NAMES_PROPERTY_KEY, null, List.class),
@@ -103,6 +104,12 @@ public class EntitySearchProcessor extends SearchProcessor {
SearchPredicateUtil.getContainsAnyPredicateGenerator().generatePredicate(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationTypeAndSubTypes, List.class));
}
+ if (!isEntityRootType()) {
+ typeNamePredicate = SearchPredicateUtil.getINPredicateGenerator().generatePredicate(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes, String.class);
+ } else {
+ typeNamePredicate = null;
+ }
+
processSearchAttributes(entityType, filterCriteria, indexAttributes, graphAttributes, allAttributes);
final boolean typeSearchByIndex = !filterClassification && typeAndSubTypesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES;
@@ -114,7 +121,9 @@ public class EntitySearchProcessor extends SearchProcessor {
graphIndexQueryBuilder.addTypeAndSubTypesQueryFilter(indexQuery, typeAndSubTypesQryStr);
// TypeName check to be done in-memory as well to address ATLAS-2121 (case sensitivity)
- inMemoryPredicate = typeNamePredicate;
+ if (typeNamePredicate != null) {
+ inMemoryPredicate = typeNamePredicate;
+ }
}
if (attrSearchByIndex) {
@@ -148,7 +157,7 @@ public class EntitySearchProcessor extends SearchProcessor {
AtlasGraphQuery query = context.getGraph().query();
if (!typeSearchByIndex) {
- query.in(Constants.TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
+ query.in(TYPE_NAME_PROPERTY_KEY, typeAndSubTypes);
}
// If we need to filter on the trait names then we need to build the query and equivalent in-memory predicate
@@ -211,21 +220,26 @@ public class EntitySearchProcessor extends SearchProcessor {
}
// Prepare the graph query and in-memory filter for the filtering phase
- filterGraphQueryPredicate = typeNamePredicate;
+ if (typeNamePredicate != null) {
+ filterGraphQueryPredicate = typeNamePredicate;
+ }
Predicate attributesPredicate = constructInMemoryPredicate(entityType, filterCriteria, allAttributes);
if (attributesPredicate != null) {
- filterGraphQueryPredicate = PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? attributesPredicate :
+ PredicateUtils.andPredicate(filterGraphQueryPredicate, attributesPredicate);
}
if (filterClassification) {
- filterGraphQueryPredicate = PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? traitPredicate :
+ PredicateUtils.andPredicate(filterGraphQueryPredicate, traitPredicate);
}
// Filter condition for the STATUS
if (context.getSearchParameters().getExcludeDeletedEntities()) {
- filterGraphQueryPredicate = PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate);
+ filterGraphQueryPredicate = filterGraphQueryPredicate == null ? activePredicate :
+ PredicateUtils.andPredicate(filterGraphQueryPredicate, activePredicate);
}
}
diff --git a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
index 152ade8..99cb1d0 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/FullTextSearchProcessor.java
@@ -32,6 +32,7 @@ import java.util.Iterator;
import java.util.List;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_WILDCARD_CLASSIFICATION;
@@ -52,7 +53,7 @@ public class FullTextSearchProcessor extends SearchProcessor {
// if search includes entity-type criteria, adding a filter here can help avoid unnecessary
// processing (and rejection) by subsequent EntitySearchProcessor
- if (context.getEntityType() != null) {
+ if (context.getEntityType() != null && context.getEntityType() != MATCH_ALL_ENTITY_TYPES) {
String typeAndSubTypeNamesQryStr = context.getEntityType().getTypeAndAllSubTypesQryStr();
if (typeAndSubTypeNamesQryStr.length() <= MAX_QUERY_STR_LENGTH_TYPES) {
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
index 7ad32bd..d3e0d74 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchContext.java
@@ -45,7 +45,9 @@ import org.slf4j.LoggerFactory;
import java.util.*;
+import static org.apache.atlas.discovery.SearchProcessor.ALL_ENTITY_TYPE_QUERY;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIFICATIONS;
@@ -56,24 +58,26 @@ import static org.apache.atlas.model.discovery.SearchParameters.WILDCARD_CLASSIF
*/
public class SearchContext {
private static final Logger LOG = LoggerFactory.getLogger(SearchContext.class);
- private final SearchParameters searchParameters;
+
private final AtlasTypeRegistry typeRegistry;
private final AtlasGraph graph;
+ private final AtlasEntityType entityType;
private final Set<String> indexedKeys;
private final Set<String> entityAttributes;
- private final AtlasEntityType entityType;
+ private final SearchParameters searchParameters;
private final AtlasClassificationType classificationType;
private final String classificationName;
- private SearchProcessor searchProcessor;
- private boolean terminateSearch = false;
private final Set<String> typeAndSubTypes;
private final Set<String> classificationTypeAndSubTypes;
private final String typeAndSubTypesQryStr;
private final String classificationTypeAndSubTypesQryStr;
+ private boolean terminateSearch = false;
+ private SearchProcessor searchProcessor;
public final static AtlasClassificationType MATCH_ALL_WILDCARD_CLASSIFICATION = new AtlasClassificationType(new AtlasClassificationDef(WILDCARD_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(ALL_CLASSIFICATIONS));
public final static AtlasClassificationType MATCH_ALL_NOT_CLASSIFIED = new AtlasClassificationType(new AtlasClassificationDef(NO_CLASSIFICATIONS));
+ public final static AtlasEntityType MATCH_ALL_ENTITY_TYPES = AtlasEntityType.getEntityRoot();
public SearchContext(SearchParameters searchParameters, AtlasTypeRegistry typeRegistry, AtlasGraph graph, Set<String> indexedKeys) throws AtlasBaseException {
this.classificationName = searchParameters.getClassification();
@@ -82,7 +86,7 @@ public class SearchContext {
this.graph = graph;
this.indexedKeys = indexedKeys;
this.entityAttributes = new HashSet<>();
- this.entityType = typeRegistry.getEntityTypeByName(searchParameters.getTypeName());
+ this.entityType = getEntityType(searchParameters.getTypeName());
this.classificationType = getClassificationType(classificationName);
// Validate if the type name exists
@@ -111,33 +115,25 @@ public class SearchContext {
// Invalid attributes will raise an exception with 400 error code
validateAttributes(classificationType, searchParameters.getTagFilters());
- if (entityType != null) {
- if (searchParameters.getIncludeSubTypes()) {
- typeAndSubTypes = entityType.getTypeAndAllSubTypes();
- typeAndSubTypesQryStr = entityType.getTypeAndAllSubTypesQryStr();
- } else {
- typeAndSubTypes = Collections.singleton(entityType.getTypeName());
- typeAndSubTypesQryStr = entityType.getTypeQryStr();
- }
+ if (classificationType != null && !isBuiltInClassificationType()) {
+ classificationTypeAndSubTypes = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypes() : Collections.singleton(classificationType.getTypeName());
+ classificationTypeAndSubTypesQryStr = searchParameters.getIncludeSubClassifications() ? classificationType.getTypeAndAllSubTypesQryStr() : classificationType.getTypeQryStr();
} else {
- typeAndSubTypes = Collections.emptySet();
- typeAndSubTypesQryStr = "";
+ classificationTypeAndSubTypes = Collections.emptySet();
+ classificationTypeAndSubTypesQryStr = "";
}
- if (classificationType != null) {
- if (classificationType == MATCH_ALL_CLASSIFIED || classificationType == MATCH_ALL_NOT_CLASSIFIED || classificationType == MATCH_ALL_WILDCARD_CLASSIFICATION) {
- classificationTypeAndSubTypes = Collections.emptySet();
- classificationTypeAndSubTypesQryStr = "";
- } else if (searchParameters.getIncludeSubClassifications()) {
- classificationTypeAndSubTypes = classificationType.getTypeAndAllSubTypes();
- classificationTypeAndSubTypesQryStr = classificationType.getTypeAndAllSubTypesQryStr();
+ if (entityType != null) {
+ if (entityType.equals(MATCH_ALL_ENTITY_TYPES)) {
+ typeAndSubTypes = Collections.singleton(ALL_ENTITY_TYPE_QUERY);
+ typeAndSubTypesQryStr = ALL_ENTITY_TYPE_QUERY;
} else {
- classificationTypeAndSubTypes = Collections.singleton(classificationType.getTypeName());
- classificationTypeAndSubTypesQryStr = classificationType.getTypeQryStr();
+ typeAndSubTypes = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypes() : Collections.singleton(entityType.getTypeName());
+ typeAndSubTypesQryStr = searchParameters.getIncludeSubTypes() ? entityType.getTypeAndAllSubTypesQryStr() : entityType.getTypeQryStr();
}
} else {
- classificationTypeAndSubTypes = Collections.emptySet();
- classificationTypeAndSubTypesQryStr = "";
+ typeAndSubTypes = Collections.emptySet();
+ typeAndSubTypesQryStr = "";
}
if (glossaryTermVertex != null) {
@@ -160,7 +156,6 @@ public class SearchContext {
addProcessor(new ClassificationSearchProcessor(this));
}
-
if (needEntityProcessor()) {
addProcessor(new EntitySearchProcessor(this));
}
@@ -190,6 +185,8 @@ public class SearchContext {
public SearchProcessor getSearchProcessor() { return searchProcessor; }
+ public String getClassificationName() {return classificationName;}
+
public boolean includeEntityType(String entityType) {
return typeAndSubTypes.isEmpty() || typeAndSubTypes.contains(entityType);
}
@@ -285,7 +282,7 @@ public class SearchContext {
}
}
- private boolean hasAttributeFilter(FilterCriteria filterCriteria) {
+ public boolean hasAttributeFilter(FilterCriteria filterCriteria) {
return filterCriteria != null &&
(CollectionUtils.isNotEmpty(filterCriteria.getCriterion()) || StringUtils.isNotEmpty(filterCriteria.getAttributeName()));
}
@@ -314,6 +311,11 @@ public class SearchContext {
return ret;
}
+ private AtlasEntityType getEntityType(String entityName) {
+ return StringUtils.equals(entityName, ALL_ENTITY_TYPES) ? MATCH_ALL_ENTITY_TYPES :
+ typeRegistry.getEntityTypeByName(entityName);
+ }
+
private AtlasVertex getGlossaryTermVertex(String termName) {
AtlasVertex ret = null;
diff --git a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
index b56d8e8..162c343 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/SearchProcessor.java
@@ -62,6 +62,7 @@ public abstract class SearchProcessor {
public static final String SPACE_STRING = " ";
public static final String BRACE_OPEN_STR = "(";
public static final String BRACE_CLOSE_STR = ")";
+ public static final String ALL_ENTITY_TYPE_QUERY = "[* TO *]";
private static final Map<SearchParameters.Operator, String> OPERATOR_MAP = new HashMap<>();
private static final Map<SearchParameters.Operator, VertexAttributePredicateGenerator> OPERATOR_PREDICATE_MAP = new HashMap<>();
@@ -131,6 +132,10 @@ public abstract class SearchProcessor {
public abstract List<AtlasVertex> execute();
public abstract long getResultCount();
+ protected boolean isEntityRootType() {
+ return context.getEntityType() == SearchContext.MATCH_ALL_ENTITY_TYPES;
+ }
+
protected int collectResultVertices(final List<AtlasVertex> ret, final int startIdx, final int limit, int resultIdx, final List<AtlasVertex> entityVertices) {
for (AtlasVertex entityVertex : entityVertices) {
resultIdx++;
@@ -204,7 +209,7 @@ public abstract class SearchProcessor {
// (AND (OR idx-att1=x idx-attr1=y) non-idx-attr=z (AND idx-attr2=xyz idx-attr2=abc))
//
protected boolean canApplyIndexFilter(AtlasStructType structType, FilterCriteria filterCriteria, boolean insideOrCondition) {
- if (filterCriteria == null) {
+ if (!context.hasAttributeFilter(filterCriteria)) {
return true;
}
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 825cda3..076284e 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -325,15 +325,9 @@ public class DiscoveryREST {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Limit/offset should be non-negative");
}
- if (StringUtils.isEmpty(parameters.getTypeName()) && !isEmpty(parameters.getEntityFilters())) {
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "EntityFilters specified without Type name");
- }
-
- if (StringUtils.isEmpty(parameters.getClassification()) && !isEmpty(parameters.getTagFilters())) {
- throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "TagFilters specified without tag name");
- }
-
- if (StringUtils.isEmpty(parameters.getTypeName()) && StringUtils.isEmpty(parameters.getClassification()) && StringUtils.isEmpty(parameters.getQuery()) && StringUtils.isEmpty(parameters.getTermName())) {
+ if (StringUtils.isEmpty(parameters.getTypeName()) && StringUtils.isEmpty(parameters.getClassification()) &&
+ StringUtils.isEmpty(parameters.getQuery()) && StringUtils.isEmpty(parameters.getTermName()) &&
+ isEmpty(parameters.getEntityFilters()) && isEmpty(parameters.getTagFilters())) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_SEARCH_PARAMS);
}
diff --git a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
index cd8f898..5e1581a 100644
--- a/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
+++ b/webapp/src/test/java/org/apache/atlas/web/adapters/TestEntitiesREST.java
@@ -17,11 +17,18 @@
*/
package org.apache.atlas.web.adapters;
+import static org.apache.atlas.TestUtilsV2.CLASSIFICATION;
import static org.apache.atlas.TestUtilsV2.COLUMN_TYPE;
import static org.apache.atlas.TestUtilsV2.DATABASE_TYPE;
import static org.apache.atlas.TestUtilsV2.FETL_CLASSIFICATION;
import static org.apache.atlas.TestUtilsV2.PHI;
+import static org.apache.atlas.TestUtilsV2.PII;
import static org.apache.atlas.TestUtilsV2.TABLE_TYPE;
+import static org.apache.atlas.model.discovery.SearchParameters.FilterCriteria.Condition.AND;
+import static org.apache.atlas.repository.Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TIMESTAMP_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPE_NAME_PROPERTY_KEY;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.RequestContext;
@@ -29,6 +36,7 @@ import org.apache.atlas.TestModules;
import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.model.discovery.AtlasSearchResult;
import org.apache.atlas.model.discovery.SearchParameters;
+import org.apache.atlas.model.discovery.SearchParameters.FilterCriteria;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
@@ -57,6 +65,7 @@ import org.testng.annotations.Test;
import javax.inject.Inject;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -77,13 +86,14 @@ public class TestEntitiesREST {
@Inject
private EntityREST entityREST;
- private AtlasEntity dbEntity;
- private AtlasEntity tableEntity;
- private AtlasEntity tableEntity2;
- private List<AtlasEntity> columns;
- private List<AtlasEntity> columns2;
- private SearchParameters searchParameters = new SearchParameters();
- private Map<String, List<String>> createdGuids = new HashMap<>();
+ private AtlasEntity dbEntity;
+ private AtlasEntity tableEntity;
+ private AtlasEntity tableEntity2;
+ private List<AtlasEntity> columns;
+ private List<AtlasEntity> columns2;
+ private SearchParameters searchParameters;
+ private Map<String, List<String>> createdGuids = new HashMap<>();
+ private Map<String, AtlasClassification> tagMap = new HashMap<>();
@BeforeClass
public void setUp() throws Exception {
@@ -97,19 +107,13 @@ public class TestEntitiesREST {
}
}
- dbEntity = TestUtilsV2.createDBEntity();
- tableEntity = TestUtilsV2.createTableEntity(dbEntity);
- tableEntity2 = TestUtilsV2.createTableEntity(dbEntity);
+ createEntities();
- final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
- final AtlasEntity colEntity2 = TestUtilsV2.createColumnEntity(tableEntity2);
- columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
- columns2 = new ArrayList<AtlasEntity>() {{ add(colEntity2); }};
+ initTagMap();
- tableEntity.setAttribute("columns", getObjIdList(columns));
- tableEntity2.setAttribute("columns", getObjIdList(columns2));
+ registerEntities();
- createEntities();
+ addTagTo(CLASSIFICATION, TABLE_TYPE);
}
@AfterMethod
@@ -117,52 +121,8 @@ public class TestEntitiesREST {
RequestContext.clear();
}
- private void createEntities() throws Exception {
- AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo();
-
- entities.addEntity(dbEntity);
- entities.addEntity(tableEntity);
- entities.addEntity(tableEntity2);
-
- for (AtlasEntity column : columns) {
- entities.addReferredEntity(column);
- }
-
- for (AtlasEntity column : columns2) {
- entities.addReferredEntity(column);
- }
-
- EntityMutationResponse response = entityREST.createOrUpdate(entities);
- List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
-
- Assert.assertNotNull(guids);
- Assert.assertEquals(guids.size(), 5);
-
- for (AtlasEntityHeader header : guids) {
- if (!createdGuids.containsKey(header.getTypeName())) {
- createdGuids.put(header.getTypeName(), new ArrayList<>());
- }
- createdGuids.get(header.getTypeName()).add(header.getGuid());
- }
- }
-
@Test
- public void testTagToMultipleEntities() throws Exception{
- AtlasClassification tag = new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }});
-
- // tag with table entities, leave rest for comparison
- ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids.get(TABLE_TYPE), tag);
- entityREST.addClassification(classificationAssociateRequest);
-
- for (int i = 0; i < createdGuids.get(TABLE_TYPE).size() - 1; i++) {
- final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(TABLE_TYPE).get(i), TestUtilsV2.CLASSIFICATION);
- Assert.assertNotNull(result_tag);
- Assert.assertEquals(result_tag.getTypeName(), tag.getTypeName());
- }
- }
-
- @Test(dependsOnMethods = "testTagToMultipleEntities")
- public void testBasicSearchWithSub() throws Exception {
+ public void testBasicSearch() throws Exception {
// search entities with classification named classification
searchParameters = new SearchParameters();
searchParameters.setIncludeSubClassifications(true);
@@ -173,8 +133,10 @@ public class TestEntitiesREST {
Assert.assertEquals(res.getEntities().size(), 2);
}
- @Test(dependsOnMethods = "testTagToMultipleEntities")
+ @Test(dependsOnMethods = "testBasicSearch")
public void testWildCardBasicSearch() throws Exception {
+
+ //table - classification
searchParameters = new SearchParameters();
searchParameters.setClassification("*");
@@ -211,30 +173,87 @@ public class TestEntitiesREST {
}
@Test(dependsOnMethods = "testWildCardBasicSearch")
- public void testBasicSearchAddCls() throws Exception {
- AtlasClassification cls = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
- put("stringAttr", "sample_string");
- put("booleanAttr", true);
- put("integerAttr", 100);
- }});
+ public void testBasicSearchWithAttr() throws Exception{
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification("cla*");
+ searchParameters.setTypeName(TABLE_TYPE);
- ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), cls);
- entityREST.addClassification(clsAssRequest);
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
- final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.PHI);
- Assert.assertNotNull(result_tag);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ // table - classification
+ // column - phi
+ addTagTo(PHI, COLUMN_TYPE);
+
+ FilterCriteria filterCriteria = new FilterCriteria();
+ filterCriteria.setAttributeName("stringAttr");
+ filterCriteria.setOperator(SearchParameters.Operator.CONTAINS);
+ filterCriteria.setAttributeValue("sample");
+
+ // basic search with tag filterCriteria
+ searchParameters = new SearchParameters();
+ searchParameters.setClassification(PHI);
+ searchParameters.setTagFilters(filterCriteria);
+
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ filterCriteria.setAttributeName("stringAttr");
+ filterCriteria.setOperator(SearchParameters.Operator.EQ);
+ filterCriteria.setAttributeValue("sample_string");
+
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ filterCriteria.setAttributeValue("SAMPLE_STRING");
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
+ }
- // search entities associated with phi
+ @Test(dependsOnMethods = "testWildCardBasicSearch")
+ public void testBasicSearchWithSubTypes() throws Exception{
+
+ // basic search with subtypes
searchParameters = new SearchParameters();
- searchParameters.setClassification(TestUtilsV2.PHI);
+ searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
+ searchParameters.setIncludeSubClassifications(true);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+
Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 1);
+ Assert.assertEquals(res.getEntities().size(), 2);
+
+ // table - classification
+ // database - fetl_classification
+ addTagTo(FETL_CLASSIFICATION, DATABASE_TYPE);
+
+ final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.FETL_CLASSIFICATION);
+ Assert.assertNotNull(result_tag);
+ Assert.assertEquals(result_tag.getTypeName(), FETL_CLASSIFICATION);
+
+ // basic search with subtypes
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 3);
+
+ // basic search without subtypes
+ searchParameters.setIncludeSubClassifications(false);
+ res = discoveryREST.searchWithParameters(searchParameters);
+
+ Assert.assertNotNull(res.getEntities());
+ Assert.assertEquals(res.getEntities().size(), 2);
}
- @Test(dependsOnMethods = "testBasicSearchAddCls")
+ @Test(dependsOnMethods = "testWildCardBasicSearch")
public void testGraphQueryFilter() throws Exception {
+
+ // database - pii, felt_classification
+ // table - pii, classification,
+ // col - phi
searchParameters = new SearchParameters();
searchParameters.setQuery("sample_string");
searchParameters.setClassification(PHI);
@@ -247,8 +266,8 @@ public class TestEntitiesREST {
searchParameters.setTagFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 1);
- Assert.assertEquals(res.getEntities().get(0).getTypeName(), DATABASE_TYPE);
+ Assert.assertEquals(res.getEntities().size(), 2);
+ Assert.assertEquals(res.getEntities().get(0).getTypeName(), COLUMN_TYPE);
AtlasClassification cls = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
put("stringAttr", "sample_string");
@@ -261,123 +280,130 @@ public class TestEntitiesREST {
final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(TABLE_TYPE).get(0), TestUtilsV2.PHI);
Assert.assertNotNull(result_tag);
+ Assert.assertEquals(result_tag.getTypeName(), PHI);
+ fc.setAttributeValue("false");
res = discoveryREST.searchWithParameters(searchParameters);
+
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 1);
- Assert.assertEquals(res.getEntities().get(0).getTypeName(), DATABASE_TYPE);
-
+ Assert.assertEquals(res.getEntities().get(0).getTypeName(), TABLE_TYPE);
}
- private void addPHICls() throws Exception {
- AtlasClassification clsPHI = new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
- put("stringAttr", "string");
- put("booleanAttr", true);
- put("integerAttr", 100);
- }});
+ @Test(dependsOnMethods = "testBasicSearch")
+ public void testBasicSearchWithFilter() throws Exception {
- // add clsPHI to col entities
- ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(COLUMN_TYPE), clsPHI);
- entityREST.addClassification(clsAssRequest);
+ //table - classification
+ searchParameters = new SearchParameters();
+ searchParameters.setIncludeSubClassifications(false);
+ searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
- final AtlasClassification result_PHI = entityREST.getClassification(createdGuids.get(COLUMN_TYPE).get(0), TestUtilsV2.PHI);
- Assert.assertNotNull(result_PHI);
- }
+ SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
+ fc.setOperator(SearchParameters.Operator.CONTAINS);
+ fc.setAttributeValue("new comments");
+ fc.setAttributeName("tag");
- @Test(dependsOnMethods = "testBasicSearchAddCls")
- public void testBasicSearch() throws Exception{
- searchParameters = new SearchParameters();
- searchParameters.setClassification("PH*");
- searchParameters.setTypeName(DATABASE_TYPE);
+ searchParameters.setTagFilters(fc);
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
- Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 1);
-
- addPHICls();
+ fc.setOperator(SearchParameters.Operator.ENDS_WITH);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
- // basic search with tag filterCriteria
- searchParameters = new SearchParameters();
- searchParameters.setClassification("PHI");
+ fc.setOperator(SearchParameters.Operator.STARTS_WITH);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
+ }
- SearchParameters.FilterCriteria filterCriteria = new SearchParameters.FilterCriteria();
- filterCriteria.setAttributeName("stringAttr");
- filterCriteria.setOperator(SearchParameters.Operator.CONTAINS);
- filterCriteria.setAttributeValue("str");
- searchParameters.setTagFilters(filterCriteria);
+ @Test
+ public void testSearchByMultiSystemAttributes() throws Exception {
- res = discoveryREST.searchWithParameters(searchParameters);
+ searchParameters = new SearchParameters();
+ searchParameters.setTypeName("_ALL_ENTITY_TYPES");
+ SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
+ SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
+ SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
- Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 3);
+ subFc1.setAttributeName(MODIFICATION_TIMESTAMP_PROPERTY_KEY);
+ subFc1.setOperator(SearchParameters.Operator.LT);
+ subFc1.setAttributeValue(String.valueOf(System.currentTimeMillis()));
- filterCriteria.setAttributeName("stringAttr");
- filterCriteria.setOperator(SearchParameters.Operator.EQ);
- filterCriteria.setAttributeValue("string");
+ subFc2.setAttributeName(TIMESTAMP_PROPERTY_KEY);
+ subFc2.setOperator(SearchParameters.Operator.LT);
+ subFc2.setAttributeValue(String.valueOf(System.currentTimeMillis()));
- res = discoveryREST.searchWithParameters(searchParameters);
+ fc.setCriterion(Arrays.asList(subFc1, subFc2));
+ fc.setCondition(AND);
+ searchParameters.setEntityFilters(fc);
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 2);
-
- filterCriteria.setAttributeValue("STRING");
- res = discoveryREST.searchWithParameters(searchParameters);
- Assert.assertNull(res.getEntities());
+ Assert.assertTrue(res.getEntities().size() > 5);
}
- @Test(dependsOnMethods = "testWildCardBasicSearch")
- public void testBasicSearchWithSubTypes() throws Exception{
- AtlasClassification fetlCls = new AtlasClassification(TestUtilsV2.FETL_CLASSIFICATION, new HashMap<String, Object>() {{
- put("tag", "sample_tag");
- }});
+ @Test
+ public void testSearchBySingleSystemAttribute() throws Exception {
+ searchParameters = new SearchParameters();
+ searchParameters.setTypeName("_ALL_ENTITY_TYPES");
- ClassificationAssociateRequest clsAssRequest = new ClassificationAssociateRequest(createdGuids.get(DATABASE_TYPE), fetlCls);
- entityREST.addClassification(clsAssRequest);
+ SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
+ fc.setAttributeName(STATE_PROPERTY_KEY);
+ fc.setOperator(SearchParameters.Operator.EQ);
+ fc.setAttributeValue("DELETED");
- final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(DATABASE_TYPE).get(0), TestUtilsV2.FETL_CLASSIFICATION);
- Assert.assertNotNull(result_tag);
- Assert.assertEquals(result_tag.getTypeName(), FETL_CLASSIFICATION);
+ searchParameters.setEntityFilters(fc);
- // basic search with subtypes
- searchParameters = new SearchParameters();
- searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
- searchParameters.setIncludeSubClassifications(true);
+ entityREST.deleteByGuid(createdGuids.get(DATABASE_TYPE).get(0));
AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
-
Assert.assertNotNull(res.getEntities());
- Assert.assertEquals(res.getEntities().size(), 3);
+ Assert.assertEquals(res.getEntities().size(), 1);
+ searchParameters.setTypeName("_ALL_ENTITY_TYPES");
+ fc.setAttributeName(TYPE_NAME_PROPERTY_KEY);
+ fc.setOperator(SearchParameters.Operator.EQ);
+ fc.setAttributeValue(TABLE_TYPE);
- // basic search without subtypes
- searchParameters.setIncludeSubClassifications(false);
res = discoveryREST.searchWithParameters(searchParameters);
-
Assert.assertNotNull(res.getEntities());
Assert.assertEquals(res.getEntities().size(), 2);
+
+ searchParameters = new SearchParameters();
+ searchParameters.setTypeName(TABLE_TYPE);
+ fc.setAttributeName(STATE_PROPERTY_KEY);
+ fc.setOperator(SearchParameters.Operator.EQ);
+ fc.setAttributeValue("DELETED");
+
+ searchParameters.setEntityFilters(fc);
+ res = discoveryREST.searchWithParameters(searchParameters);
+ Assert.assertNull(res.getEntities());
}
- @Test(dependsOnMethods = "testTagToMultipleEntities")
- public void testBasicSearchWithFilter() throws Exception {
+ @Test
+ public void testSearchBySystemAttributesWithQuery() throws Exception {
searchParameters = new SearchParameters();
- searchParameters.setIncludeSubClassifications(true);
- searchParameters.setClassification(TestUtilsV2.CLASSIFICATION);
+ searchParameters.setTypeName("_ALL_ENTITY_TYPES");
SearchParameters.FilterCriteria fc = new SearchParameters.FilterCriteria();
- fc.setOperator(SearchParameters.Operator.CONTAINS);
- fc.setAttributeValue("new comments");
- fc.setAttributeName("tag");
- searchParameters.setTagFilters(fc);
+ SearchParameters.FilterCriteria subFc1 = new SearchParameters.FilterCriteria();
+ SearchParameters.FilterCriteria subFc2 = new SearchParameters.FilterCriteria();
- AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
- Assert.assertNull(res.getEntities());
+ subFc1.setAttributeName(TIMESTAMP_PROPERTY_KEY);
+ subFc1.setOperator(SearchParameters.Operator.LT);
+ subFc1.setAttributeValue(String.valueOf(System.currentTimeMillis()));
- fc.setOperator(SearchParameters.Operator.ENDS_WITH);
- res = discoveryREST.searchWithParameters(searchParameters);
- Assert.assertNull(res.getEntities());
+ subFc2.setAttributeName(STATE_PROPERTY_KEY);
+ subFc2.setOperator(SearchParameters.Operator.EQ);
+ subFc2.setAttributeValue("DELETED");
- fc.setOperator(SearchParameters.Operator.STARTS_WITH);
- res = discoveryREST.searchWithParameters(searchParameters);
+ fc.setCriterion(Arrays.asList(subFc1, subFc2));
+ fc.setCondition(AND);
+
+ searchParameters.setEntityFilters(fc);
+ searchParameters.setQuery("sample_string");
+
+ AtlasSearchResult res = discoveryREST.searchWithParameters(searchParameters);
Assert.assertNull(res.getEntities());
}
@@ -435,6 +461,75 @@ public class TestEntitiesREST {
*
*/
+ private void createEntities() {
+ dbEntity = TestUtilsV2.createDBEntity();
+ tableEntity = TestUtilsV2.createTableEntity(dbEntity);
+ tableEntity2 = TestUtilsV2.createTableEntity(dbEntity);
+
+ final AtlasEntity colEntity = TestUtilsV2.createColumnEntity(tableEntity);
+ final AtlasEntity colEntity2 = TestUtilsV2.createColumnEntity(tableEntity2);
+ columns = new ArrayList<AtlasEntity>() {{ add(colEntity); }};
+ columns2 = new ArrayList<AtlasEntity>() {{ add(colEntity2); }};
+
+ tableEntity.setAttribute("columns", getObjIdList(columns));
+ tableEntity2.setAttribute("columns", getObjIdList(columns2));
+ }
+
+ private void initTagMap() {
+ tagMap.put(PHI, new AtlasClassification(TestUtilsV2.PHI, new HashMap<String, Object>() {{
+ put("stringAttr", "sample_string");
+ put("booleanAttr", true);
+ put("integerAttr", 100);
+ }}));
+
+ tagMap.put(PII, new AtlasClassification(TestUtilsV2.PII, new HashMap<String, Object>() {{ put("cls", "clsName"); }}));
+
+ tagMap.put(CLASSIFICATION, new AtlasClassification(TestUtilsV2.CLASSIFICATION, new HashMap<String, Object>() {{ put("tag", "tagName"); }}));
+
+ tagMap.put(FETL_CLASSIFICATION, new AtlasClassification(FETL_CLASSIFICATION, new HashMap<String, Object>() {{ put("cls", "clsName"); put("tag", "tagName"); }}));
+ }
+
+ private void addTagTo(String tagName, String type) throws Exception {
+ AtlasClassification tag = tagMap.get(tagName);
+ ClassificationAssociateRequest classificationAssociateRequest = new ClassificationAssociateRequest(createdGuids.get(type), tag);
+ entityREST.addClassification(classificationAssociateRequest);
+
+ for (int i = 0; i < createdGuids.get(type).size() - 1; i++) {
+ final AtlasClassification result_tag = entityREST.getClassification(createdGuids.get(type).get(i), tagName);
+ Assert.assertNotNull(result_tag);
+ Assert.assertEquals(result_tag.getTypeName(), tag.getTypeName());
+ }
+ }
+
+ private void registerEntities() throws Exception {
+ AtlasEntitiesWithExtInfo entities = new AtlasEntitiesWithExtInfo();
+
+ entities.addEntity(dbEntity);
+ entities.addEntity(tableEntity);
+ entities.addEntity(tableEntity2);
+
+ for (AtlasEntity column : columns) {
+ entities.addReferredEntity(column);
+ }
+
+ for (AtlasEntity column : columns2) {
+ entities.addReferredEntity(column);
+ }
+
+ EntityMutationResponse response = entityREST.createOrUpdate(entities);
+ List<AtlasEntityHeader> guids = response.getEntitiesByOperation(EntityMutations.EntityOperation.CREATE);
+
+ Assert.assertNotNull(guids);
+ Assert.assertEquals(guids.size(), 5);
+
+ for (AtlasEntityHeader header : guids) {
+ if (!createdGuids.containsKey(header.getTypeName())) {
+ createdGuids.put(header.getTypeName(), new ArrayList<>());
+ }
+ createdGuids.get(header.getTypeName()).add(header.getGuid());
+ }
+ }
+
private void verifyAttributes(List<AtlasEntity> retrievedEntities) throws Exception {
AtlasEntity retrievedDBEntity = null;
AtlasEntity retrievedTableEntity = null;