You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2020/02/17 01:00:21 UTC
[atlas] branch branch-2.0 updated: ATLAS-3613: updated search to
support namespace attributes
This is an automated email from the ASF dual-hosted git repository.
madhan pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 0996287 ATLAS-3613: updated search to support namespace attributes
0996287 is described below
commit 0996287b3c56b73a2a8a9d9019d65c82c23f7f6a
Author: Madhan Neethiraj <ma...@apache.org>
AuthorDate: Mon Feb 10 17:53:38 2020 -0800
ATLAS-3613: updated search to support namespace attributes
(cherry picked from commit ac808af07c69191e34a279be169b8f01e5d35240)
---
.../apache/atlas/type/AtlasClassificationType.java | 5 +
.../org/apache/atlas/type/AtlasEntityType.java | 154 ++++++++++++++-------
.../org/apache/atlas/type/AtlasStructType.java | 16 ++-
.../apache/atlas/query/GremlinQueryComposer.java | 4 +-
.../org/apache/atlas/query/IdentifierHelper.java | 27 ++--
.../apache/atlas/query/RegistryBasedLookup.java | 43 ++----
.../store/graph/v2/AtlasEntityStoreV2.java | 10 +-
.../store/graph/v2/EntityGraphMapper.java | 34 ++---
.../store/graph/v2/EntityGraphRetriever.java | 33 +++--
.../graph/v2/AtlasNamespaceDefStoreV2Test.java | 6 +-
10 files changed, 199 insertions(+), 133 deletions(-)
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
index 1518bbc..4d0179c 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasClassificationType.java
@@ -238,6 +238,11 @@ public class AtlasClassificationType extends AtlasStructType {
classificationDef.setSubTypes(subTypes);
}
+ @Override
+ public AtlasAttribute getSystemAttribute(String attributeName) {
+ return AtlasClassificationType.CLASSIFICATION_ROOT.allAttributes.get(attributeName);
+ }
+
private void addSubType(AtlasClassificationType subType) {
subTypes.add(subType.getTypeName());
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
index 7594153..2824feb 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasEntityType.java
@@ -55,7 +55,8 @@ import java.util.Set;
public class AtlasEntityType extends AtlasStructType {
private static final Logger LOG = LoggerFactory.getLogger(AtlasEntityType.class);
- public static final AtlasEntityType ENTITY_ROOT = initRootEntityType();
+ public static final AtlasEntityType ENTITY_ROOT = new AtlasRootEntityType();
+
private static final String NAME = "name";
private static final String DESCRIPTION = "description";
private static final String OWNER = "owner";
@@ -64,6 +65,8 @@ public class AtlasEntityType extends AtlasStructType {
private static final String OPTION_SCHEMA_ATTRIBUTES = "schemaAttributes";
private static final String INTERNAL_TYPENAME = "__internal";
+ private static final char NS_ATTRIBUTE_NAME_SEPARATOR = '.';
+
private static final char DYN_ATTRIBUTE_NAME_SEPARATOR = '.';
private static final char DYN_ATTRIBUTE_OPEN_DELIM = '{';
private static final char DYN_ATTRIBUTE_CLOSE_DELIM = '}';
@@ -90,7 +93,7 @@ public class AtlasEntityType extends AtlasStructType {
private List<AtlasAttribute> dynEvalTriggerAttributes = Collections.emptyList();
private Map<String,List<TemplateToken>> parsedTemplates = Collections.emptyMap();
private Set<String> tagPropagationEdges = Collections.emptySet();
- private Map<String, List<AtlasNamespaceAttribute>> namespaceAttributes = Collections.emptyMap();
+ private Map<String, Map<String, AtlasNamespaceAttribute>> namespaceAttributes = Collections.emptyMap();
public AtlasEntityType(AtlasEntityDef entityDef) {
super(entityDef);
@@ -237,24 +240,22 @@ public class AtlasEntityType extends AtlasStructType {
}
}
- Map<String, List<AtlasNamespaceAttribute>> superTypeNamespaces = superType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> superTypeNamespaces = superType.getNamespaceAttributes();
if (MapUtils.isNotEmpty(superTypeNamespaces)) {
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : superTypeNamespaces.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> superTypeNsAttrs = entry.getValue();
- List<AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName);
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : superTypeNamespaces.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> superTypeNsAttrs = entry.getValue();
+ Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName);
if (nsAttrs == null) {
- nsAttrs = new ArrayList<>();
+ nsAttrs = new HashMap<>();
namespaceAttributes.put(nsName, nsAttrs);
}
- for (AtlasNamespaceAttribute superTypeNsAttr : superTypeNsAttrs) {
- if (!nsAttrs.contains(superTypeNsAttr)) {
- nsAttrs.add(superTypeNsAttr);
- }
+ for (Map.Entry<String, AtlasNamespaceAttribute> nsAttrEntry : superTypeNsAttrs.entrySet()) {
+ nsAttrs.put(nsAttrEntry.getKey(), nsAttrEntry.getValue());
}
}
}
@@ -306,12 +307,12 @@ public class AtlasEntityType extends AtlasStructType {
Map<String, List<AtlasAttributeDef>> namespaceAttributeDefs = new HashMap<>();
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : namespaceAttributes.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> nsAttrs = entry.getValue();
- List<AtlasAttributeDef> nsAttrDefs = new ArrayList<>();
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : namespaceAttributes.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> nsAttrs = entry.getValue();
+ List<AtlasAttributeDef> nsAttrDefs = new ArrayList<>();
- for (AtlasNamespaceAttribute nsAttr : nsAttrs) {
+ for (AtlasNamespaceAttribute nsAttr : nsAttrs.values()) {
nsAttrDefs.add(nsAttr.getAttributeDef());
}
@@ -329,6 +330,29 @@ public class AtlasEntityType extends AtlasStructType {
}
}
+ @Override
+ public AtlasAttribute getSystemAttribute(String attributeName) {
+ return AtlasEntityType.ENTITY_ROOT.allAttributes.get(attributeName);
+ }
+
+ @Override
+ public AtlasNamespaceAttribute getNamespaceAttribute(String nsAttrQualifiedName) {
+ AtlasNamespaceAttribute ret = null;
+
+ if (nsAttrQualifiedName != null) {
+ int idxSep = nsAttrQualifiedName.indexOf(AtlasEntityType.NS_ATTRIBUTE_NAME_SEPARATOR);
+
+ if (idxSep != -1) {
+ String nsName = nsAttrQualifiedName.substring(0, idxSep);
+ String nsAttrName = nsAttrQualifiedName.substring(idxSep + 1);
+
+ ret = getNamespaceAttribute(nsName, nsAttrName);
+ }
+ }
+
+ return ret;
+ }
+
public Set<String> getSuperTypes() {
return entityDef.getSuperTypes();
}
@@ -407,14 +431,21 @@ public class AtlasEntityType extends AtlasStructType {
return CollectionUtils.isNotEmpty(tagPropagationEdges) ? tagPropagationEdges.toArray(new String[tagPropagationEdges.size()]) : null;
}
- public Map<String, List<AtlasNamespaceAttribute>> getNamespaceAttributes() {
+ public Map<String, Map<String, AtlasNamespaceAttribute>> getNamespaceAttributes() {
return namespaceAttributes;
}
- public List<AtlasNamespaceAttribute> getNamespaceAttributes(String nsName) {
+ public Map<String, AtlasNamespaceAttribute> getNamespaceAttributes(String nsName) {
return namespaceAttributes.get(nsName);
}
+ public AtlasNamespaceAttribute getNamespaceAttribute(String nsName, String nsAttrName) {
+ Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName);
+ AtlasNamespaceAttribute ret = nsAttrs != null ? nsAttrs.get(nsAttrName) : null;
+
+ return ret;
+ }
+
public Map<String,List<TemplateToken>> getParsedTemplates() { return parsedTemplates; }
public AtlasAttribute getRelationshipAttribute(String attributeName, String relationshipType) {
@@ -506,16 +537,16 @@ public class AtlasEntityType extends AtlasStructType {
}
public void addNamespaceAttribute(AtlasNamespaceAttribute attribute) {
- String nsName = attribute.getDefinedInType().getTypeName();
- List<AtlasNamespaceAttribute> attributes = namespaceAttributes.get(nsName);
+ String nsName = attribute.getDefinedInType().getTypeName();
+ Map<String, AtlasNamespaceAttribute> nsAttrs = namespaceAttributes.get(nsName);
- if (attributes == null) {
- attributes = new ArrayList<>();
+ if (nsAttrs == null) {
+ nsAttrs = new HashMap<>();
- namespaceAttributes.put(nsName, attributes);
+ namespaceAttributes.put(nsName, nsAttrs);
}
- attributes.add(attribute);
+ nsAttrs.put(attribute.getName(), attribute);
}
public String getQualifiedAttributeName(String attrName) throws AtlasBaseException {
@@ -739,30 +770,6 @@ public class AtlasEntityType extends AtlasStructType {
}
}
- private static AtlasEntityType initRootEntityType() {
- List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{
- add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true));
- add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true));
- add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
-
- add(new AtlasAttributeDef(GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
- add(new AtlasAttributeDef(HISTORICAL_GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
- add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true));
- add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
- add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
- }};
-
- AtlasEntityDef entityDef = new AtlasEntityDef(ENTITY_ROOT_NAME, "Root entity for system attributes", "1.0", attributeDefs);
-
- return new AtlasEntityType(entityDef);
- }
-
private void addSubType(AtlasEntityType subType) {
subTypes.add(subType.getTypeName());
}
@@ -1198,4 +1205,53 @@ public class AtlasEntityType extends AtlasStructType {
return adj;
}
+
+ /* this class provides abstractions that help basic-search and dsl-search to deal with
+ * system-attributes and namespace-attributes
+ */
+ private static class AtlasRootEntityType extends AtlasEntityType {
+ private AtlasTypeRegistry typeRegistry = null;
+
+ public AtlasRootEntityType() {
+ super(getRootEntityDef());
+ }
+
+ @Override
+ void resolveReferences(AtlasTypeRegistry typeRegistry) throws AtlasBaseException {
+ super.resolveReferences(typeRegistry);
+
+ // save typeRegistry for use in getNamespaceAttribute()
+ this.typeRegistry = typeRegistry;
+ }
+
+ @Override
+ public AtlasNamespaceAttribute getNamespaceAttribute(String nsName, String nsAttrName) {
+ AtlasNamespaceType nsType = typeRegistry != null ? typeRegistry.getNamespaceTypeByName(nsName) : null;
+ AtlasAttribute nsAttr = nsType != null ? nsType.getAttribute(nsAttrName) : null;
+
+ return (nsAttr instanceof AtlasNamespaceAttribute) ? (AtlasNamespaceAttribute) nsAttr : null;
+ }
+
+ private static AtlasEntityDef getRootEntityDef() {
+ List<AtlasAttributeDef> attributeDefs = new ArrayList<AtlasAttributeDef>() {{
+ add(new AtlasAttributeDef(TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true));
+ add(new AtlasAttributeDef(MODIFICATION_TIMESTAMP_PROPERTY_KEY, ATLAS_TYPE_DATE, false, true));
+ add(new AtlasAttributeDef(MODIFIED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CREATED_BY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(STATE_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+
+ add(new AtlasAttributeDef(GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
+ add(new AtlasAttributeDef(HISTORICAL_GUID_PROPERTY_KEY, ATLAS_TYPE_STRING, true, true));
+ add(new AtlasAttributeDef(TYPE_NAME_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CLASSIFICATION_TEXT_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(PROPAGATED_CLASSIFICATION_NAMES_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(IS_INCOMPLETE_PROPERTY_KEY, ATLAS_TYPE_INT, false, true));
+ add(new AtlasAttributeDef(LABELS_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ add(new AtlasAttributeDef(CUSTOM_ATTRIBUTES_PROPERTY_KEY, ATLAS_TYPE_STRING, false, true));
+ }};
+
+ return new AtlasEntityDef(ENTITY_ROOT_NAME, "Root entity for system attributes", "1.0", attributeDefs);
+ }
+ }
}
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 6bd3a11..5c62a24 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -226,17 +226,19 @@ public class AtlasStructType extends AtlasType {
ret = getSystemAttribute(attributeName);
}
+ if (ret == null) {
+ ret = getNamespaceAttribute(attributeName);
+ }
+
return ret;
}
public AtlasAttribute getSystemAttribute(String attributeName) {
- AtlasAttribute ret = null;
- if (this instanceof AtlasEntityType) {
- ret = AtlasEntityType.ENTITY_ROOT.allAttributes.get(attributeName);
- } else if (this instanceof AtlasClassificationType) {
- ret = AtlasClassificationType.CLASSIFICATION_ROOT.allAttributes.get(attributeName);
- }
- return ret;
+ return null;
+ }
+
+ public AtlasAttribute getNamespaceAttribute(String attributeName) {
+ return null;
}
@Override
diff --git a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
index 294dc00..6a02955 100644
--- a/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
+++ b/repository/src/main/java/org/apache/atlas/query/GremlinQueryComposer.java
@@ -117,9 +117,9 @@ public class GremlinQueryComposer {
if (ia.isTrait()) {
String traitName = ia.get();
- if (traitName.equalsIgnoreCase(ALL_CLASSIFICATIONS)) {
+ if (traitName.equals(ALL_CLASSIFICATIONS)) {
addTrait(GremlinClause.ANY_TRAIT, ia);
- } else if (traitName.equalsIgnoreCase(NO_CLASSIFICATIONS)) {
+ } else if (traitName.equals(NO_CLASSIFICATIONS)) {
addTrait(GremlinClause.NO_TRAIT, ia);
} else {
addTrait(GremlinClause.TRAIT, ia);
diff --git a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
index 0fdb57b..14ca49e 100644
--- a/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
+++ b/repository/src/main/java/org/apache/atlas/query/IdentifierHelper.java
@@ -20,6 +20,8 @@ package org.apache.atlas.query;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.type.AtlasNamespaceType;
+import org.apache.atlas.type.AtlasType;
import org.apache.commons.lang.StringUtils;
import java.util.regex.Matcher;
@@ -145,7 +147,6 @@ public class IdentifierHelper {
raw = context.getTypeNameFromAlias(this.raw);
}
- updateParts();
updateTypeInfo(lookup, context);
setIsTrait(context, lookup, attributeName);
updateEdgeInfo(lookup, context);
@@ -180,15 +181,29 @@ public class IdentifierHelper {
}
private void updateTypeInfo(org.apache.atlas.query.Lookup lookup, GremlinQueryComposer.Context context) {
+ parts = StringUtils.split(raw, ".");
+
+ // check if this is a namespace attribute
+ if (parts.length == 2) {
+ try {
+ AtlasType type = lookup.getType(parts[0]);
+
+ if (type instanceof AtlasNamespaceType) {
+ parts = new String[1];
+ parts[0] = raw;
+ }
+ } catch (AtlasBaseException excp) {
+ // ignore
+ }
+ }
+
if (parts.length == 1) {
typeName = context.hasAlias(parts[0]) ?
context.getTypeNameFromAlias(parts[0]) :
context.getActiveTypeName();
qualifiedName = getDefaultQualifiedNameForSinglePartName(context, parts[0]);
attributeName = parts[0];
- }
-
- if (parts.length == 2) {
+ } else if (parts.length == 2) {
boolean isAttrOfActiveType = lookup.hasAttribute(context, parts[0]);
if (isAttrOfActiveType) {
attributeName = parts[0];
@@ -239,10 +254,6 @@ public class IdentifierHelper {
}
}
- private void updateParts() {
- parts = StringUtils.split(raw, ".");
- }
-
public String getQualifiedName() {
return qualifiedName;
}
diff --git a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
index 2878a74..2b34abf 100644
--- a/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
+++ b/repository/src/main/java/org/apache/atlas/query/RegistryBasedLookup.java
@@ -21,7 +21,6 @@ package org.apache.atlas.query;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.TypeCategory;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
-import org.apache.atlas.repository.Constants;
import org.apache.atlas.type.*;
import org.apache.commons.lang.StringUtils;
@@ -29,20 +28,12 @@ import java.util.*;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_CLASSIFIED;
import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_NOT_CLASSIFIED;
+import static org.apache.atlas.discovery.SearchContext.MATCH_ALL_ENTITY_TYPES;
import static org.apache.atlas.model.discovery.SearchParameters.ALL_CLASSIFICATIONS;
import static org.apache.atlas.model.discovery.SearchParameters.NO_CLASSIFICATIONS;
+import static org.apache.atlas.model.discovery.SearchParameters.ALL_ENTITY_TYPES;
class RegistryBasedLookup implements Lookup {
- private static final Set<String> SYSTEM_ATTRIBUTES = new HashSet<>(
- Arrays.asList(Constants.GUID_PROPERTY_KEY,
- Constants.MODIFIED_BY_KEY,
- Constants.CREATED_BY_KEY,
- Constants.STATE_PROPERTY_KEY,
- Constants.TIMESTAMP_PROPERTY_KEY,
- Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
- Constants.HOME_ID_KEY
- ));
-
private static final Map<String, String> NUMERIC_ATTRIBUTES = new HashMap<String, String>() {{
put(AtlasBaseTypeDef.ATLAS_TYPE_SHORT, "");
put(AtlasBaseTypeDef.ATLAS_TYPE_INT, "");
@@ -63,10 +54,12 @@ class RegistryBasedLookup implements Lookup {
public AtlasType getType(String typeName) throws AtlasBaseException {
AtlasType ret;
- if (typeName.equalsIgnoreCase(ALL_CLASSIFICATIONS)) {
+ if (typeName.equals(ALL_CLASSIFICATIONS)) {
ret = MATCH_ALL_CLASSIFIED;
- } else if (typeName.equalsIgnoreCase(NO_CLASSIFICATIONS)) {
+ } else if (typeName.equals(NO_CLASSIFICATIONS)) {
ret = MATCH_ALL_NOT_CLASSIFIED;
+ } else if (typeName.equals(ALL_ENTITY_TYPES)) {
+ ret = MATCH_ALL_ENTITY_TYPES;
} else {
ret = typeRegistry.getType(typeName);
}
@@ -81,15 +74,7 @@ class RegistryBasedLookup implements Lookup {
return "";
}
- if(isSystemAttribute(name)) {
- return name;
- } else {
- return et.getQualifiedAttributeName(name);
- }
- }
-
- private boolean isSystemAttribute(String s) {
- return SYSTEM_ATTRIBUTES.contains(s);
+ return et.getQualifiedAttributeName(name);
}
@Override
@@ -99,10 +84,6 @@ class RegistryBasedLookup implements Lookup {
return false;
}
- if(isSystemAttribute(attributeName)) {
- return true;
- }
-
AtlasType at = getAttributeType(et, attributeName);
if(at == null) {
return false;
@@ -144,8 +125,7 @@ class RegistryBasedLookup implements Lookup {
public boolean hasAttribute(GremlinQueryComposer.Context context, String typeName) {
AtlasEntityType entityType = context.getActiveEntityType();
- return (entityType != null) &&
- (isSystemAttribute(typeName) || entityType.hasAttribute(typeName) || entityType.hasRelationshipAttribute(typeName));
+ return getAttribute(entityType, typeName) != null;
}
@Override
@@ -178,6 +158,8 @@ class RegistryBasedLookup implements Lookup {
t = MATCH_ALL_CLASSIFIED;
} else if (typeName.equalsIgnoreCase(NO_CLASSIFICATIONS)) {
t = MATCH_ALL_NOT_CLASSIFIED;
+ } else if (typeName.equalsIgnoreCase(ALL_ENTITY_TYPES)) {
+ t = MATCH_ALL_ENTITY_TYPES;
} else {
t = typeRegistry.getType(typeName);
}
@@ -257,6 +239,11 @@ class RegistryBasedLookup implements Lookup {
@Override
public String getVertexPropertyName(String typeName, String attrName) {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName);
+
+ if (entityType == null && StringUtils.equals(typeName, ALL_ENTITY_TYPES)) {
+ entityType = MATCH_ALL_ENTITY_TYPES;
+ }
+
AtlasStructType.AtlasAttribute attribute = getAttribute(entityType, attrName);
if (attribute == null) {
return null;
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index f511e2f..c536f3b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -1377,19 +1377,19 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
private void validateNamespaceAttributes(AtlasVertex entityVertex, AtlasEntityType entityType, Map<String, Map<String, Object>> entityNamespaces, boolean isOverwrite) throws AtlasBaseException {
List<String> messages = new ArrayList<>();
- Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
for (String nsName : entityNamespaces.keySet()) {
- if (!entityNamespaces.containsKey(nsName)) {
+ if (!entityTypeNamespaces.containsKey(nsName)) {
messages.add(nsName + ": invalid namespace for entity type " + entityType.getTypeName());
continue;
}
- List<AtlasNamespaceAttribute> entityTypeNsAttributes = entityTypeNamespaces.get(nsName);
- Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName);
+ Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entityTypeNamespaces.get(nsName);
+ Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName);
- for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) {
+ for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) {
AtlasType attrType = nsAttribute.getAttributeType();
String attrName = nsAttribute.getName();
Object attrValue = entityNsAttributes.get(attrName);
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 113325d..7461931 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -427,14 +427,14 @@ public class EntityGraphMapper {
LOG.debug("==> setNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces);
}
- Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
- Map<String, Object> entityNsAttributes = MapUtils.isEmpty(entityNamespaces) ? null : entityNamespaces.get(nsName);
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
+ Map<String, Object> entityNsAttributes = MapUtils.isEmpty(entityNamespaces) ? null : entityNamespaces.get(nsName);
- for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) {
+ for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) {
String nsAttrName = nsAttribute.getName();
Object nsAttrExistingValue = entityVertex.getProperty(nsAttribute.getVertexPropertyName(), Object.class);
Object nsAttrNewValue = MapUtils.isEmpty(entityNsAttributes) ? null : entityNsAttributes.get(nsAttrName);
@@ -480,19 +480,19 @@ public class EntityGraphMapper {
LOG.debug("==> addOrUpdateNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces);
}
- Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
if (MapUtils.isNotEmpty(entityTypeNamespaces) && MapUtils.isNotEmpty(entityNamespaces)) {
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
- Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName);
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
+ Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName);
if (MapUtils.isEmpty(entityNsAttributes)) {
continue;
}
- for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) {
+ for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) {
String nsAttrName = nsAttribute.getName();
if (!entityNsAttributes.containsKey(nsAttrName)) {
@@ -528,12 +528,12 @@ public class EntityGraphMapper {
LOG.debug("==> removeNamespaceAttributes(entityVertex={}, entityType={}, entityNamespaces={}", entityVertex, entityType.getTypeName(), entityNamespaces);
}
- Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType.getNamespaceAttributes();
if (MapUtils.isNotEmpty(entityTypeNamespaces) && MapUtils.isNotEmpty(entityNamespaces)) {
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> entityTypeNsAttributes = entry.getValue();
if (!entityNamespaces.containsKey(nsName)) { // nothing to remove for this namespace
continue;
@@ -541,7 +541,7 @@ public class EntityGraphMapper {
Map<String, Object> entityNsAttributes = entityNamespaces.get(nsName);
- for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes) {
+ for (AtlasNamespaceAttribute nsAttribute : entityTypeNsAttributes.values()) {
// if (entityNsAttributes is empty) remove all attributes in this namespace
// else remove the attribute only if its given in entityNsAttributes
if (MapUtils.isEmpty(entityNsAttributes) || entityNsAttributes.containsKey(nsAttribute.getName())) {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index 55c1cac..3c22d9a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -678,21 +678,26 @@ public class EntityGraphRetriever {
if (CollectionUtils.isNotEmpty(attributes)) {
for (String attrName : attributes) {
- String nonQualifiedAttrName = toNonQualifiedName(attrName);
- if (ret.hasAttribute(attrName)) {
- continue;
- }
-
- AtlasAttribute attribute = entityType.getAttribute(nonQualifiedAttrName);
+ AtlasAttribute attribute = entityType.getAttribute(attrName);
if (attribute == null) {
- attribute = entityType.getRelationshipAttribute(nonQualifiedAttrName, null);
+ attrName = toNonQualifiedName(attrName);
+
+ if (ret.hasAttribute(attrName)) {
+ continue;
+ }
+
+ attribute = entityType.getAttribute(attrName);
+
+ if (attribute == null) {
+ attribute = entityType.getRelationshipAttribute(attrName, null);
+ }
}
Object attrValue = getVertexAttribute(entityVertex, attribute);
if (attrValue != null) {
- ret.setAttribute(nonQualifiedAttrName, attrValue);
+ ret.setAttribute(attrName, attrValue);
}
}
}
@@ -765,15 +770,15 @@ public class EntityGraphRetriever {
}
private void mapNamespaceAttributes(AtlasVertex entityVertex, AtlasEntity entity) throws AtlasBaseException {
- AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
- Map<String, List<AtlasNamespaceAttribute>> entityTypeNamespaces = entityType != null ? entityType.getNamespaceAttributes() : null;
+ AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
+ Map<String, Map<String, AtlasNamespaceAttribute>> entityTypeNamespaces = entityType != null ? entityType.getNamespaceAttributes() : null;
if (MapUtils.isNotEmpty(entityTypeNamespaces)) {
- for (Map.Entry<String, List<AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
- String nsName = entry.getKey();
- List<AtlasNamespaceAttribute> nsAttributes = entry.getValue();
+ for (Map.Entry<String, Map<String, AtlasNamespaceAttribute>> entry : entityTypeNamespaces.entrySet()) {
+ String nsName = entry.getKey();
+ Map<String, AtlasNamespaceAttribute> nsAttributes = entry.getValue();
- for (AtlasNamespaceAttribute nsAttribute : nsAttributes) {
+ for (AtlasNamespaceAttribute nsAttribute : nsAttributes.values()) {
Object nsAttrValue = mapVertexToAttribute(entityVertex, nsAttribute, null, false, false);
if (nsAttrValue != null) {
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
index 8f6afc7..cbac18c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/AtlasNamespaceDefStoreV2Test.java
@@ -25,7 +25,7 @@ import org.apache.atlas.model.typedef.AtlasNamespaceDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasNamespaceType;
+import org.apache.atlas.type.AtlasNamespaceType.AtlasNamespaceAttribute;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.testng.Assert;
@@ -84,7 +84,7 @@ public class AtlasNamespaceDefStoreV2Test {
createNamespaceTypes(namespaceName);
Assert.assertEquals(typeRegistry.getAllNamespaceDefs().size(), 1);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table");
- Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes();
Assert.assertEquals(m1.get(namespaceName).size(), 2);
}
@@ -117,7 +117,7 @@ public class AtlasNamespaceDefStoreV2Test {
updateNamespaceTypeDefs(namespaceDef);
typeDefStore.updateTypesDef(typesDefs);
AtlasEntityType entityType = typeRegistry.getEntityTypeByName("hive_table");
- Map<String, List<AtlasNamespaceType.AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes();
+ Map<String, Map<String, AtlasNamespaceAttribute>> m1 = entityType.getNamespaceAttributes();
Assert.assertEquals(m1.get(namespaceName).size(), 3);
}