You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2016/11/18 08:23:42 UTC
incubator-atlas git commit: ATLAS-1242: update TypesResource API
implementation to use new v2 TypesREST API
Repository: incubator-atlas
Updated Branches:
refs/heads/master 710c17f55 -> 95083cb01
ATLAS-1242: update TypesResource API implementation to use new v2 TypesREST API
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/95083cb0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/95083cb0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/95083cb0
Branch: refs/heads/master
Commit: 95083cb01fd2daeded1d55f6a2ce4791a933c974
Parents: 710c17f
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Nov 17 14:12:07 2016 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Fri Nov 18 00:23:07 2016 -0800
----------------------------------------------------------------------
.../java/org/apache/atlas/AtlasErrorCode.java | 1 +
.../org/apache/atlas/model/SearchFilter.java | 2 +-
release-log.txt | 1 +
.../atlas/typesystem/types/utils/TypesUtil.java | 2 +-
.../java/org/apache/atlas/util/RestUtils.java | 520 +++++++++++++++++++
.../atlas/web/resources/TypesResource.java | 117 +++--
.../web/resources/TypesJerseyResourceIT.java | 2 +-
7 files changed, 600 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 6a35d2b..8ee3458 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -54,6 +54,7 @@ public enum AtlasErrorCode {
TYPE_ALREADY_EXISTS(409, "ATLAS4091E", "Given type {0} already exists"),
TYPE_HAS_REFERENCES(409, "ATLAS4092E", "Given type {0} has references"),
TYPE_MATCH_FAILED(409, "ATLAS4093E", "Given type {0} doesn't match {1}"),
+ INVALID_TYPE_DEFINITION(409, "ATLAS4094E", "Invalid type definition {0}"),
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/intg/src/main/java/org/apache/atlas/model/SearchFilter.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/SearchFilter.java b/intg/src/main/java/org/apache/atlas/model/SearchFilter.java
index 64800a5..b5f6c3a 100644
--- a/intg/src/main/java/org/apache/atlas/model/SearchFilter.java
+++ b/intg/src/main/java/org/apache/atlas/model/SearchFilter.java
@@ -41,7 +41,7 @@ import javax.xml.bind.annotation.XmlRootElement;
public class SearchFilter {
public static final String PARAM_TYPE = "type";
public static final String PARAM_SUPERTYPE = "supertype";
- public static final String PARAM_NOT_SUPERTYPE = "notSupertype";
+ public static final String PARAM_NOT_SUPERTYPE = "notsupertype";
/**
* to specify whether the result should be sorted? If yes, whether asc or desc.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a3f4f20..6a17548 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
+ATLAS-1242 update TypesResource API implementation to use new v2 TypesREST API
ATLAS-1306 bootstrap type-load ignores model file contents if a type in the file already exists
ATLAS-1299 The project org.apache.atlas:atlas-hbase-client-shaded - build error (shwethags)
ATLAS-1246 Upgrade versions of dependencies (shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
index 77cbd39..7923a36 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
@@ -82,7 +82,7 @@ public class TypesUtil {
public static HierarchicalTypeDefinition<TraitType> createTraitTypeDef(String name, String description, String version,
ImmutableSet<String> superTypes, AttributeDefinition... attrDefs) {
- return new HierarchicalTypeDefinition<>(TraitType.class, name, description, superTypes, attrDefs);
+ return new HierarchicalTypeDefinition<>(TraitType.class, name, description, version, superTypes, attrDefs);
}
public static StructTypeDefinition createStructTypeDef(String name, AttributeDefinition... attrDefs) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/main/java/org/apache/atlas/util/RestUtils.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/util/RestUtils.java b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java
new file mode 100644
index 0000000..bc937e4
--- /dev/null
+++ b/webapp/src/main/java/org/apache/atlas/util/RestUtils.java
@@ -0,0 +1,520 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.util;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumElementDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef.Cardinality;
+import org.apache.atlas.model.typedef.AtlasTypeDefHeader;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasEntityType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.type.AtlasTypeUtil;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.atlas.AtlasErrorCode.INVALID_TYPE_DEFINITION;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_ON_DELETE;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_FOREIGN_KEY;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_TYPE_MAPPED_FROM_REF;
+import static org.apache.atlas.model.typedef.AtlasStructDef.AtlasConstraintDef.CONSTRAINT_PARAM_VAL_CASCADE;
+import static org.apache.atlas.type.AtlasTypeUtil.isArrayType;
+
+
+public final class RestUtils {
+ private RestUtils() {}
+ private static final Logger LOG = LoggerFactory.getLogger(RestUtils.class);
+
+ public static TypesDef toTypesDef(AtlasEnumDef enumDef) {
+ TypesDef ret = null;
+
+ if (enumDef != null) {
+ String enumName = enumDef.getName();
+ String enumDesc = enumDef.getDescription();
+ String enumVersion = enumDef.getTypeVersion();
+ EnumValue[] enumValues = getEnumValues(enumDef.getElementDefs());
+
+ if (enumName != null && enumValues != null && enumValues.length > 0) {
+ EnumTypeDefinition enumTypeDef = new EnumTypeDefinition(enumName, enumDesc, enumVersion, enumValues);
+
+ ret = TypesUtil.getTypesDef(ImmutableList.of(enumTypeDef),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ }
+ }
+
+ return ret;
+ }
+
+ public static TypesDef toTypesDef(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException {
+ String typeName = structDef.getName();
+ String typeDesc = structDef.getDescription();
+ AttributeDefinition[] attributes = getAttributes(structDef, registry);
+ StructTypeDefinition structType = TypesUtil.createStructTypeDef(typeName, typeDesc, attributes);
+
+ TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.of(structType),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+
+ return ret;
+ }
+
+ public static TypesDef toTypesDef(AtlasEntityDef entityDef, AtlasTypeRegistry registry) throws AtlasBaseException {
+ String typeName = entityDef.getName();
+ String typeDesc = entityDef.getDescription();
+ String typeVersion = entityDef.getTypeVersion();
+ ImmutableSet superTypes = ImmutableSet.copyOf(entityDef.getSuperTypes());
+ AttributeDefinition[] attributes = getAttributes(entityDef, registry);
+
+ HierarchicalTypeDefinition<ClassType> classType = TypesUtil.createClassTypeDef(typeName, typeDesc, typeVersion,
+ superTypes, attributes);
+ TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(classType));
+
+ return ret;
+ }
+
+ public static TypesDef toTypesDef(AtlasClassificationDef classifDef, AtlasTypeRegistry registry) throws AtlasBaseException {
+ String typeName = classifDef.getName();
+ String typeDesc = classifDef.getDescription();
+ String typeVersion = classifDef.getTypeVersion();
+ ImmutableSet superTypes = ImmutableSet.copyOf(classifDef.getSuperTypes());
+ AttributeDefinition[] attributes = getAttributes(classifDef, registry);
+
+ HierarchicalTypeDefinition traitType = TypesUtil.createTraitTypeDef(typeName, typeDesc, typeVersion, superTypes,
+ attributes);
+ TypesDef ret = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(),
+ ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(traitType),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ return ret;
+ }
+
+
+
+ public static AtlasTypesDef toAtlasTypesDef(String typeDefinition, AtlasTypeRegistry registry) throws AtlasBaseException {
+ AtlasTypesDef ret = new AtlasTypesDef();
+
+ try {
+ if (StringUtils.isEmpty(typeDefinition)) {
+ throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition);
+ }
+
+ TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
+ if (CollectionUtils.isNotEmpty(typesDef.enumTypesAsJavaList())) {
+ List<AtlasEnumDef> enumDefs = toAtlasEnumDefs(typesDef.enumTypesAsJavaList());
+ ret.setEnumDefs(enumDefs);
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.structTypesAsJavaList())) {
+ List<AtlasStructDef> structDefs = toAtlasStructDefs(typesDef.structTypesAsJavaList());
+ ret.setStructDefs(structDefs);
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.classTypesAsJavaList())) {
+ List<AtlasEntityDef> entityDefs = toAtlasEntityDefs(typesDef.classTypesAsJavaList(), registry);
+ ret.setEntityDefs(entityDefs);
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.traitTypesAsJavaList())) {
+ List<AtlasClassificationDef> classificationDefs = toAtlasClassificationDefs(typesDef.traitTypesAsJavaList());
+ ret.setClassificationDefs(classificationDefs);
+ }
+
+ } catch (Exception e) {
+ LOG.error("Invalid type definition = {}", typeDefinition, e);
+ throw new AtlasBaseException(INVALID_TYPE_DEFINITION, typeDefinition);
+ }
+
+ return ret;
+ }
+
+ public static ImmutableList<String> getTypeNames(List<AtlasTypeDefHeader> atlasTypesDefs) {
+ List<String> ret = new ArrayList<String>();
+ if (CollectionUtils.isNotEmpty(atlasTypesDefs)) {
+ for (AtlasTypeDefHeader atlasTypesDef : atlasTypesDefs) {
+ ret.add(atlasTypesDef.getName());
+ }
+ }
+
+ return ImmutableList.copyOf(ret);
+ }
+
+ public static List<String> getTypeNames(AtlasTypesDef typesDef) {
+ List<AtlasTypeDefHeader> atlasTypesDefs = AtlasTypeUtil.toTypeDefHeader(typesDef);
+ return getTypeNames(atlasTypesDefs);
+ }
+
+ private static List<AtlasEnumDef> toAtlasEnumDefs(List<EnumTypeDefinition> enumTypeDefinitions) {
+ List<AtlasEnumDef> ret = new ArrayList<AtlasEnumDef>();
+
+ for (EnumTypeDefinition enumType : enumTypeDefinitions) {
+ AtlasEnumDef enumDef = new AtlasEnumDef();
+ enumDef.setName(enumType.name);
+ enumDef.setDescription(enumType.description);
+ enumDef.setTypeVersion(enumType.version);
+ enumDef.setElementDefs(getAtlasEnumElementDefs(enumType.enumValues));
+
+ ret.add(enumDef);
+ }
+
+ return ret;
+ }
+
+ private static List<AtlasStructDef> toAtlasStructDefs(List<StructTypeDefinition> structTypeDefinitions)
+ throws AtlasBaseException {
+ List<AtlasStructDef> ret = new ArrayList<AtlasStructDef>();
+
+ for (StructTypeDefinition structType : structTypeDefinitions) {
+ AtlasStructDef structDef = new AtlasStructDef();
+ List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>();
+
+ structDef.setName(structType.typeName);
+ structDef.setDescription(structType.typeDescription);
+ structDef.setTypeVersion(structType.typeVersion);
+
+ AttributeDefinition[] attrDefinitions = structType.attributeDefinitions;
+ for (AttributeDefinition attrDefinition : attrDefinitions) {
+ attrDefs.add(toAtlasAttributeDef(attrDefinition));
+ }
+
+ structDef.setAttributeDefs(attrDefs);
+ ret.add(structDef);
+ }
+
+ return ret;
+ }
+
+ private static List<AtlasClassificationDef> toAtlasClassificationDefs(List<HierarchicalTypeDefinition<TraitType>> traitTypeDefinitions)
+ throws AtlasBaseException {
+ List<AtlasClassificationDef> ret = new ArrayList<AtlasClassificationDef>();
+
+ for (HierarchicalTypeDefinition<TraitType> traitType : traitTypeDefinitions) {
+ AtlasClassificationDef classifDef = new AtlasClassificationDef();
+ List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>();
+
+ classifDef.setName(traitType.typeName);
+ classifDef.setDescription(traitType.typeDescription);
+ classifDef.setTypeVersion(traitType.typeVersion);
+ classifDef.setSuperTypes(traitType.superTypes);
+
+ AttributeDefinition[] attrDefinitions = traitType.attributeDefinitions;
+ for (AttributeDefinition attrDefinition : attrDefinitions) {
+ attrDefs.add(toAtlasAttributeDef(attrDefinition));
+ }
+
+ classifDef.setAttributeDefs(attrDefs);
+ ret.add(classifDef);
+ }
+
+ return ret;
+ }
+
+ private static List<AtlasEntityDef> toAtlasEntityDefs(List<HierarchicalTypeDefinition<ClassType>> classTypeDefinitions,
+ AtlasTypeRegistry registry) throws AtlasBaseException {
+ List<AtlasEntityDef> atlasEntityDefs = new ArrayList<AtlasEntityDef>();
+
+ for (HierarchicalTypeDefinition<ClassType> classType : classTypeDefinitions) {
+ List<AtlasAttributeDef> attrDefs = new ArrayList<AtlasAttributeDef>();
+ AtlasEntityDef atlasEntityDef = new AtlasEntityDef();
+ String classTypeDefName = classType.typeName;
+
+ atlasEntityDef.setName(classTypeDefName);
+ atlasEntityDef.setDescription(classType.typeDescription);
+ atlasEntityDef.setTypeVersion(classType.typeVersion);
+ atlasEntityDef.setSuperTypes(classType.superTypes);
+
+ AttributeDefinition[] attrDefinitions = classType.attributeDefinitions;
+ for (AttributeDefinition oldAttr : attrDefinitions) {
+ AtlasAttributeDef newAttr = toAtlasAttributeDef(oldAttr);
+
+ // isComposite and reverseAttributeName applicable only for entities/classes.
+ if (oldAttr.isComposite) {
+ String attrType = oldAttr.dataTypeName;
+ attrType = isArrayType(attrType) ? getArrayTypeName(attrType) : attrType;
+
+ if (!AtlasTypeUtil.isBuiltInType(attrType)) {
+ String refAttrName = null;
+
+ // 1. Check if attribute datatype is present in payload definition, if present get the typeDefinition,
+ // check all its attributes and find attribute that matches with classTypeDefName and reverseAttributeName
+ HierarchicalTypeDefinition<ClassType> refType = findClassType(classTypeDefinitions, attrType);
+ if (refType != null) {
+ for (AttributeDefinition refAttr : refType.attributeDefinitions) {
+ String refAttrDataTypeName = refAttr.dataTypeName;
+ String refAttrRevAttrName = refAttr.reverseAttributeName;
+
+ if (StringUtils.equals(refAttrDataTypeName, classTypeDefName) &&
+ StringUtils.equals(refAttrRevAttrName, oldAttr.name)) {
+ refAttrName = refAttr.name;
+ break;
+ }
+ }
+ }
+
+ // 2. Check if attribute present in typeRegistry. If present fetch all attributes associated with the type and
+ // check revAttrName equals base type attr name AND classTypeDefName equals attribute name
+ else {
+ if (registry.isRegisteredType(attrType)) {
+ AtlasType atlasType = registry.getType(attrType);
+
+ if (isEntity(atlasType)) {
+ AtlasEntityType entityType = (AtlasEntityType) atlasType;
+ List<AtlasAttributeDef> atlasAttrDefs = entityType.getEntityDef().getAttributeDefs();
+
+ for (AtlasAttributeDef attrDef : atlasAttrDefs) {
+ boolean isForeignKey = entityType.isForeignKeyAttribute(attrDef.getName());
+
+ if (isForeignKey) {
+ AtlasType attribType = entityType.getAttributeType(attrDef.getName());
+
+ if (attribType.getTypeCategory() == TypeCategory.ARRAY) {
+ attribType = ((AtlasArrayType) attribType).getElementType();
+ }
+
+ if (attribType.getTypeCategory() == TypeCategory.ENTITY) {
+ String revAttrName = ((AtlasEntityType) attribType).
+ getMappedFromRefAttribute(entityType.getTypeName(), attrDef.getName());
+
+ if (StringUtils.equals(classTypeDefName , attrDef.getTypeName()) &&
+ StringUtils.equals(oldAttr.name, revAttrName)) {
+ refAttrName = attrDef.getName();
+ }
+ }
+ }
+
+ }
+ }
+ }
+ }
+
+ if (StringUtils.isNotBlank(refAttrName)) { // ex: hive_table.columns, hive_column.table
+ Map<String, Object> params = new HashMap<>();
+ params.put(AtlasConstraintDef.CONSTRAINT_PARAM_REF_ATTRIBUTE, refAttrName);
+
+ newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_MAPPED_FROM_REF, params));
+ } else { // ex: hive_table.partitionKeys, with no reverseAttribute-reference
+ newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY));
+ }
+ }
+
+ } else if (StringUtils.isNotEmpty(oldAttr.reverseAttributeName)) {
+ Map<String, Object> params = new HashMap<>();
+ params.put(CONSTRAINT_PARAM_ON_DELETE, CONSTRAINT_PARAM_VAL_CASCADE);
+
+ newAttr.addConstraint(new AtlasConstraintDef(CONSTRAINT_TYPE_FOREIGN_KEY, params));
+ }
+ attrDefs.add(newAttr);
+ }
+
+ atlasEntityDef.setAttributeDefs(attrDefs);
+ atlasEntityDefs.add(atlasEntityDef);
+ }
+
+ return atlasEntityDefs;
+ }
+
+ private static String getArrayTypeName(String attrType) {
+ String ret = null;
+ if (isArrayType(attrType)) {
+ Set<String> typeNames = AtlasTypeUtil.getReferencedTypeNames(attrType);
+ if (typeNames.size() > 0) {
+ ret = typeNames.iterator().next();
+ }
+ }
+
+ return ret;
+ }
+
+ private static List<AtlasEnumElementDef> getAtlasEnumElementDefs(EnumValue[] enums) {
+ List<AtlasEnumElementDef> ret = new ArrayList<AtlasEnumElementDef>();
+
+ for (EnumValue enumElem : enums) {
+ ret.add(new AtlasEnumElementDef(enumElem.value, null, enumElem.ordinal));
+ }
+
+ return ret;
+ }
+
+ private static EnumValue[] getEnumValues(List<AtlasEnumElementDef> enumDefs) {
+ List<EnumValue> ret = new ArrayList<EnumValue>();
+
+ if (CollectionUtils.isNotEmpty(enumDefs)) {
+ for (AtlasEnumElementDef enumDef : enumDefs) {
+ if (enumDef != null) {
+ ret.add(new EnumValue(enumDef.getValue(), enumDef.getOrdinal()));
+ }
+ }
+ }
+
+ return ret.toArray(new EnumValue[ret.size()]);
+ }
+
+ private static AtlasAttributeDef toAtlasAttributeDef(AttributeDefinition attrDefinition) {
+ AtlasAttributeDef ret = new AtlasAttributeDef();
+
+ ret.setName(attrDefinition.name);
+ ret.setTypeName(attrDefinition.dataTypeName);
+ ret.setIsIndexable(attrDefinition.isIndexable);
+ ret.setIsUnique(attrDefinition.isUnique);
+
+ // Multiplicity attribute mapping
+ Multiplicity multiplicity = attrDefinition.multiplicity;
+ int minCount = multiplicity.lower;
+ int maxCount = multiplicity.upper;
+ boolean isUnique = multiplicity.isUnique;
+
+ if (minCount == 0) {
+ ret.setIsOptional(true);
+ ret.setValuesMinCount(0);
+ } else {
+ ret.setIsOptional(false);
+ ret.setValuesMinCount(minCount);
+ }
+
+ if (maxCount < 2) {
+ ret.setCardinality(Cardinality.SINGLE);
+ ret.setValuesMaxCount(1);
+ } else {
+ if (!isUnique) {
+ ret.setCardinality(Cardinality.LIST);
+ } else {
+ ret.setCardinality(Cardinality.SET);
+ }
+
+ ret.setValuesMaxCount(maxCount);
+ }
+
+ return ret;
+ }
+
+ private static AttributeDefinition[] getAttributes(AtlasStructDef structDef, AtlasTypeRegistry registry) throws AtlasBaseException {
+ List<AttributeDefinition> ret = new ArrayList<>();
+ List<AtlasAttributeDef> attrDefs = structDef.getAttributeDefs();
+
+ if (CollectionUtils.isNotEmpty(attrDefs)) {
+
+ for (AtlasAttributeDef attrDef : attrDefs) {
+ String name = attrDef.getName();
+ String dataTypeName = attrDef.getTypeName();
+ Boolean isUnique = attrDef.getIsUnique();
+ Boolean isIndexable = attrDef.getIsIndexable();
+ String reverseAttribName = null;
+ boolean isComposite;
+
+ // Multiplicity mapping
+ final int lower;
+ final int upper;
+
+ if (attrDef.getCardinality() == Cardinality.SINGLE) {
+ lower = attrDef.getIsOptional() ? 0 : 1;
+ upper = 1;
+ } else {
+ if(attrDef.getIsOptional()) {
+ lower = 0;
+ } else {
+ lower = attrDef.getValuesMinCount() < 1 ? 1 : attrDef.getValuesMinCount();
+ }
+
+ upper = attrDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attrDef.getValuesMaxCount();
+ }
+
+ Multiplicity multiplicity = new Multiplicity(lower, upper, Cardinality.SET.equals(attrDef.getCardinality()));
+
+ // Constraint checks:
+ // 1. [ mappedFromRef -> isComposite ]
+ // 2. [ foreignKey(onDelete=cascade) -> reverseAttribute ]
+ AtlasStructType structType = (AtlasStructType) registry.getType(structDef.getName());
+ boolean isForeignKey = structType.isForeignKeyAttribute(attrDef.getName());
+ boolean isMappedFromRef = structType.isMappedFromRefAttribute(attrDef.getName());
+ AtlasType attrType = structType.getAttributeType(attrDef.getName());
+
+ if (attrType != null && isForeignKey) {
+ if (attrType.getTypeCategory() == TypeCategory.ARRAY) {
+ attrType = ((AtlasArrayType) attrType).getElementType();
+ }
+
+ if (attrType.getTypeCategory() == TypeCategory.ENTITY) {
+ reverseAttribName = ((AtlasStructType) attrType).
+ getMappedFromRefAttribute(structType.getTypeName(), attrDef.getName());
+ }
+ }
+ isComposite = isMappedFromRef || (isForeignKey && StringUtils.isBlank(reverseAttribName));
+ ret.add(new AttributeDefinition(name, dataTypeName, multiplicity, isComposite, isUnique, isIndexable, reverseAttribName));
+ }
+ }
+
+ return ret.toArray(new AttributeDefinition[ret.size()]);
+ }
+
+ private static HierarchicalTypeDefinition<ClassType> findClassType(List<HierarchicalTypeDefinition<ClassType>> classDefs,
+ String typeName) {
+ HierarchicalTypeDefinition<ClassType> ret = null;
+
+ if (CollectionUtils.isNotEmpty(classDefs)) {
+ for (HierarchicalTypeDefinition<ClassType> classType : classDefs) {
+ if (classType.typeName.equalsIgnoreCase(typeName)) {
+ ret = classType;
+ }
+ }
+ }
+
+ return ret;
+ }
+
+ private static boolean isEntity(AtlasType type) {
+ return type.getTypeCategory() == TypeCategory.ENTITY;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
index 1eafc7d..e899fcf 100755
--- a/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
+++ b/webapp/src/main/java/org/apache/atlas/web/resources/TypesResource.java
@@ -19,15 +19,26 @@
package org.apache.atlas.web.resources;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.core.ResourceContext;
import org.apache.atlas.AtlasClient;
+import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
-import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.util.RestUtils;
import org.apache.atlas.utils.AtlasPerfTracer;
+import org.apache.atlas.web.rest.TypesREST;
import org.apache.atlas.web.util.Servlets;
-import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
@@ -49,9 +60,7 @@ import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
/**
* This class provides RESTful API for Types.
@@ -64,17 +73,18 @@ import java.util.Map;
@Path("types")
@Singleton
public class TypesResource {
-
private static final Logger LOG = LoggerFactory.getLogger(TypesResource.class);
private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.TypesResource");
-
- private final MetadataService metadataService;
+ private static AtlasTypeRegistry typeRegistry;
@Inject
- public TypesResource(MetadataService metadataService) {
- this.metadataService = metadataService;
+ public TypesResource(AtlasTypeRegistry typeRegistry) {
+ this.typeRegistry = typeRegistry;
}
+ @Context
+ private ResourceContext resourceContext;
+
/**
* Submits a type definition corresponding to a given type representing a meta model of a
* domain. Could represent things like Hive Database, Hive Table, etc.
@@ -83,7 +93,10 @@ public class TypesResource {
@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response submit(@Context HttpServletRequest request) {
+ TypesREST typesRest = resourceContext.getResource(TypesREST.class);
AtlasPerfTracer perf = null;
+ JSONArray typesResponse = new JSONArray();
+
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.submit()");
@@ -92,12 +105,12 @@ public class TypesResource {
final String typeDefinition = Servlets.getRequestPayload(request);
LOG.info("Creating type with definition {} ", typeDefinition);
- JSONObject typesJson = metadataService.createType(typeDefinition);
- final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES);
+ AtlasTypesDef createTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry);
+ AtlasTypesDef createdTypesDef = typesRest.createAtlasTypeDefs(createTypesDef);
+ List<String> typeNames = RestUtils.getTypeNames(createdTypesDef);
- JSONArray typesResponse = new JSONArray();
- for (int i = 0; i < typesJsonArray.length(); i++) {
- final String name = typesJsonArray.getString(i);
+ for (int i = 0; i < typeNames.size(); i++) {
+ final String name = typeNames.get(i);
typesResponse.put(new JSONObject() {{
put(AtlasClient.NAME, name);
}});
@@ -107,10 +120,10 @@ public class TypesResource {
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
response.put(AtlasClient.TYPES, typesResponse);
return Response.status(ClientResponse.Status.CREATED).entity(response).build();
- } catch (TypeExistsException e) {
+ } catch (AtlasBaseException e) {
LOG.error("Type already exists", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.CONFLICT));
- } catch (AtlasException | IllegalArgumentException e) {
+ } catch (IllegalArgumentException e) {
LOG.error("Unable to persist types", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.BAD_REQUEST));
} catch (Throwable e) {
@@ -134,7 +147,9 @@ public class TypesResource {
@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON})
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response update(@Context HttpServletRequest request) {
+ TypesREST typesRest = resourceContext.getResource(TypesREST.class);
AtlasPerfTracer perf = null;
+ JSONArray typesResponse = new JSONArray();
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.update()");
@@ -143,12 +158,12 @@ public class TypesResource {
final String typeDefinition = Servlets.getRequestPayload(request);
LOG.info("Updating type with definition {} ", typeDefinition);
- JSONObject typesJson = metadataService.updateType(typeDefinition);
- final JSONArray typesJsonArray = typesJson.getJSONArray(AtlasClient.TYPES);
+ AtlasTypesDef updateTypesDef = RestUtils.toAtlasTypesDef(typeDefinition, typeRegistry);
+ AtlasTypesDef updatedTypesDef = typesRest.updateAtlasTypeDefs(updateTypesDef);
+ List<String> typeNames = RestUtils.getTypeNames(updatedTypesDef);
- JSONArray typesResponse = new JSONArray();
- for (int i = 0; i < typesJsonArray.length(); i++) {
- final String name = typesJsonArray.getString(i);
+ for (int i = 0; i < typeNames.size(); i++) {
+ final String name = typeNames.get(i);
typesResponse.put(new JSONObject() {{
put(AtlasClient.NAME, name);
}});
@@ -181,21 +196,49 @@ public class TypesResource {
@Path("{typeName}")
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response getDefinition(@Context HttpServletRequest request, @PathParam("typeName") String typeName) {
+ TypesREST typesRest = resourceContext.getResource(TypesREST.class);
+ JSONObject response = new JSONObject();
AtlasPerfTracer perf = null;
+
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getDefinition(" + typeName + ")");
}
- final String typeDefinition = metadataService.getTypeDefinition(typeName);
+ TypeCategory typeCategory = typeRegistry.getType(typeName).getTypeCategory();
+ TypesDef typesDef = null;
- JSONObject response = new JSONObject();
+ if (typeCategory != null) {
+ switch (typeCategory) {
+ case ENUM:
+ AtlasEnumDef enumDef = typesRest.getEnumDefByName(typeName);
+ typesDef = RestUtils.toTypesDef(enumDef);
+ break;
+ case STRUCT:
+ AtlasStructDef structDef = typesRest.getStructDefByName(typeName);
+ typesDef = RestUtils.toTypesDef(structDef, typeRegistry);
+ break;
+ case ENTITY:
+ AtlasEntityDef entityDef = typesRest.getEntityDefByName(typeName);
+ typesDef = RestUtils.toTypesDef(entityDef, typeRegistry);
+ break;
+ case CLASSIFICATION:
+ AtlasClassificationDef classificationDef = typesRest.getClassificationDefByName(typeName);
+ typesDef = RestUtils.toTypesDef(classificationDef, typeRegistry);
+ break;
+ default:
+ typesDef = new TypesDef();
+ break;
+ }
+ }
+
+ final String typeDefinition = TypesSerialization.toJson(typesDef);
response.put(AtlasClient.TYPENAME, typeName);
response.put(AtlasClient.DEFINITION, new JSONObject(typeDefinition));
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
return Response.ok(response).build();
- } catch (AtlasException e) {
+ } catch (AtlasBaseException e) {
LOG.error("Unable to get type definition for type {}", typeName, e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.NOT_FOUND));
} catch (JSONException | IllegalArgumentException e) {
@@ -225,29 +268,26 @@ public class TypesResource {
@Produces(Servlets.JSON_MEDIA_TYPE)
public Response getTypesByFilter(@Context HttpServletRequest request, @QueryParam("type") String typeCategory,
@QueryParam("supertype") String supertype,
- @QueryParam("notsupertype") String notsupertype) {
+ @QueryParam("notsupertype") String notsupertype) throws AtlasBaseException {
+ TypesREST typesRest = resourceContext.getResource(TypesREST.class);
+ JSONObject response = new JSONObject();
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "TypesResource.getTypesByFilter(" + typeCategory + ")");
}
- Map<TypeCache.TYPE_FILTER, String> filterMap = new HashMap<>();
- addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.CATEGORY, typeCategory);
- addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.SUPERTYPE, supertype);
- addToFilterIfNotEmpty(filterMap, TypeCache.TYPE_FILTER.NOT_SUPERTYPE, notsupertype);
- List<String> result = metadataService.getTypeNames(filterMap);
+ List<String> result = RestUtils.getTypeNames(typesRest.getTypeDefHeaders());
- JSONObject response = new JSONObject();
response.put(AtlasClient.RESULTS, new JSONArray(result));
response.put(AtlasClient.COUNT, result.size());
response.put(AtlasClient.REQUEST_ID, Servlets.getRequestId());
return Response.ok(response).build();
- } catch (IllegalArgumentException | AtlasException ie) {
- LOG.error("Unsupported typeName while retrieving type list {}", typeCategory);
+ } catch (AtlasBaseException e) {
+ LOG.error("Given search filter did not yield any results");
throw new WebApplicationException(
- Servlets.getErrorResponse(new Exception("Unsupported type " + typeCategory, ie), Response.Status.BAD_REQUEST));
+ Servlets.getErrorResponse(new Exception("Given search filter did not yield any results "), Response.Status.BAD_REQUEST));
} catch (Throwable e) {
LOG.error("Unable to get types list", e);
throw new WebApplicationException(Servlets.getErrorResponse(e, Response.Status.INTERNAL_SERVER_ERROR));
@@ -255,11 +295,4 @@ public class TypesResource {
AtlasPerfTracer.log(perf);
}
}
-
- private void addToFilterIfNotEmpty(Map<TypeCache.TYPE_FILTER, String> filterMap, TypeCache.TYPE_FILTER filterType,
- String filterValue) {
- if (StringUtils.isNotEmpty(filterValue)) {
- filterMap.put(filterType, filterValue);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/95083cb0/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
index 7d03689..b6dae4d 100755
--- a/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
+++ b/webapp/src/test/java/org/apache/atlas/web/resources/TypesJerseyResourceIT.java
@@ -78,7 +78,7 @@ public class TypesJerseyResourceIT extends BaseResourceIT {
@Test
public void testSubmit() throws Exception {
for (HierarchicalTypeDefinition typeDefinition : typeDefinitions) {
- String typesAsJSON = TypesSerialization.toJson(typeDefinition);
+ String typesAsJSON = TypesSerialization.toJson(typeDefinition, false);
System.out.println("typesAsJSON = " + typesAsJSON);
WebResource resource = service.path("api/atlas/types");