You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2015/12/03 06:50:52 UTC
[2/2] incubator-atlas git commit: ATLAS-171 Ability to update type
definition(shwethags via sumasai)
ATLAS-171 Ability to update type definition(shwethags via sumasai)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/bf5672c5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/bf5672c5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/bf5672c5
Branch: refs/heads/master
Commit: bf5672c545ff2e076c0d1e0eac20526626478155
Parents: 919120f
Author: Suma Shivaprasad <su...@gmail.com>
Authored: Thu Dec 3 11:19:45 2015 +0530
Committer: Suma Shivaprasad <su...@gmail.com>
Committed: Thu Dec 3 11:19:45 2015 +0530
----------------------------------------------------------------------
.../hive/model/HiveDataModelGenerator.java | 4 +-
.../main/java/org/apache/atlas/AtlasClient.java | 62 +++++-
.../atlas/listener/TypesChangeListener.java | 4 +-
pom.xml | 1 +
release-log.txt | 1 +
.../graph/GraphBackedSearchIndexer.java | 5 +
.../typestore/GraphBackedTypeStore.java | 3 +-
.../atlas/services/DefaultMetadataService.java | 53 +++++-
.../apache/atlas/services/MetadataService.java | 8 +
.../apache/atlas/BaseHiveRepositoryTest.java | 6 +-
.../test/java/org/apache/atlas/TestUtils.java | 3 +-
.../typestore/GraphBackedTypeStoreTest.java | 64 ++++++-
.../typesystem/types/AbstractDataType.java | 10 +
.../atlas/typesystem/types/AttributeInfo.java | 37 ++++
.../apache/atlas/typesystem/types/EnumType.java | 20 ++
.../typesystem/types/HierarchicalType.java | 30 ++-
.../atlas/typesystem/types/IDataType.java | 3 +
.../atlas/typesystem/types/StructType.java | 24 ++-
.../atlas/typesystem/types/TypeSystem.java | 189 ++++++++++++-------
.../typesystem/types/TypeUpdateException.java | 39 ++++
.../atlas/typesystem/types/TypeUtils.java | 57 ++++--
.../atlas/typesystem/types/utils/TypesUtil.java | 9 +
.../apache/atlas/typesystem/types/BaseTest.java | 9 +-
.../atlas/typesystem/types/ClassTest.java | 30 ++-
.../apache/atlas/typesystem/types/EnumTest.java | 44 ++++-
.../typesystem/types/HierarchicalTypeTest.java | 84 +++++++++
.../atlas/typesystem/types/StructTest.java | 28 ++-
.../atlas/typesystem/types/TraitTest.java | 27 ++-
.../typesystem/types/TypeUpdateBaseTest.java | 98 ++++++++++
.../org/apache/atlas/examples/QuickStart.java | 3 +-
.../atlas/web/resources/TypesResource.java | 45 +++++
.../atlas/web/resources/BaseResourceIT.java | 3 +-
.../web/resources/EntityJerseyResourceIT.java | 36 +++-
.../MetadataDiscoveryJerseyResourceIT.java | 4 +-
.../web/resources/TypesJerseyResourceIT.java | 29 +++
35 files changed, 936 insertions(+), 136 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
index 2571295..994c813 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
@@ -34,7 +34,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,7 +102,7 @@ public class HiveDataModelGenerator {
}
public TypesDef getTypesDef() {
- return TypeUtils.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
+ return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
getClassTypeDefinitions());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/client/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java
index 7b6bcb7..2e8a268 100755
--- a/client/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasClient.java
@@ -25,7 +25,9 @@ import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
import org.apache.atlas.security.SecureClientUtils;
import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
+import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.security.UserGroupInformation;
@@ -132,6 +134,7 @@ public class AtlasClient {
//Type operations
CREATE_TYPE(BASE_URI + TYPES, HttpMethod.POST),
+ UPDATE_TYPE(BASE_URI + TYPES, HttpMethod.PUT),
GET_TYPE(BASE_URI + TYPES, HttpMethod.GET),
LIST_TYPES(BASE_URI + TYPES, HttpMethod.GET),
LIST_TRAIT_TYPES(BASE_URI + TYPES + "?type=trait", HttpMethod.GET),
@@ -181,13 +184,45 @@ public class AtlasClient {
* @return result json object
* @throws AtlasServiceException
*/
- public JSONObject createType(String typeAsJson) throws AtlasServiceException {
- return callAPI(API.CREATE_TYPE, typeAsJson);
+ public List<String> createType(String typeAsJson) throws AtlasServiceException {
+ JSONObject response = callAPI(API.CREATE_TYPE, typeAsJson);
+ return extractResults(response, AtlasClient.TYPES);
+ }
+
+ /**
+ * Register the given type(meta model)
+ * @param typeDef type definition
+ * @return result json object
+ * @throws AtlasServiceException
+ */
+ public List<String> createType(TypesDef typeDef) throws AtlasServiceException {
+ return createType(TypesSerialization.toJson(typeDef));
+ }
+
+ /**
+ * Register the given type(meta model)
+ * @param typeAsJson type definition a jaon
+ * @return result json object
+ * @throws AtlasServiceException
+ */
+ public List<String> updateType(String typeAsJson) throws AtlasServiceException {
+ JSONObject response = callAPI(API.UPDATE_TYPE, typeAsJson);
+ return extractResults(response, AtlasClient.TYPES);
+ }
+
+ /**
+ * Register the given type(meta model)
+ * @param typeDef type definition
+ * @return result json object
+ * @throws AtlasServiceException
+ */
+ public List<String> updateType(TypesDef typeDef) throws AtlasServiceException {
+ return updateType(TypesSerialization.toJson(typeDef));
}
public List<String> listTypes() throws AtlasServiceException {
final JSONObject jsonObject = callAPI(API.LIST_TYPES, null);
- return extractResults(jsonObject);
+ return extractResults(jsonObject, AtlasClient.RESULTS);
}
public String getType(String typeName) throws AtlasServiceException {
@@ -230,6 +265,14 @@ public class AtlasClient {
return createEntity(new JSONArray(Arrays.asList(entitiesAsJson)));
}
+ public JSONArray createEntity(Referenceable... entities) throws AtlasServiceException {
+ JSONArray entityArray = new JSONArray(entities.length);
+ for (Referenceable entity : entities) {
+ entityArray.put(InstanceSerialization.toJson(entity, true));
+ }
+ return createEntity(entityArray);
+ }
+
/**
* Get an entity given the entity id
* @param guid entity id
@@ -286,15 +329,20 @@ public class AtlasClient {
WebResource resource = getResource(API.LIST_ENTITIES);
resource = resource.queryParam(TYPE, entityType);
JSONObject jsonResponse = callAPIWithResource(API.LIST_ENTITIES, resource);
- return extractResults(jsonResponse);
+ return extractResults(jsonResponse, AtlasClient.RESULTS);
}
- private List<String> extractResults(JSONObject jsonResponse) throws AtlasServiceException {
+ private List<String> extractResults(JSONObject jsonResponse, String key) throws AtlasServiceException {
try {
- JSONArray results = jsonResponse.getJSONArray(AtlasClient.RESULTS);
+ JSONArray results = jsonResponse.getJSONArray(key);
ArrayList<String> resultsList = new ArrayList<>();
for (int index = 0; index < results.length(); index++) {
- resultsList.add(results.getString(index));
+ Object element = results.get(index);
+ if (element instanceof String) {
+ resultsList.add((String) element);
+ } else if (element instanceof JSONObject) {
+ resultsList.add(((JSONObject) element).getString(AtlasClient.NAME));
+ }
}
return resultsList;
} catch (JSONException e) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java b/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
index dee396a..5ff6d4a 100644
--- a/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
+++ b/common/src/main/java/org/apache/atlas/listener/TypesChangeListener.java
@@ -44,6 +44,6 @@ public interface TypesChangeListener {
*/
// void onRemove(String typeName) throws MetadataException;
- // This is upon updating an existing type to the store
- // void onChange() throws MetadataException;
+ //This is upon updating an existing type to the store
+ void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ede8c53..2f7ee82 100755
--- a/pom.xml
+++ b/pom.xml
@@ -1426,6 +1426,7 @@
<reuseForks>false</reuseForks>
<forkCount>1</forkCount>
<threadCount>5</threadCount>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
</configuration>
<executions>
<execution>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1c71dd1..9e3919d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-54 Rename configs in hive hook (shwethags)
ATLAS-3 Mixed Index creation fails with Date types (sumasai via shwethags)
ALL CHANGES:
+ATLAS-171 Ability to update type definition(shwethags via sumasai)
ATLAS-352 Improve write performance on type and entity creation with Hbase (sumasai)
ATLAS-350 Document jaas config details for atlas (tbeerbower via shwethags)
ATLAS-344 Document HBase permissions for secure cluster (tbeerbower via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 10babed..3ea5fde 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -161,6 +161,11 @@ public class GraphBackedSearchIndexer implements SearchIndexer {
commit();
}
+ @Override
+ public void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException {
+ onAdd(dataTypes);
+ }
+
private void addIndexForType(IDataType dataType) {
switch (dataType.getTypeCategory()) {
case PRIMITIVE:
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
index eed297f..9789be7 100755
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -45,6 +45,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -253,7 +254,7 @@ public class GraphBackedTypeStore implements ITypeStore {
throw new IllegalArgumentException("Unhandled type category " + typeCategory);
}
}
- return TypeUtils.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
+ return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
}
private EnumTypeDefinition getEnumType(Vertex vertex) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 7dfe165..db61de9 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -50,7 +50,6 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.codehaus.jettison.json.JSONArray;
@@ -148,8 +147,7 @@ public class DefaultMetadataService implements MetadataService {
private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException {
if (!typeSystem.isRegistered(type.typeName)) {
- TypesDef typesDef = TypeUtils
- .getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
ImmutableList.of(type));
createType(TypesSerialization.toJson(typesDef));
@@ -191,6 +189,34 @@ public class DefaultMetadataService implements MetadataService {
}
}
+ @Override
+ public JSONObject updateType(String typeDefinition) throws AtlasException {
+ ParamChecker.notEmpty(typeDefinition, "type definition cannot be empty");
+ TypesDef typesDef = validateTypeDefinition(typeDefinition);
+
+ try {
+ final Map<String, IDataType> typesAdded = typeSystem.updateTypes(typesDef);
+
+ try {
+ /* Create indexes first so that if index creation fails then we rollback
+ the typesystem and also do not persist the graph
+ */
+ onTypesUpdated(typesAdded);
+ typeStore.store(typeSystem, ImmutableList.copyOf(typesAdded.keySet()));
+ } catch (Throwable t) {
+ typeSystem.removeTypes(typesAdded.keySet());
+ throw new AtlasException("Unable to persist types ", t);
+ }
+
+ return new JSONObject() {{
+ put(AtlasClient.TYPES, typesAdded.keySet());
+ }};
+ } catch (JSONException e) {
+ LOG.error("Unable to create response for types={}", typeDefinition, e);
+ throw new AtlasException("Unable to create response ", e);
+ }
+ }
+
private TypesDef validateTypeDefinition(String typeDefinition) {
try {
TypesDef typesDef = TypesSerialization.fromJson(typeDefinition);
@@ -343,7 +369,7 @@ public class DefaultMetadataService implements MetadataService {
repository.updateEntity(guid, property, value);
- onEntityUpdated(repository.getEntityDefinition(guid), property, value);
+ onEntityUpdated(repository.getEntityDefinition(guid));
}
private void validateTypeExists(String entityType) throws AtlasException {
@@ -466,7 +492,24 @@ public class DefaultMetadataService implements MetadataService {
}
}
- private void onEntityUpdated(ITypedReferenceableInstance entity, String property, String value)
+ private void onTypesUpdated(Map<String, IDataType> typesUpdated) throws AtlasException {
+ Map<TypesChangeListener, Throwable> caughtExceptions = new HashMap<>();
+ for (Provider<TypesChangeListener> indexerProvider : typeChangeListeners) {
+ final TypesChangeListener listener = indexerProvider.get();
+ try {
+ listener.onChange(typesUpdated.values());
+ } catch (IndexCreationException ice) {
+ LOG.error("Index creation for listener {} failed ", indexerProvider, ice);
+ caughtExceptions.put(listener, ice);
+ }
+ }
+
+ if (caughtExceptions.size() > 0) {
+ throw new IndexCreationException("Index creation failed for types " + typesUpdated.keySet() + ". Aborting");
+ }
+ }
+
+ private void onEntityUpdated(ITypedReferenceableInstance entity)
throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntityUpdated(entity);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/main/java/org/apache/atlas/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/MetadataService.java b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
index d048cc6..f027b79 100755
--- a/repository/src/main/java/org/apache/atlas/services/MetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
@@ -40,6 +40,14 @@ public interface MetadataService {
JSONObject createType(String typeDefinition) throws AtlasException;
/**
+ * Updates the given types in the type definition
+ * @param typeDefinition
+ * @return
+ * @throws AtlasException
+ */
+ JSONObject updateType(String typeDefinition) throws AtlasException;
+
+ /**
* Return the definition for the given type.
*
* @param typeName name for this type, must be unique
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
index 6b715ac..f9378e4 100644
--- a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
@@ -28,7 +28,6 @@ import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
@@ -41,10 +40,7 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import javax.inject.Inject;
@@ -170,7 +166,7 @@ public class BaseHiveRepositoryTest {
HierarchicalTypeDefinition<TraitType> jdbcTraitDef = TypesUtil.createTraitTypeDef("JdbcAccess", null);
- return TypeUtils.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
ImmutableList.of(dimTraitDef, factTraitDef, piiTraitDef, metricTraitDef, etlTraitDef, jdbcTraitDef),
ImmutableList.of(dbClsDef, storageDescClsDef, columnClsDef, tblClsDef, loadProcessClsDef, viewClsDef, partClsDef));
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index 564dbfe..01a8158 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -38,7 +38,6 @@ import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.commons.lang.RandomStringUtils;
import org.testng.Assert;
@@ -272,7 +271,7 @@ public final class TestUtils {
createTraitTypeDef("fetl" + CLASSIFICATION, ImmutableList.of(CLASSIFICATION),
createRequiredAttrDef("tag", DataTypes.STRING_TYPE));
- return TypeUtils.getTypesDef(ImmutableList.of(enumTypeDefinition),
+ return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition),
ImmutableList.of(structTypeDefinition, partitionDefinition),
ImmutableList.of(classificationTypeDefinition, fetlClassificationTypeDefinition, piiTypeDefinition),
ImmutableList.of(superTypeDefinition, databaseTypeDefinition, columnsDefinition, tableTypeDefinition));
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
index 720c7dd..8402eed 100755
--- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
@@ -18,13 +18,13 @@
package org.apache.atlas.repository.typestore;
+import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.GraphTransaction;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.repository.graph.GraphHelper;
@@ -33,21 +33,30 @@ import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
import org.apache.atlas.typesystem.types.EnumValue;
import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
-import org.testng.Assert;
import javax.inject.Inject;
import java.util.List;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
+
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
@Inject
@@ -77,7 +86,6 @@ public class GraphBackedTypeStoreTest {
}
@Test
- @GraphTransaction
public void testStore() throws AtlasException {
typeStore.store(ts);
dumpGraph();
@@ -137,4 +145,54 @@ public class GraphBackedTypeStoreTest {
ts.reset();
ts.defineTypes(types);
}
+
+ @Test(dependsOnMethods = "testStore")
+ public void testTypeUpdate() throws Exception {
+ //Add enum value
+ EnumTypeDefinition orgLevelEnum = new EnumTypeDefinition("OrgLevel", new EnumValue("L1", 1),
+ new EnumValue("L2", 2), new EnumValue("L3", 3));
+
+ //Add attribute
+ StructTypeDefinition addressDetails =
+ createStructTypeDef("Address", createRequiredAttrDef("street", DataTypes.STRING_TYPE),
+ createRequiredAttrDef("city", DataTypes.STRING_TYPE),
+ createOptionalAttrDef("state", DataTypes.STRING_TYPE));
+
+ //Add supertype
+ HierarchicalTypeDefinition<ClassType> superTypeDef = createClassTypeDef("Division", ImmutableList.<String>of(),
+ createOptionalAttrDef("dname", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> deptTypeDef = createClassTypeDef("Department",
+ ImmutableList.of(superTypeDef.typeName), createRequiredAttrDef("name", DataTypes.STRING_TYPE),
+ new AttributeDefinition("employees", String.format("array<%s>", "Person"), Multiplicity.COLLECTION,
+ true, "department"));
+ TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.of(orgLevelEnum), ImmutableList.of(addressDetails),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(deptTypeDef, superTypeDef));
+
+ ts.updateTypes(typesDef);
+ typeStore.store(ts, ImmutableList.of(orgLevelEnum.name, addressDetails.typeName, superTypeDef.typeName,
+ deptTypeDef.typeName));
+
+ //Validate the updated types
+ TypesDef types = typeStore.restore();
+ ts.reset();
+ ts.defineTypes(types);
+
+ //Assert new enum value
+ EnumType orgLevel = ts.getDataType(EnumType.class, orgLevelEnum.name);
+ Assert.assertEquals(orgLevel.name, orgLevelEnum.name);
+ Assert.assertEquals(orgLevel.values().size(), orgLevelEnum.enumValues.length);
+ Assert.assertEquals(orgLevel.fromValue("L3").ordinal, 3);
+
+ //Assert new attribute
+ StructType addressType = ts.getDataType(StructType.class, addressDetails.typeName);
+ Assert.assertEquals(addressType.numFields, 3);
+ Assert.assertEquals(addressType.fieldMapping.fields.get("state").dataType(), DataTypes.STRING_TYPE);
+
+ //Assert new super type
+ ClassType deptType = ts.getDataType(ClassType.class, deptTypeDef.typeName);
+ Assert.assertTrue(deptType.superTypes.contains(superTypeDef.typeName));
+ Assert.assertNotNull(ts.getDataType(ClassType.class, superTypeDef.typeName));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
index f2d15eb..ed0400a 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AbstractDataType.java
@@ -40,7 +40,17 @@ abstract class AbstractDataType<T> implements IDataType<T> {
} else {
TypeUtils.outputVal(val == null ? "<null>" : val.toString(), buf, prefix);
}
+ }
+ /**
+ * Validate that current definition can be updated with the new definition
+ * @param newType
+ */
+ @Override
+ public void validateUpdate(IDataType newType) throws TypeUpdateException {
+ if (!getName().equals(newType.getName()) || !getClass().getName().equals(newType.getClass().getName())) {
+ throw new TypeUpdateException(newType);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
index 3e1ef18..330d1cb 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/AttributeInfo.java
@@ -71,6 +71,43 @@ public class AttributeInfo {
'}';
}
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ AttributeInfo that = (AttributeInfo) o;
+
+ if (isComposite != that.isComposite) {
+ return false;
+ }
+ if (isUnique != that.isUnique) {
+ return false;
+ }
+ if (isIndexable != that.isIndexable) {
+ return false;
+ }
+ if (!dataType.getName().equals(that.dataType.getName())) {
+ return false;
+ }
+ if (!multiplicity.equals(that.multiplicity)) {
+ return false;
+ }
+ if (!name.equals(that.name)) {
+ return false;
+ }
+ if (reverseAttributeName != null ? !reverseAttributeName.equals(that.reverseAttributeName) :
+ that.reverseAttributeName != null) {
+ return false;
+ }
+
+ return true;
+ }
+
public String toJson() throws JSONException {
JSONObject json = new JSONObject();
json.put("name", name);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
index f6927da..1439303 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/EnumType.java
@@ -76,6 +76,26 @@ public class EnumType extends AbstractDataType<EnumValue> {
return DataTypes.TypeCategory.ENUM;
}
+ @Override
+ public void validateUpdate(IDataType newType) throws TypeUpdateException {
+ super.validateUpdate(newType);
+
+ EnumType enumType = (EnumType)newType;
+ for (EnumValue enumValue : values()) {
+ //The old enum value should be part of new enum definition as well
+ if (!enumType.valueMap.containsKey(enumValue.value)) {
+ throw new TypeUpdateException("Value " + enumValue.value + " is missing in new type");
+ }
+
+ //The ordinal for old enum value can't change
+ EnumValue newEnumValue = enumType.valueMap.get(enumValue.value);
+ if (enumValue.ordinal != newEnumValue.ordinal) {
+ throw new TypeUpdateException(String.format("Ordinal mismatch %s(%s) != %s(%s)", enumValue.value,
+ enumValue.ordinal, newEnumValue.value, newEnumValue.ordinal));
+ }
+ }
+ }
+
public EnumValue fromOrdinal(int o) {
return ordinalMap.get(o);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
index 5bb04c6..5a331a6 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/HierarchicalType.java
@@ -105,6 +105,30 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
return (cType == this || cType.superTypePaths.containsKey(getName()));
}
+ /**
+ * Validate that current definition can be updated with the new definition
+ * @param newType
+ * @return true if the current definition can be updated with the new definition, else false
+ */
+ @Override
+ public void validateUpdate(IDataType newType) throws TypeUpdateException {
+ super.validateUpdate(newType);
+
+ HierarchicalType newHierarchicalType = (HierarchicalType) newType;
+
+ //validate on supertypes
+ if (!newHierarchicalType.superTypes.containsAll(superTypes)) {
+ throw new TypeUpdateException(newType, "New type doesn't contain all super types of old type");
+ }
+
+ //validate on fields
+ try {
+ TypeUtils.validateUpdate(fieldMapping, newHierarchicalType.fieldMapping);
+ } catch (TypeUpdateException e) {
+ throw new TypeUpdateException(newType, e);
+ }
+ }
+
protected void setupSuperTypesGraph() throws AtlasException {
setupSuperTypesGraph(superTypes);
}
@@ -147,9 +171,9 @@ public abstract class HierarchicalType<ST extends HierarchicalType, T> extends A
protected Pair<FieldMapping, ImmutableMap<String, String>> constructFieldMapping(ImmutableList<String> superTypes,
AttributeInfo... fields) throws AtlasException {
- Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
- Map<String, Integer> fieldPos = new HashMap<String, Integer>();
- Map<String, Integer> fieldNullPos = new HashMap<String, Integer>();
+ Map<String, AttributeInfo> fieldsMap = new LinkedHashMap();
+ Map<String, Integer> fieldPos = new HashMap();
+ Map<String, Integer> fieldNullPos = new HashMap();
Map<String, String> attributeNameToType = new HashMap<>();
int numBools = 0;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
index 7cb2620..d9b1b34 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/IDataType.java
@@ -28,4 +28,7 @@ public interface IDataType<T> {
DataTypes.TypeCategory getTypeCategory();
void output(T val, Appendable buf, String prefix) throws AtlasException;
+
+ void validateUpdate(IDataType newType) throws TypeUpdateException;
}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
index 448827e..1a40484 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/StructType.java
@@ -18,7 +18,6 @@
package org.apache.atlas.typesystem.types;
-import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
@@ -49,11 +48,11 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
this.handler = null;
}
- protected StructType(TypeSystem typeSystem, String name, ImmutableList<String> superTypes, AttributeInfo... fields)
+ protected StructType(TypeSystem typeSystem, String name, AttributeInfo... fields)
throws AtlasException {
this.typeSystem = typeSystem;
this.name = name;
- this.fieldMapping = constructFieldMapping(superTypes, fields);
+ this.fieldMapping = constructFieldMapping(fields);
infoToNameMap = TypeUtils.buildAttrInfoToNameMap(this.fieldMapping);
this.numFields = this.fieldMapping.fields.size();
this.handler = new TypedStructHandler(this);
@@ -68,7 +67,24 @@ public class StructType extends AbstractDataType<IStruct> implements IConstructa
return name;
}
- protected FieldMapping constructFieldMapping(ImmutableList<String> superTypes, AttributeInfo... fields)
+ /**
+ * Validate that current definition can be updated with the new definition
+ * @param newType
+ * @return true if the current definition can be updated with the new definition, else false
+ */
+ @Override
+ public void validateUpdate(IDataType newType) throws TypeUpdateException {
+ super.validateUpdate(newType);
+
+ StructType newStructType = (StructType) newType;
+ try {
+ TypeUtils.validateUpdate(fieldMapping, newStructType.fieldMapping);
+ } catch (TypeUpdateException e) {
+ throw new TypeUpdateException(newType, e);
+ }
+ }
+
+ protected FieldMapping constructFieldMapping(AttributeInfo... fields)
throws AtlasException {
Map<String, AttributeInfo> fieldsMap = new LinkedHashMap<String, AttributeInfo>();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
index a6367b1..d9b1edb 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
@@ -35,7 +35,6 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -193,7 +192,7 @@ public class TypeSystem {
infos[i] = new AttributeInfo(this, attrDefs[i], tempTypes);
}
- return new StructType(this, name, null, infos);
+ return new StructType(this, name, infos);
}
public TraitType defineTraitType(HierarchicalTypeDefinition<TraitType> traitDef) throws AtlasException {
@@ -225,6 +224,18 @@ public class TypeSystem {
return transientTypes.defineTypes();
}
+ public Map<String, IDataType> updateTypes(TypesDef typesDef) throws AtlasException {
+ ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList());
+ ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList());
+ ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs =
+ ImmutableList.copyOf(typesDef.traitTypesAsJavaList());
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs =
+ ImmutableList.copyOf(typesDef.classTypesAsJavaList());
+
+ TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs);
+ return transientTypes.defineTypes(true);
+ }
+
public Map<String, IDataType> defineTypes(TypesDef typesDef) throws AtlasException {
ImmutableList<EnumTypeDefinition> enumDefs = ImmutableList.copyOf(typesDef.enumTypesAsJavaList());
ImmutableList<StructTypeDefinition> structDefs = ImmutableList.copyOf(typesDef.structTypesAsJavaList());
@@ -241,17 +252,12 @@ public class TypeSystem {
ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs,
ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs) throws AtlasException {
TransientTypeSystem transientTypes = new TransientTypeSystem(enumDefs, structDefs, traitDefs, classDefs);
- Map<String, IDataType> definedTypes = transientTypes.defineTypes();
- // LOG.debug("Defined new types " + Arrays.toString(definedTypes.keySet().toArray(new
- // String[definedTypes.size()])));
- return definedTypes;
+ return transientTypes.defineTypes();
}
public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException {
assert elemType != null;
DataTypes.ArrayType dT = new DataTypes.ArrayType(elemType);
- // types.put(dT.getName(), dT);
- // typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ARRAY, dT.getName());
return dT;
}
@@ -259,8 +265,6 @@ public class TypeSystem {
assert keyType != null;
assert valueType != null;
DataTypes.MapType dT = new DataTypes.MapType(keyType, valueType);
- // types.put(dT.getName(), dT);
- // typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.MAP, dT.getName());
return dT;
}
@@ -303,15 +307,16 @@ public class TypeSystem {
final ImmutableList<HierarchicalTypeDefinition<TraitType>> traitDefs;
final ImmutableList<HierarchicalTypeDefinition<ClassType>> classDefs;
private final ImmutableList<EnumTypeDefinition> enumDefs;
+
Map<String, StructTypeDefinition> structNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<TraitType>> traitNameToDefMap = new HashMap<>();
Map<String, HierarchicalTypeDefinition<ClassType>> classNameToDefMap = new HashMap<>();
- Set<String> transientTypes;
+ Map<String, IDataType> transientTypes = null;
- List<AttributeInfo> recursiveRefs;
- List<DataTypes.ArrayType> recursiveArrayTypes;
- List<DataTypes.MapType> recursiveMapTypes;
+ List<AttributeInfo> recursiveRefs = new ArrayList<>();
+ List<DataTypes.ArrayType> recursiveArrayTypes = new ArrayList<>();
+ List<DataTypes.MapType> recursiveMapTypes = new ArrayList<>();
TransientTypeSystem(ImmutableList<EnumTypeDefinition> enumDefs, ImmutableList<StructTypeDefinition> structDefs,
@@ -321,17 +326,13 @@ public class TypeSystem {
this.structDefs = structDefs;
this.traitDefs = traitDefs;
this.classDefs = classDefs;
- structNameToDefMap = new HashMap<>();
- traitNameToDefMap = new HashMap<>();
- classNameToDefMap = new HashMap<>();
-
- recursiveRefs = new ArrayList<>();
- recursiveArrayTypes = new ArrayList<>();
- recursiveMapTypes = new ArrayList<>();
- transientTypes = new LinkedHashSet<>();
+ transientTypes = new HashMap<>();
}
- private IDataType dataType(String name) {
+ private IDataType dataType(String name) throws AtlasException {
+ if (transientTypes.containsKey(name)) {
+ return transientTypes.get(name);
+ }
return TypeSystem.this.types.get(name);
}
@@ -340,52 +341,50 @@ public class TypeSystem {
* - validate cannot redefine types
* - setup shallow Type instances to facilitate recursive type graphs
*/
- private void step1() throws AtlasException {
+ private void step1(boolean update) throws AtlasException {
for (EnumTypeDefinition eDef : enumDefs) {
assert eDef.name != null;
- if (types.containsKey(eDef.name)) {
+ if (!update && (transientTypes.containsKey(eDef.name) || types.containsKey(eDef.name))) {
throw new AtlasException(String.format("Redefinition of type %s not supported", eDef.name));
}
EnumType eT = new EnumType(this, eDef.name, eDef.enumValues);
- TypeSystem.this.types.put(eDef.name, eT);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.ENUM, eDef.name);
- transientTypes.add(eDef.name);
+ transientTypes.put(eDef.name, eT);
}
for (StructTypeDefinition sDef : structDefs) {
assert sDef.typeName != null;
- if (dataType(sDef.typeName) != null) {
+ if (!update && (transientTypes.containsKey(sDef.typeName) || types.containsKey(sDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", sDef.typeName));
}
- TypeSystem.this.types
- .put(sDef.typeName, new StructType(this, sDef.typeName, sDef.attributeDefinitions.length));
+ StructType sT = new StructType(this, sDef.typeName, sDef.attributeDefinitions.length);
structNameToDefMap.put(sDef.typeName, sDef);
- transientTypes.add(sDef.typeName);
+ transientTypes.put(sDef.typeName, sT);
}
for (HierarchicalTypeDefinition<TraitType> traitDef : traitDefs) {
assert traitDef.typeName != null;
- if (types.containsKey(traitDef.typeName)) {
+ if (!update &&
+ (transientTypes.containsKey(traitDef.typeName) || types.containsKey(traitDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", traitDef.typeName));
}
-
- TypeSystem.this.types.put(traitDef.typeName, new TraitType(this, traitDef.typeName, traitDef.superTypes,
- traitDef.attributeDefinitions.length));
+ TraitType tT = new TraitType(this, traitDef.typeName, traitDef.superTypes,
+ traitDef.attributeDefinitions.length);
traitNameToDefMap.put(traitDef.typeName, traitDef);
- transientTypes.add(traitDef.typeName);
+ transientTypes.put(traitDef.typeName, tT);
}
for (HierarchicalTypeDefinition<ClassType> classDef : classDefs) {
assert classDef.typeName != null;
- if (types.containsKey(classDef.typeName)) {
+ if (!update &&
+ (transientTypes.containsKey(classDef.typeName) || types.containsKey(classDef.typeName))) {
throw new TypeExistsException(String.format("Cannot redefine type %s", classDef.typeName));
}
- TypeSystem.this.types.put(classDef.typeName, new ClassType(this, classDef.typeName, classDef.superTypes,
- classDef.attributeDefinitions.length));
+ ClassType cT = new ClassType(this, classDef.typeName, classDef.superTypes,
+ classDef.attributeDefinitions.length);
classNameToDefMap.put(classDef.typeName, classDef);
- transientTypes.add(classDef.typeName);
+ transientTypes.put(classDef.typeName, cT);
}
}
@@ -438,20 +437,20 @@ public class TypeSystem {
private AttributeInfo constructAttributeInfo(AttributeDefinition attrDef) throws AtlasException {
AttributeInfo info = new AttributeInfo(this, attrDef, null);
- if (transientTypes.contains(attrDef.dataTypeName)) {
+ if (transientTypes.keySet().contains(attrDef.dataTypeName)) {
recursiveRefs.add(info);
}
if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.ARRAY) {
DataTypes.ArrayType arrType = (DataTypes.ArrayType) info.dataType();
- if (transientTypes.contains(arrType.getElemType().getName())) {
+ if (transientTypes.keySet().contains(arrType.getElemType().getName())) {
recursiveArrayTypes.add(arrType);
}
}
if (info.dataType().getTypeCategory() == DataTypes.TypeCategory.MAP) {
DataTypes.MapType mapType = (DataTypes.MapType) info.dataType();
- if (transientTypes.contains(mapType.getKeyType().getName())) {
+ if (transientTypes.keySet().contains(mapType.getKeyType().getName())) {
recursiveMapTypes.add(mapType);
- } else if (transientTypes.contains(mapType.getValueType().getName())) {
+ } else if (transientTypes.keySet().contains(mapType.getValueType().getName())) {
recursiveMapTypes.add(mapType);
}
}
@@ -472,8 +471,8 @@ public class TypeSystem {
infos[i] = constructAttributeInfo(def.attributeDefinitions[i]);
}
- StructType type = new StructType(TypeSystem.this, def.typeName, null, infos);
- TypeSystem.this.types.put(def.typeName, type);
+ StructType type = new StructType(this, def.typeName, infos);
+ transientTypes.put(def.typeName, type);
return type;
}
@@ -487,11 +486,12 @@ public class TypeSystem {
try {
Constructor<U> cons = cls.getDeclaredConstructor(TypeSystem.class, String.class, ImmutableList.class,
AttributeInfo[].class);
- U type = cons.newInstance(TypeSystem.this, def.typeName, def.superTypes, infos);
- TypeSystem.this.types.put(def.typeName, type);
+ U type = cons.newInstance(this, def.typeName, def.superTypes, infos);
+ transientTypes.put(def.typeName, type);
return type;
} catch (Exception e) {
- throw new AtlasException(String.format("Cannot construct Type of MetaType %s", cls.getName()), e);
+ e.printStackTrace();
+ throw new AtlasException(String.format("Cannot construct Type of MetaType %s - %s", cls.getName(), def.typeName), e);
}
}
@@ -516,17 +516,14 @@ public class TypeSystem {
for (StructTypeDefinition structDef : structDefs) {
constructStructureType(structDef);
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.STRUCT, structDef.typeName);
}
for (TraitType traitType : traitTypes) {
constructHierarchicalType(TraitType.class, traitNameToDefMap.get(traitType.getName()));
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.TRAIT, traitType.getName());
}
for (ClassType classType : classTypes) {
constructHierarchicalType(ClassType.class, classNameToDefMap.get(classType.getName()));
- typeCategoriesToTypeNamesMap.put(DataTypes.TypeCategory.CLASS, classType.getName());
}
}
@@ -547,35 +544,89 @@ public class TypeSystem {
}
}
+ /**
+ * Step 5:
+ * - Validate that the update can be done
+ */
+ private void step5() throws TypeUpdateException {
+ //If the type is modified, validate that update can be done
+ for (IDataType newType : transientTypes.values()) {
+ if (TypeSystem.this.types.containsKey(newType.getName())) {
+ IDataType oldType = TypeSystem.this.types.get(newType.getName());
+ oldType.validateUpdate(newType);
+ }
+ }
+ }
+
Map<String, IDataType> defineTypes() throws AtlasException {
- try {
- step1();
- step2();
+ return defineTypes(false);
+ }
- step3();
- step4();
- } catch (AtlasException me) {
- for (String sT : transientTypes) {
- types.remove(sT);
- }
- throw me;
+ Map<String, IDataType> defineTypes(boolean update) throws AtlasException {
+ step1(update);
+ step2();
+
+ step3();
+ step4();
+
+ if (update) {
+ step5();
}
Map<String, IDataType> newTypes = new HashMap<>();
- for (String tName : transientTypes) {
- newTypes.put(tName, dataType(tName));
+ for (Map.Entry<String, IDataType> typeEntry : transientTypes.entrySet()) {
+ String typeName = typeEntry.getKey();
+ IDataType type = typeEntry.getValue();
+
+ //Add/replace the new type in the typesystem
+ TypeSystem.this.types.put(typeName, type);
+ typeCategoriesToTypeNamesMap.put(type.getTypeCategory(), typeName);
+
+ newTypes.put(typeName, type);
}
return newTypes;
}
@Override
public ImmutableList<String> getTypeNames() {
- return TypeSystem.this.getTypeNames();
+ Set<String> typeNames = transientTypes.keySet();
+ typeNames.addAll(TypeSystem.this.getTypeNames());
+ return ImmutableList.copyOf(typeNames);
}
+ //get from transient types. Else, from main type system
@Override
public <T> T getDataType(Class<T> cls, String name) throws AtlasException {
+ if (transientTypes != null) {
+ if (transientTypes.containsKey(name)) {
+ try {
+ return cls.cast(transientTypes.get(name));
+ } catch (ClassCastException cce) {
+ throw new AtlasException(cce);
+ }
+ }
+
+ /*
+ * is this an Array Type?
+ */
+ String arrElemType = TypeUtils.parseAsArrayType(name);
+ if (arrElemType != null) {
+ IDataType dT = defineArrayType(getDataType(IDataType.class, arrElemType));
+ return cls.cast(dT);
+ }
+
+ /*
+ * is this a Map Type?
+ */
+ String[] mapType = TypeUtils.parseAsMapType(name);
+ if (mapType != null) {
+ IDataType dT =
+ defineMapType(getDataType(IDataType.class, mapType[0]), getDataType(IDataType.class, mapType[1]));
+ return cls.cast(dT);
+ }
+ }
+
return TypeSystem.this.getDataType(cls, name);
}
@@ -605,12 +656,12 @@ public class TypeSystem {
@Override
public DataTypes.ArrayType defineArrayType(IDataType elemType) throws AtlasException {
- throw new AtlasException("Internal Error: define type called on TrasientTypeSystem");
+ return super.defineArrayType(elemType);
}
@Override
public DataTypes.MapType defineMapType(IDataType keyType, IDataType valueType) throws AtlasException {
- throw new AtlasException("Internal Error: define type called on TrasientTypeSystem");
+ return super.defineMapType(keyType, valueType);
}
}
@@ -631,7 +682,7 @@ public class TypeSystem {
infos[0] = new AttributeInfo(TypeSystem.this, idAttr, null);
infos[1] = new AttributeInfo(TypeSystem.this, typNmAttr, null);
- StructType type = new StructType(TypeSystem.this, TYP_NAME, null, infos);
+ StructType type = new StructType(TypeSystem.this, TYP_NAME, infos);
TypeSystem.this.types.put(TYP_NAME, type);
} catch (AtlasException me) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java
new file mode 100644
index 0000000..33d1cb5
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUpdateException.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import org.apache.atlas.AtlasException;
+
+public class TypeUpdateException extends AtlasException {
+ public TypeUpdateException(IDataType newType) {
+ super(newType.getName() + " can't be updated");
+ }
+
+ public TypeUpdateException(IDataType newType, Exception e) {
+ super(newType.getName() + " can't be updated - " + e.getMessage(), e);
+ }
+
+ public TypeUpdateException(String message) {
+ super(message);
+ }
+
+ public TypeUpdateException(IDataType newType, String message) {
+ super(newType.getName() + " can't be updated - " + message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
index 7017a7e..9d2480b 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeUtils.java
@@ -19,16 +19,15 @@
package org.apache.atlas.typesystem.types;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.TypesDef;
-import scala.collection.JavaConversions;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -59,23 +58,16 @@ public class TypeUtils {
}
public static Map<AttributeInfo, List<String>> buildAttrInfoToNameMap(FieldMapping f) {
- Map<AttributeInfo, List<String>> b = new HashMap<AttributeInfo, List<String>>();
+ Map<AttributeInfo, List<String>> b = new HashMap();
for (Map.Entry<String, AttributeInfo> e : f.fields.entrySet()) {
List<String> names = b.get(e.getValue());
if (names == null) {
- names = new ArrayList<String>();
+ names = new ArrayList<>();
b.put(e.getValue(), names);
}
names.add(e.getKey());
}
- return ImmutableMap.copyOf(b);
- }
-
- public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums,
- ImmutableList<StructTypeDefinition> structs, ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
- ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
- return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
- JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
+ return b;
}
protected static class Pair<L, R> {
@@ -87,4 +79,43 @@ public class TypeUtils {
this.right = right;
}
}
+
+ /**
+ * Validates that the old field mapping can be replaced with new field mapping
+ * @param oldFieldMapping
+ * @param newFieldMapping
+ */
+ public static void validateUpdate(FieldMapping oldFieldMapping, FieldMapping newFieldMapping)
+ throws TypeUpdateException {
+ Map<String, AttributeInfo> newFields = newFieldMapping.fields;
+ for (AttributeInfo attribute : oldFieldMapping.fields.values()) {
+ if (newFields.containsKey(attribute.name)) {
+ AttributeInfo newAttribute = newFields.get(attribute.name);
+ //If old attribute is also in new definition, only allowed change is multiplicity change from REQUIRED to OPTIONAL
+ if (!newAttribute.equals(attribute)) {
+ if (attribute.multiplicity == Multiplicity.REQUIRED
+ && newAttribute.multiplicity == Multiplicity.OPTIONAL) {
+ continue;
+ } else {
+ throw new TypeUpdateException("Attribute " + attribute.name + " can't be updated");
+ }
+ }
+
+ } else {
+ //If old attribute is missing in new definition, return false as attributes can't be deleted
+ throw new TypeUpdateException("Old Attribute " + attribute.name + " is missing");
+ }
+ }
+
+ //Only new attributes
+ Set<String> newAttributes = new HashSet<>(ImmutableList.copyOf(newFields.keySet()));
+ newAttributes.removeAll(oldFieldMapping.fields.keySet());
+ for (String attributeName : newAttributes) {
+ AttributeInfo newAttribute = newFields.get(attributeName);
+ //New required attribute can't be added
+ if (newAttribute.multiplicity == Multiplicity.REQUIRED) {
+ throw new TypeUpdateException("Can't add required attribute " + attributeName);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
index 69956a3..ee82ce5 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/utils/TypesUtil.java
@@ -19,6 +19,7 @@
package org.apache.atlas.typesystem.types.utils;
import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.EnumTypeDefinition;
@@ -28,6 +29,7 @@ import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
+import scala.collection.JavaConversions;
/**
* Types utilities class.
@@ -74,4 +76,11 @@ public class TypesUtil {
ImmutableList<String> superTypes, AttributeDefinition... attrDefs) {
return new HierarchicalTypeDefinition<>(ClassType.class, name, superTypes, attrDefs);
}
+
+ public static TypesDef getTypesDef(ImmutableList<EnumTypeDefinition> enums,
+ ImmutableList<StructTypeDefinition> structs, ImmutableList<HierarchicalTypeDefinition<TraitType>> traits,
+ ImmutableList<HierarchicalTypeDefinition<ClassType>> classes) {
+ return new TypesDef(JavaConversions.asScalaBuffer(enums), JavaConversions.asScalaBuffer(structs),
+ JavaConversions.asScalaBuffer(traits), JavaConversions.asScalaBuffer(classes));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
index a9fce04..93dd099 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/BaseTest.java
@@ -25,6 +25,7 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang.RandomStringUtils;
import org.testng.annotations.BeforeMethod;
import java.math.BigDecimal;
@@ -96,8 +97,8 @@ public abstract class BaseTest {
System.out.println("defined recursiveStructType = " + recursiveStructType);
}
- protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition... tDefs) throws AtlasException {
-
+ protected Map<String, IDataType> defineTraits(HierarchicalTypeDefinition<TraitType>... tDefs)
+ throws AtlasException {
return getTypeSystem().defineTraitTypes(tDefs);
}
@@ -168,4 +169,8 @@ public abstract class BaseTest {
return hrDept;
}
+
+ protected String newName() {
+ return RandomStringUtils.randomAlphanumeric(10);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
index d394337..f454641 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/ClassTest.java
@@ -18,14 +18,17 @@
package org.apache.atlas.typesystem.types;
+import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
-import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-public class ClassTest extends BaseTest {
+public class ClassTest extends HierarchicalTypeTest<ClassType> {
@BeforeMethod
public void setup() throws Exception {
@@ -67,6 +70,29 @@ public class ClassTest extends BaseTest {
"\t\tlevel : \t\t1\n" +
"\t}}]\n" +
"}");
+ }
+
+ @Override
+ protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, AttributeDefinition... attributes) {
+ return new HierarchicalTypeDefinition(ClassType.class, name, null, attributes);
+ }
+
+ @Override
+ protected HierarchicalTypeDefinition<ClassType> getTypeDefinition(String name, ImmutableList<String> superTypes,
+ AttributeDefinition... attributes) {
+ return new HierarchicalTypeDefinition(ClassType.class, name, superTypes, attributes);
+ }
+
+ @Override
+ protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of((HierarchicalTypeDefinition<ClassType>) typeDefinition));
+ }
+ @Override
+ protected TypesDef getTypesDef(HierarchicalTypeDefinition<ClassType>... typeDefinitions) {
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.copyOf(typeDefinitions));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
index 2735f3c..ea7a798 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/EnumTest.java
@@ -27,9 +27,11 @@ import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
-import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
import java.math.BigDecimal;
import java.math.BigInteger;
@@ -59,7 +61,47 @@ public class EnumTest extends BaseTest {
ts.defineEnumType("LockLevel", new EnumValue("DB", 1), new EnumValue("TABLE", 2),
new EnumValue("PARTITION", 3));
+ }
+
+ @Test
+ public void testTypeUpdate() throws Exception {
+ TypeSystem ts = getTypeSystem();
+ EnumTypeDefinition etd = new EnumTypeDefinition(newName(), new EnumValue("A", 1));
+ TypesDef typesDef = getTypesDef(etd);
+ ts.defineTypes(typesDef);
+
+ //Allow adding new enum
+ etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 1), new EnumValue("B", 2));
+ typesDef = getTypesDef(etd);
+ ts.updateTypes(typesDef);
+
+ //Don't allow deleting enum
+ etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 1));
+ typesDef = getTypesDef(etd);
+ try {
+ ts.updateTypes(typesDef);
+ Assert.fail("Expected TypeUpdateException");
+ } catch (TypeUpdateException e) {
+ //assert that type is not updated when validation fails
+ EnumType enumType = ts.getDataType(EnumType.class, etd.name);
+ Assert.assertEquals(enumType.values().size(), 2);
+ }
+
+ //Don't allow changing ordinal of existing enum value
+ etd = new EnumTypeDefinition(etd.name, new EnumValue("A", 2));
+ typesDef = getTypesDef(etd);
+ try {
+ ts.updateTypes(typesDef);
+ Assert.fail("Expected TypeUpdateException");
+ } catch (TypeUpdateException e) {
+ //expected
+ }
+ }
+ private TypesDef getTypesDef(EnumTypeDefinition enumTypeDefinition) {
+ return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
}
protected void fillStruct(Struct s) throws AtlasException {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeTest.java
new file mode 100644
index 0000000..4e83492
--- /dev/null
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/HierarchicalTypeTest.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types;
+
+import com.google.common.collect.ImmutableList;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public abstract class HierarchicalTypeTest<T extends HierarchicalType> extends TypeUpdateBaseTest {
+ @Test(enabled = false)
+ public void testTypeUpdate() throws Exception {
+ testTypeUpdateForAttributes();
+
+ //Test super types
+ HierarchicalTypeDefinition classType =
+ getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
+ TypeSystem ts = getTypeSystem();
+ ts.defineTypes(getTypesDef(classType));
+
+ //Add super type with optional attribute
+ HierarchicalTypeDefinition superType =
+ getTypeDefinition(newName(), TypesUtil.createOptionalAttrDef("s", DataTypes.INT_TYPE));
+ classType = getTypeDefinition(classType.typeName, ImmutableList.of(superType.typeName),
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
+ ts.updateTypes(getTypesDef(superType, classType));
+
+
+ //Add super type with required attribute should fail
+ HierarchicalTypeDefinition superTypeRequired =
+ getTypeDefinition(newName(), TypesUtil.createRequiredAttrDef("s", DataTypes.INT_TYPE));
+ classType = getTypeDefinition(classType.typeName,
+ ImmutableList.of(superTypeRequired.typeName, superType.typeName),
+ TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
+ try {
+ ts.updateTypes(getTypesDef(superTypeRequired, classType));
+ Assert.fail("Expected TypeUpdateException");
+ } catch (TypeUpdateException e) {
+ //expected
+ }
+
+ //Deleting super type should fail
+ classType = getTypeDefinition(classType.typeName, TypesUtil.createRequiredAttrDef("a", DataTypes.INT_TYPE));
+ try {
+ ts.updateTypes(getTypesDef(superType, classType));
+ Assert.fail("Expected TypeUpdateException");
+ } catch (TypeUpdateException e) {
+ //expected
+ }
+ }
+
+ @Override
+ protected abstract HierarchicalTypeDefinition<T> getTypeDefinition(String name, AttributeDefinition... attributes);
+
+ protected abstract HierarchicalTypeDefinition<T> getTypeDefinition(String name, ImmutableList<String> superTypes,
+ AttributeDefinition... attributes);
+
+ @Override
+ protected abstract TypesDef getTypesDef(StructTypeDefinition typeDefinition);
+
+ protected abstract TypesDef getTypesDef(HierarchicalTypeDefinition<T>... typeDefinitions);
+
+ @Override
+ protected int getNumberOfFields(TypeSystem ts, String typeName) throws Exception {
+ return ts.getDataType(HierarchicalType.class, typeName).numFields;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
index 17ce4ea..b49e5ce 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/StructTest.java
@@ -18,14 +18,17 @@
package org.apache.atlas.typesystem.types;
+import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
-import org.testng.annotations.Test;
import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
-public class StructTest extends BaseTest {
+public class StructTest extends TypeUpdateBaseTest {
StructType structType;
StructType recursiveStructType;
@@ -78,4 +81,25 @@ public class StructTest extends BaseTest {
"}");
}
+ @Test
+ public void testTypeUpdate() throws Exception {
+ testTypeUpdateForAttributes();
+ }
+
+ @Override
+ protected int getNumberOfFields(TypeSystem ts, String typeName) throws Exception {
+ return ts.getDataType(StructType.class, typeName).numFields;
+ }
+
+ @Override
+ protected StructTypeDefinition getTypeDefinition(String name, AttributeDefinition... attributes) {
+ return new StructTypeDefinition(name, attributes);
+ }
+
+ @Override
+ protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(typeDefinition),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/bf5672c5/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
index 8330ecb..4482816 100755
--- a/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/typesystem/types/TraitTest.java
@@ -23,9 +23,12 @@ import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.IStruct;
import org.apache.atlas.typesystem.ITypedStruct;
import org.apache.atlas.typesystem.Struct;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.Assert;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+
import java.util.HashMap;
import java.util.Map;
@@ -33,7 +36,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-public class TraitTest extends BaseTest {
+public class TraitTest extends HierarchicalTypeTest<TraitType> {
@BeforeMethod
@@ -213,8 +216,30 @@ public class TraitTest extends BaseTest {
"\tA.C.D.c : \t3\n" +
"\tA.C.D.d : \t3\n" +
"}");
+ }
+
+ @Override
+ protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, AttributeDefinition... attributes) {
+ return new HierarchicalTypeDefinition(TraitType.class, name, null, attributes);
+ }
+ @Override
+ protected HierarchicalTypeDefinition<TraitType> getTypeDefinition(String name, ImmutableList<String> superTypes,
+ AttributeDefinition... attributes) {
+ return new HierarchicalTypeDefinition(TraitType.class, name, superTypes, attributes);
}
+ @Override
+ protected TypesDef getTypesDef(StructTypeDefinition typeDefinition) {
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.of((HierarchicalTypeDefinition<TraitType>) typeDefinition),
+ ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ }
+
+ @Override
+ protected TypesDef getTypesDef(HierarchicalTypeDefinition<TraitType>... typeDefinitions) {
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.copyOf(typeDefinitions), ImmutableList.<HierarchicalTypeDefinition<ClassType>>of());
+ }
}