You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/09/29 16:31:20 UTC
incubator-atlas git commit: ATLAS-1184 ReservedTypesRegistrar checks
for existence of 1st class type (svimal2106 via shwethags)
Repository: incubator-atlas
Updated Branches:
refs/heads/master 100749b49 -> 4c56c61f8
ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/4c56c61f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/4c56c61f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/4c56c61f
Branch: refs/heads/master
Commit: 4c56c61f85c4c38a1d2bec3088130f2ea5f1e8fe
Parents: 100749b
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Thu Sep 29 11:50:47 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Thu Sep 29 11:50:47 2016 +0530
----------------------------------------------------------------------
release-log.txt | 1 +
.../atlas/services/ReservedTypesRegistrar.java | 55 ++++++++++++++++----
.../test/java/org/apache/atlas/TestUtils.java | 49 +++++++++++++++++
.../services/ReservedTypesRegistrarTest.java | 32 ++++++------
4 files changed, 112 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 0d78273..1a691ca 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
+ATLAS-1184 ReservedTypesRegistrar checks for existence of 1st class type (svimal2106 via shwethags)
ATLAS-1199 Atlas UI not loading after fresh build due to jquery-asBreadcrumbs plugin upgrade (kevalbhatt via shwethags)
ATLAS-1174 Framework to apply updates to types in the type-system (sarath.kum4r@gmail.com via shwethags)
ATLAS-1155 Errors in Eclipse when I bring in the latest code (davidrad via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
index 41c0155..6e0b586 100644
--- a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
+++ b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
@@ -18,12 +18,12 @@
package org.apache.atlas.services;
+import com.google.common.collect.ImmutableList;
import org.apache.atlas.AtlasException;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.*;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,8 +31,10 @@ import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
+import java.util.List;
public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
@@ -81,13 +83,46 @@ public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
LOG.error("Error while deserializing JSON in {}", typeDefName);
throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e);
}
- HierarchicalTypeDefinition<ClassType> classDef = typesDef.classTypesAsJavaList().get(0);
- if (!typeSystem.isRegistered(classDef.typeName)) {
- metadataService.createType(typeDefJSON);
- LOG.info("Registered types in {}", typeDefName);
- } else {
- LOG.warn("class {} already registered, ignoring types in {}", classDef.typeName,
- typeDefName);
+ List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new ArrayList<>();
+ List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new ArrayList<>();
+ List<EnumTypeDefinition> createEnumDefList = new ArrayList<>();
+ List<StructTypeDefinition> createStructDefList = new ArrayList<>();
+
+ for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){
+ if(!typeSystem.isRegistered(classTypeDef.typeName)){
+ LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName);
+ createClassDefList.add(classTypeDef);
+ }
+ }
+
+ for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){
+ if(!typeSystem.isRegistered(traitTypeDef.typeName)){
+ LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName);
+ createTraitDefList.add(traitTypeDef);
+ }
+ }
+
+ for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){
+ if(!typeSystem.isRegistered(structTypeDef.typeName)){
+ LOG.debug("StructType {} is not registered. Adding to create type list", structTypeDef.typeName);
+ createStructDefList.add(structTypeDef);
+ }
+ }
+
+ for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){
+ if(!typeSystem.isRegistered(enumTypeDef.name)){
+ LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name);
+ createEnumDefList.add(enumTypeDef);
+ }
+ }
+
+ TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList), ImmutableList.copyOf(createStructDefList),
+ ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList));
+
+ String createTypeJSON = TypesSerialization.toJson(createTypes);
+ if(createTypeJSON != null) {
+ metadataService.createType(createTypeJSON);
+ LOG.info("Created types definition JSON {}", createTypeJSON);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index bd9df62..b27854e 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -239,6 +239,55 @@ public final class TestUtils {
public static final String NAME = "name";
+ public static TypesDef simpleType(){
+ HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+ createClassTypeDef("h_type", ImmutableSet.<String>of(),
+ createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+ new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+ HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+ createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+ EnumValue values[] = {new EnumValue("ONE", 1),};
+
+ EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
+ return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+ ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition));
+ }
+
+ public static TypesDef simpleTypeUpdated(){
+ HierarchicalTypeDefinition<ClassType> superTypeDefinition =
+ createClassTypeDef("h_type", ImmutableSet.<String>of(),
+ createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+ createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+ createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+ StructTypeDefinition structTypeDefinition = new StructTypeDefinition("s_type", "structType",
+ new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE)});
+
+ HierarchicalTypeDefinition<TraitType> traitTypeDefinition =
+ createTraitTypeDef("t_type", "traitType", ImmutableSet.<String>of());
+
+ EnumValue values[] = {new EnumValue("ONE", 1),};
+
+ EnumTypeDefinition enumTypeDefinition = new EnumTypeDefinition("e_type", "enumType", values);
+ return TypesUtil.getTypesDef(ImmutableList.of(enumTypeDefinition), ImmutableList.of(structTypeDefinition),
+ ImmutableList.of(traitTypeDefinition), ImmutableList.of(superTypeDefinition, newSuperTypeDefinition));
+ }
+
+ public static TypesDef simpleTypeUpdatedDiff() {
+ HierarchicalTypeDefinition<ClassType> newSuperTypeDefinition =
+ createClassTypeDef("new_h_type", ImmutableSet.<String>of(),
+ createOptionalAttrDef("attr", DataTypes.STRING_TYPE));
+
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(), ImmutableList.of(newSuperTypeDefinition));
+ }
+
public static TypesDef defineHiveTypes() {
String _description = "_description";
HierarchicalTypeDefinition<ClassType> superTypeDefinition =
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/4c56c61f/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
index d602bd8..effab15 100644
--- a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
+++ b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
@@ -23,6 +23,8 @@ import org.apache.atlas.TestUtils;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
@@ -55,17 +57,6 @@ public class ReservedTypesRegistrarTest {
}
@Test
- public void testRegisterFirstChecksClassTypeIsRegistered() throws AtlasException {
- ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- TypesDef typesDef = TestUtils.defineHiveTypes();
- String typesJson = TypesSerialization.toJson(typesDef);
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
- InOrder inOrder = inOrder(typeSystem, metadataService);
- inOrder.verify(typeSystem).isRegistered(typesDef.classTypesAsJavaList().get(0).typeName);
- inOrder.verify(metadataService).createType(typesJson);
- }
-
- @Test
public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
TypesDef typesDef = TestUtils.defineHiveTypes();
@@ -90,12 +81,23 @@ public class ReservedTypesRegistrarTest {
}
@Test
- public void testShouldNotRegisterIfTypeIsAlreadyRegistered() throws AtlasException {
+ public void testCreateAndUpdateType() throws AtlasException{
ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- TypesDef typesDef = TestUtils.defineHiveTypes();
+ TypesDef typesDef = TestUtils.simpleType();
String typesJson = TypesSerialization.toJson(typesDef);
- when(typeSystem.isRegistered(typesDef.classTypesAsJavaList().get(0).typeName)).thenReturn(true);
reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
- verifyZeroInteractions(metadataService);
+ verify(metadataService).createType(typesJson);
+
+ //test update simple type
+ TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated();
+ String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef);
+ TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff();
+ String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff);
+ when(typeSystem.isRegistered("h_type")).thenReturn(true);
+ when(typeSystem.isRegistered("t_type")).thenReturn(true);
+ when(typeSystem.isRegistered("s_type")).thenReturn(true);
+ when(typeSystem.isRegistered("e_type")).thenReturn(true);
+ reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", updatedTypesJson);
+ verify(metadataService).createType(simpleTypeUpdatedDiffJson);
}
}