You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2016/11/08 00:00:12 UTC
[1/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
Repository: incubator-atlas
Updated Branches:
refs/heads/master 2ea3a455e -> def9e385c
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
deleted file mode 100644
index e0cabe9..0000000
--- a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrar.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.*;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.List;
-
-public class ReservedTypesRegistrar implements IBootstrapTypesRegistrar {
-
- private static final Logger LOG = LoggerFactory.getLogger(ReservedTypesRegistrar.class);
-
- static String getTypesDir() {
- return System.getProperty("atlas.home")+ File.separator+"models";
- }
-
- @Override
- public void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
- throws AtlasException {
- File typesDir = new File(typesDirName);
- if (!typesDir.exists()) {
- LOG.info("No types directory {} found - not registering any reserved types", typesDirName);
- return;
- }
-
- File[] typeDefFiles = typesDir.listFiles();
- //TODO - Enforce a dependency order among models registered by definition and not by modifiedTime as below
- // Workaround - Sort by modifiedTime to get the dependency of models in the right order - first hdfs, followed by hive and hive is needed by storm, falcon models.
- // Sorting them by time will ensure the right order since the modules are in the correct order in pom.
- Arrays.sort(typeDefFiles, new Comparator<File>() {
- public int compare(File f1, File f2) {
- return Long.valueOf(f1.lastModified()).compareTo(f2.lastModified());
- }
- });
-
- for (File typeDefFile : typeDefFiles) {
- try {
- if (typeDefFile.isFile()) {
- String typeDefJSON = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
- registerType(typeSystem, metadataService, typeDefFile.getAbsolutePath(), typeDefJSON);
- }
- } catch (IOException e) {
- LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), e);
- } catch (AtlasException e) {
- LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), e);
- throw e;
- }
- }
-
- }
-
- void registerType(TypeSystem typeSystem, MetadataService metadataService, String typeDefName, String typeDefJSON)
- throws AtlasException {
- TypesDef typesDef = null;
- try {
- typesDef = TypesSerialization.fromJson(typeDefJSON);
- } catch (Exception e) {
- LOG.error("Error while deserializing JSON in {}", typeDefName);
- throw new ReservedTypesRegistrationException("Error while deserializing JSON in " + typeDefName, e);
- }
- List<HierarchicalTypeDefinition<ClassType>> createClassDefList = new ArrayList<>();
- List<HierarchicalTypeDefinition<TraitType>> createTraitDefList = new ArrayList<>();
- List<EnumTypeDefinition> createEnumDefList = new ArrayList<>();
- List<StructTypeDefinition> createStructDefList = new ArrayList<>();
-
- for(HierarchicalTypeDefinition<ClassType> classTypeDef:typesDef.classTypesAsJavaList()){
- if(!typeSystem.isRegistered(classTypeDef.typeName)){
- LOG.debug("ClassType {} is not registered. Adding to create type list", classTypeDef.typeName);
- createClassDefList.add(classTypeDef);
- }
- }
-
- for(HierarchicalTypeDefinition<TraitType> traitTypeDef:typesDef.traitTypesAsJavaList()){
- if(!typeSystem.isRegistered(traitTypeDef.typeName)){
- LOG.debug("TraitType {} is not registered. Adding to create type list", traitTypeDef.typeName);
- createTraitDefList.add(traitTypeDef);
- }
- }
-
- for(StructTypeDefinition structTypeDef:typesDef.structTypesAsJavaList()){
- if(!typeSystem.isRegistered(structTypeDef.typeName)){
- LOG.debug("StructType {} is not registered. Adding to create type list", structTypeDef.typeName);
- createStructDefList.add(structTypeDef);
- }
- }
-
- for(EnumTypeDefinition enumTypeDef:typesDef.enumTypesAsJavaList()){
- if(!typeSystem.isRegistered(enumTypeDef.name)){
- LOG.debug("EnumType {} is not registered. Adding to create type list", enumTypeDef.name);
- createEnumDefList.add(enumTypeDef);
- }
- }
-
- TypesDef createTypes = TypesUtil.getTypesDef(ImmutableList.copyOf(createEnumDefList), ImmutableList.copyOf(createStructDefList),
- ImmutableList.copyOf(createTraitDefList), ImmutableList.copyOf(createClassDefList));
-
- if (! createTypes.isEmpty()) {
- String createTypeJSON = TypesSerialization.toJson(createTypes);
- if (createTypeJSON != null) {
- metadataService.createType(createTypeJSON);
- LOG.info("Created types definition JSON {}", createTypeJSON);
- }
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java b/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java
deleted file mode 100644
index 4b3b31d..0000000
--- a/repository/src/main/java/org/apache/atlas/services/ReservedTypesRegistrationException.java
+++ /dev/null
@@ -1,26 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.atlas.services;
-
-public class ReservedTypesRegistrationException extends RuntimeException {
- public ReservedTypesRegistrationException(String message, Exception e) {
- super(message, e);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
index 03d155c..8851b79 100644
--- a/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/BaseRepositoryTest.java
@@ -44,6 +44,9 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.testng.annotations.Guice;
+import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
+import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
+
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
@@ -67,6 +70,7 @@ public class BaseRepositoryTest {
protected void setUp() throws Exception {
//force graph initialization / built in type registration
TestUtils.getGraph();
+ setUpDefaultTypes();
setUpTypes();
new GraphBackedSearchIndexer(new AtlasTypeRegistry());
TestUtils.resetRequestContext();
@@ -395,4 +399,42 @@ public class BaseRepositoryTest {
// return the reference to created instance with guid
return new Id(guids.get(guids.size() - 1), 0, referenceable.getTypeName());
}
+
+ private void setUpDefaultTypes() throws Exception {
+ TypesDef typesDef = createDefaultTypeDefinitions();
+ String typesAsJSON = TypesSerialization.toJson(typesDef);
+ metadataService.createType(typesAsJSON);
+ }
+
+ TypesDef createDefaultTypeDefinitions() {
+ HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
+ .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
+ new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
+
+ HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
+ .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
+ new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
+ TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
+ new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
+
+ HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
+ .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
+ ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
+ .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
+ ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
+
+ HierarchicalTypeDefinition<ClassType> processType = TypesUtil
+ .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
+ ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
+ new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
+ Multiplicity.OPTIONAL, false, null),
+ new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
+ Multiplicity.OPTIONAL, false, null));
+
+ return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
+ ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
+ ImmutableList.of(referenceableType, assetType, infraType, datasetType, processType));
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index abb8e94..88edf8f 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -33,7 +33,6 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.services.DefaultMetadataService;
import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
@@ -527,7 +526,6 @@ public final class TestUtils {
ITypeStore typeStore = new GraphBackedTypeStore();
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(repo,
typeStore,
- new ReservedTypesRegistrar(),
Collections.singletonList(indexerProvider),
new ArrayList<Provider<EntityChangeListener>>(), TypeSystem.getInstance(), config, typeCache);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
index 3ffed90..cdbde1b 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStoreTest.java
@@ -190,7 +190,7 @@ public class AtlasTypeDefGraphStoreTest {
try {
existingTypesDef = typeDefStore.searchTypesDef(new SearchFilter());
} catch (AtlasBaseException e) {
- fail("Shouldn't have failed during Search");
+ // ignore
}
assertNotEquals(atlasTypesDef, existingTypesDef, "Types to be created already exist in the system");
@@ -204,7 +204,7 @@ public class AtlasTypeDefGraphStoreTest {
assertTrue(createdTypesDef.getEntityDefs().containsAll(atlasTypesDef.getEntityDefs()), "EntityDef creation failed");
} catch (AtlasBaseException e) {
- fail("Creation of Types should've been a success");
+ fail("Creation of Types should've been a success", e);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
deleted file mode 100644
index 9722a72..0000000
--- a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import com.google.inject.Provider;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.listener.EntityChangeListener;
-import org.apache.atlas.listener.TypesChangeListener;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.typestore.ITypeStore;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.ha.HAConfiguration;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.commons.configuration.Configuration;
-import org.mockito.Matchers;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-
-public class DefaultMetadataServiceMockTest {
-
- @Mock
- private IBootstrapTypesRegistrar typesRegistrar;
-
- @Mock
- private TypeSystem typeSystem;
-
- @Mock
- private MetadataRepository metadataRepository;
-
- @Mock
- private ITypeStore typeStore;
-
- @Mock
- private Configuration configuration;
-
- @BeforeMethod
- public void setup() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test
- public void testShouldInvokeTypesRegistrarOnCreation() throws AtlasException {
- when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
- when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
- when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(false);
- DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
- mock(ITypeStore.class),
- typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
-
- verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
- typeSystem, defaultMetadataService);
- }
-
- @Test
- public void testShouldNotRestoreTypesIfHAIsEnabled() throws AtlasException {
- when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
- when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
-
- new DefaultMetadataService(metadataRepository, typeStore,
- typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
-
- verifyZeroInteractions(typeStore);
- verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
- verifyZeroInteractions(typesRegistrar);
- }
-
- @Test
- public void testShouldRestoreTypeSystemOnServerActive() throws AtlasException {
- when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
- when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
-
- TypesDef typesDef = mock(TypesDef.class);
- when(typeStore.restore()).thenReturn(typesDef);
- when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
-
- DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
- typeStore,
- typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
- defaultMetadataService.instanceIsActive();
-
- verify(typeStore).restore();
- verify(typeSystem).defineTypes(typesDef);
- verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
- typeSystem, defaultMetadataService);
- }
-
- @Test
- public void testShouldOnlyRestoreCacheOnServerActiveIfAlreadyDoneOnce() throws AtlasException {
- when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
- when(configuration.getBoolean(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(true);
-
- TypesDef typesDef = mock(TypesDef.class);
- when(typeStore.restore()).thenReturn(typesDef);
- when(typeSystem.isRegistered(any(String.class))).thenReturn(true);
-
- TypeSystem.TransientTypeSystem transientTypeSystem = mock(TypeSystem.TransientTypeSystem.class);
- HashMap<String, IDataType> typesAdded = new HashMap<>();
- when(transientTypeSystem.getTypesAdded()).thenReturn(typesAdded);
- when(typeSystem.createTransientTypeSystem(typesDef, true)).
- thenReturn(transientTypeSystem);
- DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
- typeStore,
- typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
-
- defaultMetadataService.instanceIsActive();
- defaultMetadataService.instanceIsPassive();
- defaultMetadataService.instanceIsActive();
-
- verify(typeStore, times(2)).restore();
- verify(typeSystem, times(1)).defineTypes(typesDef);
- verify(typesRegistrar, times(1)).
- registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, defaultMetadataService);
- verify(typeSystem, times(1)).createTransientTypeSystem(typesDef, true);
- verify(typeSystem, times(1)).commitTypes(typesAdded);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java b/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
deleted file mode 100644
index effab15..0000000
--- a/repository/src/test/java/org/apache/atlas/services/ReservedTypesRegistrarTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.TestUtils;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUtils;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.mockito.InOrder;
-import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import static org.mockito.Mockito.inOrder;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.verifyZeroInteractions;
-import static org.mockito.Mockito.when;
-
-public class ReservedTypesRegistrarTest {
-
- @Mock
- private TypeSystem typeSystem;
-
- @Mock
- private MetadataService metadataService;
-
- @BeforeMethod
- public void setup() {
- MockitoAnnotations.initMocks(this);
- }
-
- @Test
- public void testRegistrationWithNoFiles() throws AtlasException {
- IBootstrapTypesRegistrar bootstrapTypesRegistrar = new ReservedTypesRegistrar();
- bootstrapTypesRegistrar.registerTypes("/some/dir/", typeSystem, metadataService);
- verifyZeroInteractions(typeSystem);
- }
-
- @Test
- public void testRegisterCreatesTypesUsingMetadataService() throws AtlasException {
- ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- TypesDef typesDef = TestUtils.defineHiveTypes();
- String typesJson = TypesSerialization.toJson(typesDef);
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
- verify(metadataService).createType(typesJson);
- }
-
- @Test(expectedExceptions = ReservedTypesRegistrationException.class)
- public void testRegisterFailsIfErrorInJson() throws AtlasException {
- ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", "invalid json");
- }
-
- @Test(expectedExceptions = AtlasException.class)
- public void testRegisterFailsOnTypeCreationException() throws AtlasException {
- ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- TypesDef typesDef = TestUtils.defineHiveTypes();
- String typesJson = TypesSerialization.toJson(typesDef);
- when(metadataService.createType(typesJson)).thenThrow(new AtlasException("some exception"));
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
- }
-
- @Test
- public void testCreateAndUpdateType() throws AtlasException{
- ReservedTypesRegistrar reservedTypesRegistrar = new ReservedTypesRegistrar();
- TypesDef typesDef = TestUtils.simpleType();
- String typesJson = TypesSerialization.toJson(typesDef);
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", typesJson);
- verify(metadataService).createType(typesJson);
-
- //test update simple type
- TypesDef updatedTypesDef = TestUtils.simpleTypeUpdated();
- String updatedTypesJson = TypesSerialization.toJson(updatedTypesDef);
- TypesDef simpleTypeUpdatedDiff = TestUtils.simpleTypeUpdatedDiff();
- String simpleTypeUpdatedDiffJson = TypesSerialization.toJson(simpleTypeUpdatedDiff);
- when(typeSystem.isRegistered("h_type")).thenReturn(true);
- when(typeSystem.isRegistered("t_type")).thenReturn(true);
- when(typeSystem.isRegistered("s_type")).thenReturn(true);
- when(typeSystem.isRegistered("e_type")).thenReturn(true);
- reservedTypesRegistrar.registerType(typeSystem, metadataService, "/some/file/model.json", updatedTypesJson);
- verify(metadataService).createType(simpleTypeUpdatedDiffJson);
- }
-}
[2/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala b/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
deleted file mode 100644
index 7caf5e8..0000000
--- a/addons/storm-bridge/src/main/scala/org/apache/atlas/storm/model/StormDataModel.scala
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.storm.model
-
-import org.apache.atlas.AtlasClient
-import org.apache.atlas.typesystem.TypesDef
-import org.apache.atlas.typesystem.builders.TypesBuilder
-import org.apache.atlas.typesystem.json.TypesSerialization
-
-
-/**
- * This represents the data model for a storm topology.
- */
-object StormDataModel extends App {
-
- var typesDef : TypesDef = null
-
- val typesBuilder = new TypesBuilder
- import typesBuilder._
-
- typesDef = types {
-
- /**
- * Model is represented as:
- * Topology is a Process Super Type inheriting inputs/outputs
- * Input DataSet(s) => Topology => Output DataSet(s)
- * Also, Topology contains the Graph of Nodes
- * Topology => Node(s) -> Spouts/Bolts
- */
- _class(StormDataTypes.STORM_TOPOLOGY.getName, List(AtlasClient.PROCESS_SUPER_TYPE)) {
- "id" ~ (string, required, indexed, unique)
- "startTime" ~ date
- "endTime" ~ date
- "conf" ~ (map(string, string), optional)
- "clusterName" ~ (string, optional, indexed)
-
- // Nodes in the Graph
- "nodes" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- }
-
- // Base class for DataProducer aka Spouts and
- // DataProcessor aka Bolts, also links from Topology
- _class(StormDataTypes.STORM_NODE.getName) {
- "name" ~ (string, required, indexed)
- "description" ~ (string, optional, indexed)
- // fully qualified driver java class name
- "driverClass" ~ (string, required, indexed)
- // spout or bolt configuration NVPs
- "conf" ~ (map(string, string), optional)
- }
-
- // Data Producer and hence only outputs
- _class(StormDataTypes.STORM_SPOUT.getName, List(StormDataTypes.STORM_NODE.getName)) {
- // "outputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- "outputs" ~ (array(string), collection)
- }
-
- // Data Processor and hence both inputs and outputs (inherited from Spout)
- _class(StormDataTypes.STORM_BOLT.getName, List(StormDataTypes.STORM_NODE.getName)) {
- // "inputs" ~ (array(StormDataTypes.STORM_NODE.getName), collection, composite)
- "inputs" ~ (array(string), collection)
- "outputs" ~ (array(string), collection, optional)
- }
-
- // Kafka Data Set
- _class(StormDataTypes.KAFKA_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "topic" ~ (string, required, unique, indexed)
- "uri" ~ (string, required)
- }
-
- // JMS Data Set
- _class(StormDataTypes.JMS_TOPIC.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "topic" ~ (string, required, unique, indexed)
- "uri" ~ (string, required)
- }
-
- // HBase Data Set
- _class(StormDataTypes.HBASE_TABLE.getName, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- "uri" ~ (string, required)
- }
- // Hive table data set already exists in atlas.
- }
-
- // add the types to atlas
- val typesAsJSON = TypesSerialization.toJson(typesDef)
- println("Storm Data Model as JSON: ")
- println(typesAsJSON)
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
index fe4c001..e0800b8 100644
--- a/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
+++ b/addons/storm-bridge/src/test/java/org/apache/atlas/storm/hook/StormAtlasHookIT.java
@@ -18,19 +18,11 @@
package org.apache.atlas.storm.hook;
-import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
-import org.apache.atlas.hive.model.HiveDataTypes;
-import org.apache.atlas.storm.model.StormDataModel;
import org.apache.atlas.storm.model.StormDataTypes;
import org.apache.atlas.typesystem.Referenceable;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration;
import org.apache.storm.ILocalCluster;
@@ -67,37 +59,6 @@ public class StormAtlasHookIT {
} else {
atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
}
- registerDataModel(new HiveDataModelGenerator());
- }
-
- private void registerDataModel(HiveDataModelGenerator dataModelGenerator) throws AtlasException,
- AtlasServiceException {
- try {
- atlasClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
- LOG.info("Hive data model is already registered! Going ahead with registration of Storm Data model");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Expected in case types do not exist
- LOG.info("Registering Hive data model");
- atlasClient.createType(dataModelGenerator.getModelAsJson());
- } else {
- throw ase;
- }
- }
-
-
- try {
- atlasClient.getType(StormDataTypes.STORM_TOPOLOGY.getName());
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- LOG.info("Registering Storm/Kafka data model");
- StormDataModel.main(new String[]{});
- TypesDef typesDef = StormDataModel.typesDef();
- String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
- LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
- atlasClient.createType(stormTypesAsJSON);
- }
- }
}
@@ -109,23 +70,6 @@ public class StormAtlasHookIT {
atlasClient = null;
}
- @Test
- public void testCreateDataModel() throws Exception {
- StormDataModel.main(new String[]{});
- TypesDef stormTypesDef = StormDataModel.typesDef();
-
- String stormTypesAsJSON = TypesSerialization.toJson(stormTypesDef);
- LOG.info("stormTypesAsJSON = {}", stormTypesAsJSON);
-
- registerDataModel(new HiveDataModelGenerator());
-
- // verify types are registered
- for (StormDataTypes stormDataType : StormDataTypes.values()) {
- Assert.assertNotNull(atlasClient.getType(stormDataType.getName()));
- }
- }
-
- @Test (dependsOnMethods = "testCreateDataModel")
public void testAddEntities() throws Exception {
StormTopology stormTopology = StormTestUtil.createTestTopology();
StormTestUtil.submitTopology(stormCluster, TOPOLOGY_NAME, stormTopology);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/distro/src/main/assemblies/standalone-package.xml
----------------------------------------------------------------------
diff --git a/distro/src/main/assemblies/standalone-package.xml b/distro/src/main/assemblies/standalone-package.xml
index 39c6512..215cb23 100755
--- a/distro/src/main/assemblies/standalone-package.xml
+++ b/distro/src/main/assemblies/standalone-package.xml
@@ -106,9 +106,9 @@
<outputDirectory>examples</outputDirectory>
</fileSet>
- <!-- addons/hdfs-model -->
+ <!-- out-of-box-models -->
<fileSet>
- <directory>../addons/hdfs-model/target/models</directory>
+ <directory>../addons/models</directory>
<outputDirectory>models</outputDirectory>
</fileSet>
@@ -130,55 +130,29 @@
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/hive-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/falcon -->
<fileSet>
<directory>../addons/falcon-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/falcon-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/sqoop -->
<fileSet>
<directory>../addons/sqoop-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/sqoop-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- addons/storm -->
<fileSet>
<directory>../addons/storm-bridge/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
- <fileSet>
- <directory>../addons/storm-bridge/target/models</directory>
- <outputDirectory>models</outputDirectory>
- </fileSet>
-
<!-- for kafka topic setup -->
<fileSet>
<directory>../notification/target/dependency/hook</directory>
<outputDirectory>hook</outputDirectory>
</fileSet>
-
- <!-- for patches -->
- <fileSet>
- <directory>../addons/hive-bridge/src/patches</directory>
- <outputDirectory>models/patches</outputDirectory>
- </fileSet>
</fileSets>
<files>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
index 709fcbc..fe38fba 100644
--- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
+++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
@@ -53,7 +53,9 @@ public enum AtlasErrorCode {
INTERNAL_ERROR(500, "ATLAS5001E", "Internal server error {0}"),
INDEX_CREATION_FAILED(500, "ATLAS5002E", "Index creation failed for {0}"),
- INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}")
+ INDEX_ROLLBACK_FAILED(500, "ATLAS5003E", "Index rollback failed for {0}"),
+ PATCH_NOT_APPLICABLE_FOR_TYPE(500, "ATLAS5004E", "{0} - invalid patch for type {1}"),
+ PATCH_FOR_UNKNOWN_TYPE(500, "ATLAS5005E", "{0} - patch references unknown type {1}")
;
private String errorCode;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
index d5ed0bc..7421da8 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasStructDef.java
@@ -279,12 +279,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
List<AtlasConstraintDef> constraintDefs) {
setName(name);
setTypeName(typeName);
- setOptional(isOptional);
+ setIsOptional(isOptional);
setCardinality(cardinality);
setValuesMinCount(valuesMinCount);
setValuesMaxCount(valuesMaxCount);
- setUnique(isUnique);
- setIndexable(isIndexable);
+ setIsUnique(isUnique);
+ setIsIndexable(isIndexable);
setConstraintDefs(constraintDefs);
}
@@ -292,12 +292,12 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
if (other != null) {
setName(other.getName());
setTypeName(other.getTypeName());
- setOptional(other.isOptional());
+ setIsOptional(other.getIsOptional());
setCardinality(other.getCardinality());
setValuesMinCount(other.getValuesMinCount());
setValuesMaxCount(other.getValuesMaxCount());
- setUnique(other.isUnique());
- setIndexable(other.isIndexable());
+ setIsUnique(other.getIsUnique());
+ setIsIndexable(other.getIsIndexable());
setConstraintDefs(other.getConstraintDefs());
}
}
@@ -318,11 +318,11 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.typeName = typeName;
}
- public boolean isOptional() {
+ public boolean getIsOptional() {
return isOptional;
}
- public void setOptional(boolean optional) { isOptional = optional; }
+ public void setIsOptional(boolean optional) { isOptional = optional; }
public void setCardinality(Cardinality cardinality) {
this.cardinality = cardinality;
@@ -348,19 +348,19 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
this.valuesMaxCount = valuesMaxCount;
}
- public boolean isUnique() {
+ public boolean getIsUnique() {
return isUnique;
}
- public void setUnique(boolean unique) {
+ public void setIsUnique(boolean unique) {
isUnique = unique;
}
- public boolean isIndexable() {
+ public boolean getIsIndexable() {
return isIndexable;
}
- public void setIndexable(boolean idexable) {
+ public void setIsIndexable(boolean idexable) {
isIndexable = idexable;
}
@@ -399,7 +399,7 @@ public class AtlasStructDef extends AtlasBaseTypeDef implements Serializable {
sb.append("AtlasAttributeDef{");
sb.append("name='").append(name).append('\'');
sb.append(", typeName='").append(typeName).append('\'');
- sb.append(", isOptional=").append(isOptional);
+ sb.append(", getIsOptional=").append(isOptional);
sb.append(", cardinality=").append(cardinality);
sb.append(", valuesMinCount=").append(valuesMinCount);
sb.append(", valuesMaxCount=").append(valuesMaxCount);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
index fb2029d..899e53f 100644
--- a/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
+++ b/intg/src/main/java/org/apache/atlas/model/typedef/AtlasTypesDef.java
@@ -19,6 +19,7 @@ package org.apache.atlas.model.typedef;
import org.apache.commons.collections.CollectionUtils;
import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnore;
import org.codehaus.jackson.annotate.JsonIgnoreProperties;
import org.codehaus.jackson.map.annotate.JsonSerialize;
@@ -91,6 +92,7 @@ public class AtlasTypesDef {
this.entityDefs = entityDefs;
}
+ @JsonIgnore
public boolean isEmpty() {
return CollectionUtils.isEmpty(enumDefs) &&
CollectionUtils.isEmpty(structDefs) &&
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
index 78b29fd..c8c2216 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasStructType.java
@@ -198,7 +198,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret;
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
@@ -218,7 +218,7 @@ public class AtlasStructType extends AtlasType {
if (value != null) {
ret = dataType.validateValue(value, fieldName, messages) && ret;
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false;
messages.add(fieldName + ": mandatory attribute value missing in type " + getTypeName());
@@ -244,7 +244,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.getAttribute(attributeName), attributeDef);
obj.setAttribute(attributeName, attributeValue);
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
obj.setAttribute(attributeName, createDefaultValue(attributeDef));
}
}
@@ -260,7 +260,7 @@ public class AtlasStructType extends AtlasType {
Object attributeValue = getNormalizedValue(obj.get(attributeName), attributeDef);
obj.put(attributeName, attributeValue);
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
obj.put(attributeName, createDefaultValue(attributeDef));
}
}
@@ -276,7 +276,7 @@ public class AtlasStructType extends AtlasType {
}
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
attributes.put(attributeDef.getName(), createDefaultValue(attributeDef));
}
}
@@ -310,7 +310,7 @@ public class AtlasStructType extends AtlasType {
ret = false; // invalid value
}
}
- } else if (!attributeDef.isOptional()) {
+ } else if (!attributeDef.getIsOptional()) {
ret = false; // mandatory attribute not present
}
@@ -322,7 +322,7 @@ public class AtlasStructType extends AtlasType {
if (attrType != null) {
if (value == null) {
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
return attrType.createDefaultValue();
}
} else {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
index 8924d44..0c118f0 100644
--- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
+++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java
@@ -64,6 +64,10 @@ public class AtlasTypeRegistry {
public Collection<String> getAllTypeNames() { return registryData.allTypes.getAllTypeNames(); }
+ public boolean isRegisteredType(String typeName) {
+ return registryData.allTypes.isKnownType(typeName);
+ }
+
public AtlasType getType(String typeName) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
LOG.debug("==> AtlasTypeRegistry.getType({})", typeName);
@@ -677,6 +681,10 @@ class TypeCache {
}
}
+ public boolean isKnownType(String typeName) {
+ return typeNameMap.containsKey(typeName);
+ }
+
public Collection<String> getAllTypeNames() {
return Collections.unmodifiableCollection(typeNameMap.keySet());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
index cd44318..aabf269 100755
--- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -44,9 +44,7 @@ import org.apache.atlas.repository.typestore.GraphBackedTypeStore;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.service.Service;
import org.apache.atlas.services.DefaultMetadataService;
-import org.apache.atlas.services.IBootstrapTypesRegistrar;
import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.TypeSystem;
@@ -92,8 +90,6 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
// bind the MetadataService interface to an implementation
bind(MetadataService.class).to(DefaultMetadataService.class).asEagerSingleton();
- bind(IBootstrapTypesRegistrar.class).to(ReservedTypesRegistrar.class);
-
// bind the DiscoveryService interface to an implementation
bind(DiscoveryService.class).to(GraphBackedDiscoveryService.class).asEagerSingleton();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 67b5362..aea54fa 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -267,8 +267,8 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
AtlasAttributeDef attributeDef) {
final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + attributeDef.getName());
AtlasCardinality cardinality = toAtlasCardinality(attributeDef.getCardinality());
- boolean isUnique = attributeDef.isUnique();
- boolean isIndexable = attributeDef.isIndexable();
+ boolean isUnique = attributeDef.getIsUnique();
+ boolean isIndexable = attributeDef.getIsIndexable();
String attribTypeName = attributeDef.getTypeName();
boolean isBuiltInType = AtlasTypeUtil.isBuiltInType(attribTypeName);
boolean isArrayType = AtlasTypeUtil.isArrayType(attribTypeName);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
new file mode 100644
index 0000000..e52ac6d
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/store/bootstrap/AtlasTypeDefStoreInitializer.java
@@ -0,0 +1,364 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.bootstrap;
+
+import org.apache.atlas.AtlasErrorCode;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
+import org.apache.atlas.model.typedef.AtlasClassificationDef;
+import org.apache.atlas.model.typedef.AtlasEntityDef;
+import org.apache.atlas.model.typedef.AtlasEnumDef;
+import org.apache.atlas.model.typedef.AtlasStructDef;
+import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.collections.CollectionUtils;
+import org.codehaus.jackson.annotate.JsonAutoDetect;
+import org.codehaus.jackson.annotate.JsonIgnoreProperties;
+import org.codehaus.jackson.map.annotate.JsonSerialize;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.NONE;
+import static org.codehaus.jackson.annotate.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+
+
+/**
+ * Class that handles initial loading of models and patches into typedef store
+ */
+public class AtlasTypeDefStoreInitializer {
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeDefStoreInitializer.class);
+
+ public void initializeStore(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
+ File typesDir = new File(typesDirName);
+ File[] typeDefFiles = typesDir.exists() ? typesDir.listFiles() : null;
+
+ if (typeDefFiles == null || typeDefFiles.length == 0) {
+ LOG.info("Types directory {} does not exist or not readable or has no typedef files", typesDirName);
+
+ return;
+ }
+
+ // sort the files by filename
+ Arrays.sort(typeDefFiles);
+
+ for (File typeDefFile : typeDefFiles) {
+ if (!typeDefFile.isFile()) {
+ continue;
+ }
+
+ try {
+ String jsonStr = new String(Files.readAllBytes(typeDefFile.toPath()), StandardCharsets.UTF_8);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+
+ if (typesDef == null || typesDef.isEmpty()) {
+ LOG.info("No type in file {}", typeDefFile.getAbsolutePath());
+
+ continue;
+ }
+
+ AtlasTypesDef typesToCreate = new AtlasTypesDef();
+
+ if (CollectionUtils.isNotEmpty(typesDef.getEnumDefs())) {
+ for (AtlasEnumDef enumDef : typesDef.getEnumDefs()) {
+ if (!typeRegistry.isRegisteredType(enumDef.getName())) {
+ typesToCreate.getEnumDefs().add(enumDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getStructDefs())) {
+ for (AtlasStructDef structDef : typesDef.getStructDefs()) {
+ if (!typeRegistry.isRegisteredType(structDef.getName())) {
+ typesToCreate.getStructDefs().add(structDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getClassificationDefs())) {
+ for (AtlasClassificationDef classificationDef : typesDef.getClassificationDefs()) {
+ if (!typeRegistry.isRegisteredType(classificationDef.getName())) {
+ typesToCreate.getClassificationDefs().add(classificationDef);
+ }
+ }
+ }
+
+ if (CollectionUtils.isNotEmpty(typesDef.getEntityDefs())) {
+ for (AtlasEntityDef entityDef : typesDef.getEntityDefs()) {
+ if (!typeRegistry.isRegisteredType(entityDef.getName())) {
+ typesToCreate.getEntityDefs().add(entityDef);
+ }
+ }
+ }
+
+ if (typesToCreate.isEmpty()) {
+ LOG.info("No new type in file {}", typeDefFile.getAbsolutePath());
+
+ continue;
+ }
+
+ LOG.info("Loading types defined in file {}", typeDefFile.getAbsolutePath());
+
+ typeDefStore.createTypesDef(typesDef);
+ } catch (Throwable t) {
+ LOG.error("error while registering types in file " + typeDefFile.getAbsolutePath(), t);
+ }
+ }
+
+ applyTypePatches(typeDefStore, typeRegistry, typesDirName);
+ }
+
+ private void applyTypePatches(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String typesDirName) {
+ String typePatchesDirName = typesDirName + File.separator + "patches";
+ File typePatchesDir = new File(typePatchesDirName);
+ File[] typePatchFiles = typePatchesDir.exists() ? typePatchesDir.listFiles() : null;
+
+ if (typePatchFiles == null || typePatchFiles.length == 0) {
+ LOG.info("Type patches directory {} does not exist or not readable or has no patches", typePatchesDirName);
+
+ return;
+ }
+
+ // sort the files by filename
+ Arrays.sort(typePatchFiles);
+
+ PatchHandler[] patchHandlers = new PatchHandler[] { new AddAttributePatchHandler(typeDefStore, typeRegistry) };
+
+ Map<String, PatchHandler> patchHandlerRegistry = new HashMap<>();
+
+ for (PatchHandler patchHandler : patchHandlers) {
+ for (String supportedAction : patchHandler.getSupportedActions()) {
+ patchHandlerRegistry.put(supportedAction, patchHandler);
+ }
+ }
+
+ for (File typePatchFile : typePatchFiles) {
+ if (!typePatchFile.isFile()) {
+ continue;
+ }
+
+ try {
+ String jsonStr = new String(Files.readAllBytes(typePatchFile.toPath()), StandardCharsets.UTF_8);
+ TypeDefPatches patches = AtlasType.fromJson(jsonStr, TypeDefPatches.class);
+
+ if (patches == null || CollectionUtils.isEmpty(patches.getPatches())) {
+ LOG.info("No patches in file {}", typePatchFile.getAbsolutePath());
+
+ continue;
+ }
+
+ for (TypeDefPatch patch : patches.getPatches()) {
+ PatchHandler patchHandler = patchHandlerRegistry.get(patch.getAction());
+
+ if (patchHandler == null) {
+ LOG.error("Unknown patch action {} in file {}. Ignored",
+ patch.getAction(), typePatchFile.getAbsolutePath());
+
+ continue;
+ }
+
+ try {
+ patchHandler.applyPatch(patch);
+ } catch (AtlasBaseException excp) {
+ LOG.error("Failed to apply " + patch.getAction() + " patch in file " +
+ typePatchFile.getAbsolutePath() + ". Ignored", excp);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("Failed to apply patches in file " + typePatchFile.getAbsolutePath() + ". Ignored", t);
+ }
+ }
+ }
+
+ /**
+ * typedef patch details
+ */
+ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.PROPERTY)
+ static class TypeDefPatch {
+ private String action;
+ private String typeName;
+ private String applyToVersion;
+ private String updateToVersion;
+ private Map<String, Object> params;
+ private List<AtlasAttributeDef> attributeDefs;
+
+ public String getAction() {
+ return action;
+ }
+
+ public void setAction(String action) {
+ this.action = action;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+
+ public String getApplyToVersion() {
+ return applyToVersion;
+ }
+
+ public void setApplyToVersion(String applyToVersion) {
+ this.applyToVersion = applyToVersion;
+ }
+
+ public String getUpdateToVersion() {
+ return updateToVersion;
+ }
+
+ public void setUpdateToVersion(String updateToVersion) {
+ this.updateToVersion = updateToVersion;
+ }
+
+ public Map<String, Object> getParams() {
+ return params;
+ }
+
+ public void setParams(Map<String, Object> params) {
+ this.params = params;
+ }
+
+ public List<AtlasAttributeDef> getAttributeDefs() {
+ return attributeDefs;
+ }
+
+ public void setAttributeDefs(List<AtlasAttributeDef> attributeDefs) {
+ this.attributeDefs = attributeDefs;
+ }
+ }
+
+ /**
+ * list of typedef patches
+ */
+ @JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE)
+ @JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL)
+ @JsonIgnoreProperties(ignoreUnknown = true)
+ @XmlRootElement
+ @XmlAccessorType(XmlAccessType.PROPERTY)
+ static class TypeDefPatches {
+ private List<TypeDefPatch> patches;
+
+ public List<TypeDefPatch> getPatches() {
+ return patches;
+ }
+
+ public void setPatches(List<TypeDefPatch> patches) {
+ this.patches = patches;
+ }
+ }
+
+ abstract class PatchHandler {
+ protected final AtlasTypeDefStore typeDefStore;
+ protected final AtlasTypeRegistry typeRegistry;
+ protected final String[] supportedActions;
+
+ protected PatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry, String[] supportedActions) {
+ this.typeDefStore = typeDefStore;
+ this.typeRegistry = typeRegistry;
+ this.supportedActions = supportedActions;
+ }
+
+ public String[] getSupportedActions() { return supportedActions; }
+
+ public abstract void applyPatch(TypeDefPatch patch) throws AtlasBaseException;
+
+ protected boolean isPatchApplicable(TypeDefPatch patch, AtlasBaseTypeDef currentTypeDef) {
+ String currentVersion = currentTypeDef.getTypeVersion();
+ String applyToVersion = patch.getApplyToVersion();
+
+ return currentVersion == null ||
+ currentVersion.equalsIgnoreCase(applyToVersion) ||
+ currentVersion.startsWith(applyToVersion + ".");
+ }
+ }
+
+ class AddAttributePatchHandler extends PatchHandler {
+ public AddAttributePatchHandler(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry) {
+ super(typeDefStore, typeRegistry, new String[] { "ADD_ATTRIBUTE" });
+ }
+
+ @Override
+ public void applyPatch(TypeDefPatch patch) throws AtlasBaseException {
+ String typeName = patch.getTypeName();
+
+ if (!typeRegistry.isRegisteredType(typeName)) {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_FOR_UNKNOWN_TYPE, patch.getAction(), typeName);
+ }
+
+ AtlasBaseTypeDef typeDef = typeRegistry.getTypeDefByName(typeName);
+
+ if (isPatchApplicable(patch, typeDef)) {
+ if (typeDef.getClass().equals(AtlasEntityDef.class)) {
+ AtlasEntityDef updatedDef = new AtlasEntityDef((AtlasEntityDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateEntityDefByName(typeName, updatedDef);
+ } else if (typeDef.getClass().equals(AtlasClassificationDef.class)) {
+ AtlasClassificationDef updatedDef = new AtlasClassificationDef((AtlasClassificationDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateClassificationDefByName(typeName, updatedDef);
+ } else if (typeDef.getClass().equals(AtlasStructDef.class)) {
+ AtlasStructDef updatedDef = new AtlasStructDef((AtlasStructDef)typeDef);
+
+ for (AtlasAttributeDef attributeDef : patch.getAttributeDefs()) {
+ updatedDef.addAttribute(attributeDef);
+ }
+ updatedDef.setTypeVersion(patch.getUpdateToVersion());
+
+ typeDefStore.updateStructDefByName(typeName, updatedDef);
+ } else {
+ throw new AtlasBaseException(AtlasErrorCode.PATCH_NOT_APPLICABLE_FOR_TYPE,
+ patch.getAction(), typeDef.getClass().getSimpleName());
+ }
+ } else {
+ LOG.info("patch skipped: typeName={}; applyToVersion={}; updateToVersion={}",
+ patch.getTypeName(), patch.getApplyToVersion(), patch.getUpdateToVersion());
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
index 2163e01..816832b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasTypeDefGraphStore.java
@@ -35,6 +35,7 @@ import org.apache.atlas.model.typedef.AtlasEnumDef.AtlasEnumDefs;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasStructDefs;
import org.apache.atlas.model.typedef.AtlasTypesDef;
+import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.util.FilterUtil;
import org.apache.atlas.store.AtlasTypeDefStore;
import org.apache.atlas.type.AtlasTypeRegistry;
@@ -45,6 +46,7 @@ import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.File;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -89,6 +91,8 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
ttr.addTypes(typesDef);
typeRegistry.commitTransientTypeRegistry(ttr);
+
+ bootstrapTypes();
}
@Override
@@ -886,6 +890,15 @@ public abstract class AtlasTypeDefGraphStore implements AtlasTypeDefStore, Activ
LOG.info("Not reacting to a Passive state change");
}
+ private void bootstrapTypes() {
+ AtlasTypeDefStoreInitializer storeInitializer = new AtlasTypeDefStoreInitializer();
+
+ String atlasHomeDir = System.getProperty("atlas.home");
+ String typesDirName = (StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir) + File.separator + "models";
+
+ storeInitializer.initializeStore(this, typeRegistry, typesDirName);
+ }
+
private void updateTypeRegistryPostCommit(AtlasTransientTypeRegistry ttr) {
new TypeRegistryUpdateHook(ttr);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
index 0dd7164..6dfe8cf 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasStructDefStoreV1.java
@@ -421,7 +421,7 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
for (AtlasAttributeDef attributeDef : structDef.getAttributeDefs()) {
if (CollectionUtils.isEmpty(currAttrNames) || !currAttrNames.contains(attributeDef.getName())) {
// new attribute - only allow if optional
- if (!attributeDef.isOptional()) {
+ if (!attributeDef.getIsOptional()) {
throw new AtlasBaseException(AtlasErrorCode.CANNOT_ADD_MANDATORY_ATTRIBUTE, structDef.getName(), attributeDef.getName());
}
}
@@ -510,13 +510,30 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
attribInfo.put("name", attributeDef.getName());
attribInfo.put("dataType", attributeDef.getTypeName());
- attribInfo.put("isUnique", attributeDef.isUnique());
- attribInfo.put("isIndexable", attributeDef.isIndexable());
+ attribInfo.put("isUnique", attributeDef.getIsUnique());
+ attribInfo.put("isIndexable", attributeDef.getIsIndexable());
attribInfo.put("isComposite", isComposite);
attribInfo.put("reverseAttributeName", reverseAttribName);
+
+ final int lower;
+ final int upper;
+
+ if (attributeDef.getCardinality() == AtlasAttributeDef.Cardinality.SINGLE) {
+ lower = attributeDef.getIsOptional() ? 0 : 1;
+ upper = 1;
+ } else {
+ if(attributeDef.getIsOptional()) {
+ lower = 0;
+ } else {
+ lower = attributeDef.getValuesMinCount() < 1 ? 1 : attributeDef.getValuesMinCount();
+ }
+
+ upper = attributeDef.getValuesMaxCount() < 2 ? Integer.MAX_VALUE : attributeDef.getValuesMaxCount();
+ }
+
Map<String, Object> multiplicity = new HashMap<>();
- multiplicity.put("lower", attributeDef.getValuesMinCount());
- multiplicity.put("upper", attributeDef.getValuesMaxCount());
+ multiplicity.put("lower", lower);
+ multiplicity.put("upper", upper);
multiplicity.put("isUnique", AtlasAttributeDef.Cardinality.SET.equals(attributeDef.getCardinality()));
attribInfo.put("multiplicity", AtlasType.toJson(multiplicity));
@@ -532,8 +549,8 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
ret.setName((String) attribInfo.get("name"));
ret.setTypeName((String) attribInfo.get("dataType"));
- ret.setUnique((Boolean) attribInfo.get("isUnique"));
- ret.setIndexable((Boolean) attribInfo.get("isIndexable"));
+ ret.setIsUnique((Boolean) attribInfo.get("isUnique"));
+ ret.setIsIndexable((Boolean) attribInfo.get("isIndexable"));
String attrTypeName = ret.getTypeName();
@@ -608,10 +625,10 @@ public class AtlasStructDefStoreV1 extends AtlasAbstractDefStoreV1 implements At
Boolean isUnique = (Boolean) multiplicity.get("isUnique");
if (minCount == null || minCount.intValue() == 0) {
- ret.setOptional(true);
+ ret.setIsOptional(true);
ret.setValuesMinCount(0);
} else {
- ret.setOptional(false);
+ ret.setIsOptional(false);
ret.setValuesMinCount(minCount.intValue());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java b/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
deleted file mode 100644
index f916169..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasPatchHandler.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import com.google.gson.FieldNamingPolicy;
-import com.google.gson.Gson;
-import com.google.gson.GsonBuilder;
-import com.google.gson.JsonDeserializationContext;
-import com.google.gson.JsonDeserializer;
-import com.google.gson.JsonElement;
-import com.google.gson.JsonParseException;
-import com.google.gson.JsonSyntaxException;
-import org.apache.atlas.services.AtlasTypePatch.PatchContent;
-import org.apache.atlas.services.AtlasTypePatch.PatchData;
-import org.apache.atlas.services.AtlasTypePatch.PatchResult;
-import org.apache.atlas.services.AtlasTypePatch.PatchStatus;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.TypeUpdateException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.IOException;
-import java.lang.reflect.Type;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-public class AtlasPatchHandler {
-
- private static final Logger LOG = LoggerFactory.getLogger(AtlasPatchHandler.class);
-
- public static void handlePatches(DefaultMetadataService metadataService, TypeSystem typeSystem) throws TypeUpdateException {
-
- Map<String, AtlasTypePatch> patchHandlerMap = initializePatchHandlerMap(metadataService, typeSystem);
-
- if (patchHandlerMap == null || patchHandlerMap.isEmpty())
- return;
-
- String patchDirName = ReservedTypesRegistrar.getTypesDir() + File.separator + "patches";
- LOG.info("Checking for any patches to be applied to the type system in " + patchDirName);
-
- File patchDir = new File(patchDirName);
- if (!patchDir.exists()) {
- LOG.info("Patch directory {} doesn't exist, not applying any patches", patchDirName);
- return;
- }
-
- File[] patchFiles = patchDir.listFiles();
- if (patchFiles == null || patchFiles.length == 0) {
- LOG.info("No patch files found in {}, not applying any patches", patchDirName);
- return;
- }
-
- // Sort the patch files based on file name.
- Arrays.sort(patchFiles, new Comparator<File>() {
- public int compare(File f1, File f2) {
- return String.valueOf(f1.getName()).compareTo(f2.getName());
- }
- });
-
- LOG.info("Found " + patchFiles.length + " patch files to process.");
- int patchNumber = 0;
- Gson gson = initializeGson();
- AtlasTypePatch typePatch;
-
- for (File patchFile : patchFiles) {
- try {
- LOG.info("Processing patch file " + (++patchNumber) + " - " + patchFile.getAbsolutePath());
- String content = new String(Files.readAllBytes(patchFile.toPath()), StandardCharsets.UTF_8);
- PatchContent patchContent = gson.fromJson(content, PatchContent.class);
- PatchData[] patchDatas = patchContent.getPatches();
- PatchResult result;
- int patchCounter = 0;
-
- for (PatchData patch : patchDatas) {
- typePatch = patchHandlerMap.get(patch.getAction());
- if (typePatch != null) {
- result = typePatch.applyPatch(patch);
-
- if (result != null) {
- LOG.info(result.getMessage() + " Patch " + (++patchCounter) + " out of " + patchDatas.length + " processed in : " + patchFile.toPath());
- if (result.getStatus().equals(PatchStatus.FAILED)) {
- throw new TypeUpdateException(result.getMessage() + " patch " + patchNumber + " failed in :" + patchFile.getAbsolutePath());
- }
- }
- }
- }
-
- } catch (IOException e) {
- throw new TypeUpdateException("Unable to read patch file from " + patchFile.getAbsolutePath());
- } catch (JsonSyntaxException e) {
- throw new TypeUpdateException("Invalid non-parseable JSON patch file in " + patchFile.getAbsolutePath());
- }
- }
-
- LOG.info("Processed " + patchFiles.length + " patch files.");
- }
-
- private static Map<String, AtlasTypePatch> initializePatchHandlerMap(DefaultMetadataService metadataService, TypeSystem typeSystem) {
- Map<String, AtlasTypePatch> patchHandlerMap = new HashMap<String, AtlasTypePatch>();
- List<AtlasTypePatch> patchers = new ArrayList<AtlasTypePatch>();
-
- // Register new patch classes here
- patchers.add(new AtlasTypeAttributePatch(metadataService, typeSystem));
-
- for (AtlasTypePatch patcher : patchers) {
- for (String action : patcher.getSupportedActions()) {
- patchHandlerMap.put(action, patcher);
- }
- }
-
- return patchHandlerMap;
- }
-
- public static Gson initializeGson() {
- GsonBuilder gsonBuilder = new GsonBuilder();
- gsonBuilder.registerTypeAdapter(Multiplicity.class, new MultiplicityDeserializer());
- gsonBuilder.setFieldNamingPolicy(FieldNamingPolicy.IDENTITY);
- Gson gson = gsonBuilder.create();
-
- return gson;
- }
-
- static class MultiplicityDeserializer implements JsonDeserializer<Multiplicity> {
- @Override
- public Multiplicity deserialize(JsonElement json, Type typeOfT, JsonDeserializationContext context)
- throws JsonParseException {
- String multiplicityString = json.getAsString().toLowerCase();
- Multiplicity m = null;
- switch (multiplicityString) {
- case "optional":
- m = Multiplicity.OPTIONAL;
- break;
- case "required":
- m = Multiplicity.REQUIRED;
- break;
- case "collection":
- m = Multiplicity.COLLECTION;
- break;
- case "set":
- m = Multiplicity.SET;
- break;
- default:
- break;
- }
- return m;
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
deleted file mode 100644
index b918c87..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasTypeAttributePatch.java
+++ /dev/null
@@ -1,296 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import com.google.common.collect.ImmutableList;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.codehaus.jettison.json.JSONException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-public class AtlasTypeAttributePatch extends AtlasTypePatch {
-
- private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeAttributePatch.class);
- private static final String STRUCT_TYPE = "STRUCT";
- private static final String CLASS_TYPE = "CLASS";
- private static final String TRAIT_TYPE = "TRAIT";
- private static final String[] SUPPORTED_ACTIONS = new String[] { "ADD_ATTRIBUTE", "UPDATE_ATTRIBUTE", "DELETE_ATTRIBUTE" };
-
- public AtlasTypeAttributePatch(DefaultMetadataService metadataService, TypeSystem typeSystem) {
- super(metadataService, typeSystem, SUPPORTED_ACTIONS);
- }
-
- /********* SAMPLE PATCH DATA ***********
- *
- *
- {
- "patches": [
- {
- "action": "ADD_ATTRIBUTE",
- "typeName": "hive_column",
- "applyToVersion": "1.0",
- "updateToVersion": "2.0",
- "params": {
- "optionalParam1" : "true",
- "optionalParam2" : "false"
- },
- "attributeDefinitions": [
- {
- "name": "position",
- "dataTypeName": "string",
- "multiplicity": "optional",
- "isComposite": false,
- "isUnique": false,
- "isIndexable": false,
- "reverseAttributeName": null
- }
- ]
- }
- ]
- }
- *
- */
-
- @Override
- public PatchResult applyPatch(PatchData patchData) {
- String typeName = patchData.getTypeName();
- String applyVersion = patchData.getApplyToVersion();
- TypesDef updatedTypeDef;
- PatchResult result;
- try {
- // Check if type exists in type system
- if (checkIfTypeExists(typeName, metadataService)) {
- TypesDef typesDef = getTypeDef(typeName);
- String currentVersion = getTypeVersion(typeName);
-
- // Check version to apply patch
- if (currentVersion == null || currentVersion.equalsIgnoreCase(applyVersion) || currentVersion.startsWith(applyVersion + ".")) {
- updatedTypeDef = updateTypesDef(typesDef, patchData);
-
- if (updatedTypeDef != null) {
- metadataService.updateType(TypesSerialization.toJson(updatedTypeDef));
- LOG.info("updated " + patchData.getTypeName() + " type from version " + patchData.getApplyToVersion() + " to " + patchData.getUpdateToVersion());
- result = new PatchResult("patch applied successfully!", PatchStatus.SUCCESS);
- } else {
- LOG.error("failed to create updated typedef for type=" +typeName+ "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() );
- result = new PatchResult("patch failed!", PatchStatus.FAILED);
- }
- } else {
- LOG.info("patch skipped for " + patchData.getTypeName());
- result = new PatchResult("patch skipped!", PatchStatus.SKIPPED);
- }
-
- } else {
- LOG.error("failed to apply patch (typeName=" + typeName + "; applyToVersion=" + applyVersion + "; updateToVersion=" + patchData.getUpdateToVersion() + "): type doesn't exist");
- result = new PatchResult("patch failed!", PatchStatus.FAILED);
- }
-
- } catch (AtlasException e) {
- LOG.error("error in updating type for " + patchData.getTypeName());
- result = new PatchResult("unable to update type", PatchStatus.FAILED);
- } catch (JSONException e) {
- LOG.error("error in updating typedef for " + patchData.getTypeName());
- result = new PatchResult("unable to update typedef", PatchStatus.FAILED);
- }
-
- return result;
- }
-
- public TypesDef updateTypesDef(TypesDef typesDef, PatchData patchData) throws AtlasException, JSONException {
- AttributeDefinition[] typeAttributes = getAttributesFromTypesDef(typesDef, patchData.getTypeName());
- AttributeDefinition[] patchAttributes = patchData.getAttributeDefinitions();
- AttributeDefinition[] newAttributes = new AttributeDefinition[0];
- String patchAction = patchData.getAction();
- TypesDef newTypeDef = null;
-
- if (patchAction != null && typeAttributes != null && patchAttributes != null) {
- switch (patchAction) {
- case "ADD_ATTRIBUTE":
- LOG.info("adding new attribute to type {}", patchData.getTypeName());
- newAttributes = addAttributes(typeAttributes, patchAttributes);
- break;
- case "DELETE_ATTRIBUTE":
- LOG.info("deleting attribute from type {}", patchData.getTypeName());
- newAttributes = deleteAttributes(typeAttributes, patchAttributes);
- break;
- case "UPDATE_ATTRIBUTE":
- LOG.info("updating attribute in type {}", patchData.getTypeName());
- newAttributes = updateAttributes(typeAttributes, patchAttributes);
- break;
- default:
- LOG.info("invalid action " + patchAction + ", supported actions: " + Arrays.toString(SUPPORTED_ACTIONS));
- break;
- }
-
- newTypeDef = createTypeDefWithNewAttributes(typesDef, patchData.getTypeName(), newAttributes, patchData.getUpdateToVersion());
- }
-
- return newTypeDef;
- }
-
- private AttributeDefinition[] addAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>(Arrays.asList(typeAttributes));
- Map<String, AttributeDefinition> typeAttrs = getAttributeNamesFromDefinition(typeAttributes);
-
- for (AttributeDefinition attr : patchAttributes) {
- if (!typeAttrs.containsKey(attr.name))
- newAttrList.add(attr);
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private AttributeDefinition[] deleteAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
- Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
-
- for (AttributeDefinition attr : typeAttributes) {
- if (!patchAttrs.containsKey(attr.name))
- newAttrList.add(attr);
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private AttributeDefinition[] updateAttributes(AttributeDefinition[] typeAttributes, AttributeDefinition[] patchAttributes) {
- ArrayList<AttributeDefinition> newAttrList = new ArrayList<AttributeDefinition>();
- Map<String, AttributeDefinition> patchAttrs = getAttributeNamesFromDefinition(patchAttributes);
- AttributeDefinition newAttr;
-
- for (AttributeDefinition attr : typeAttributes) {
- newAttr = patchAttrs.get(attr.name);
- if (patchAttrs.containsKey(attr.name) && checkIfAttributeUpdatable(attr, newAttr)) {
- newAttrList.add(newAttr);
- } else {
- newAttrList.add(attr);
- }
- }
-
- return newAttrList.toArray(new AttributeDefinition[newAttrList.size()]);
- }
-
- private TypesDef createTypeDefWithNewAttributes(TypesDef typesDef, String typeName, AttributeDefinition[] newAttributes, String newVersion) throws AtlasException {
- ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
- ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
- ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
- String dataType = getDataType(typeName).toUpperCase();
- switch (dataType) {
- case STRUCT_TYPE:
- StructTypeDefinition structType = typesDef.structTypesAsJavaList().get(0);
- structs.add(new StructTypeDefinition(structType.typeName, structType.typeDescription, newVersion, newAttributes));
- break;
- case CLASS_TYPE:
- HierarchicalTypeDefinition<ClassType> classType = typesDef.classTypesAsJavaList().get(0);
- classTypes.add(new HierarchicalTypeDefinition(ClassType.class, classType.typeName, classType.typeDescription, newVersion, classType.superTypes, newAttributes));
- break;
- case TRAIT_TYPE:
- HierarchicalTypeDefinition<TraitType> traitType = typesDef.traitTypesAsJavaList().get(0);
- traits.add(new HierarchicalTypeDefinition(TraitType.class, traitType.typeName, traitType.typeDescription, newVersion, traitType.superTypes, newAttributes));
- break;
- default:
- LOG.error("unhandled datatype {} when creating new typedef", dataType);
- }
-
- return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
- }
-
- private AttributeDefinition[] getAttributesFromTypesDef(TypesDef typesDef, String typeName) throws AtlasException {
- AttributeDefinition[] typeAttributes = null;
- String dataType = getDataType(typeName).toUpperCase();
- switch (dataType) {
- case STRUCT_TYPE:
- typeAttributes = typesDef.structTypesAsJavaList().get(0).attributeDefinitions;
- break;
- case CLASS_TYPE:
- typeAttributes = typesDef.classTypesAsJavaList().get(0).attributeDefinitions;
- break;
- case TRAIT_TYPE:
- typeAttributes = typesDef.traitTypesAsJavaList().get(0).attributeDefinitions;
- break;
- default:
- LOG.error("unhandled datatype {}", dataType);
- }
-
- return typeAttributes;
- }
-
- private Map<String, AttributeDefinition> getAttributeNamesFromDefinition(AttributeDefinition[] attribDef) {
- Map<String, AttributeDefinition> attrsMap = new HashMap<String, AttributeDefinition>();
- for (AttributeDefinition attr : attribDef) {
- attrsMap.put(attr.name, attr);
- }
-
- return attrsMap;
- }
-
- private boolean checkIfAttributeUpdatable(AttributeDefinition attr, AttributeDefinition newAttr) {
- boolean result = false;
- if (!attr.equals(newAttr) && (attr.multiplicity == Multiplicity.REQUIRED
- && newAttr.multiplicity == Multiplicity.OPTIONAL)) {
- result = true;
- }
-
- return result;
- }
-
- // Returns the datatype the typename belongs to - PRIMITIVE, ENUM, ARRAY, MAP, STRUCT, TRAIT, CLASS
- private String getDataType(String typeName) throws AtlasException {
- IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
- return dataType.getTypeCategory().toString();
- }
-
- private String getTypeVersion(String typeName) throws AtlasException {
- return typeSystem.getDataType(IDataType.class, typeName).getVersion();
- }
-
- private TypesDef getTypeDef(String typeName) throws AtlasException {
- return TypesSerialization.fromJson(metadataService.getTypeDefinition(typeName));
- }
-
- private static boolean checkIfTypeExists(String typeName, DefaultMetadataService metadataService) {
- boolean result = true;
- try {
- metadataService.getTypeDefinition(typeName);
- } catch (AtlasException e) {
- result = false;
- }
-
- return result;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java b/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
deleted file mode 100644
index 0698efb..0000000
--- a/repository/src/main/java/org/apache/atlas/services/AtlasTypePatch.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.TypeSystem;
-
-import java.util.Map;
-
-
-public abstract class AtlasTypePatch {
- protected final TypeSystem typeSystem;
- protected final DefaultMetadataService metadataService;
- protected final String[] supportedActions;
-
- protected AtlasTypePatch(DefaultMetadataService metadataService, TypeSystem typeSystem, String[] supportedActions) {
- this.metadataService = metadataService;
- this.typeSystem = typeSystem;
- this.supportedActions = supportedActions;
- }
-
- public final String[] getSupportedActions() { return supportedActions; }
-
- public abstract PatchResult applyPatch(PatchData patch);
-
- public enum PatchStatus { SUCCESS, FAILED, SKIPPED }
-
- public class PatchResult {
- private String message;
- private PatchStatus status;
-
- public PatchResult(String message, PatchStatus status) {
- this.message = message;
- this.status = status;
- }
-
- public String getMessage() { return message; }
-
- public void setMessage(String message) { this.message = message; }
-
- public PatchStatus getStatus() { return status; }
-
- public void setStatus(PatchStatus status) { this.status = status; }
-
- }
-
- /**
- * A class to capture patch content.
- */
- public class PatchContent {
- private PatchData[] patches;
-
- public PatchData[] getPatches() {
- return patches;
- }
- }
-
- public static class PatchData {
- private String action;
- private String typeName;
- private String applyToVersion;
- private String updateToVersion;
- private Map<String, String> params;
- private AttributeDefinition[] attributeDefinitions;
-
- public PatchData(String action, String typeName, String applyToVersion, String updateToVersion, Map<String, String> params, AttributeDefinition[] attributeDefinitions) {
- this.action = action;
- this.typeName = typeName;
- this.applyToVersion = applyToVersion;
- this.updateToVersion = updateToVersion;
- this.params = params;
- this.attributeDefinitions = attributeDefinitions;
- }
-
- public String getAction() { return action; }
-
- public String getTypeName() { return typeName; }
-
- public String getApplyToVersion() { return applyToVersion; }
-
- public String getUpdateToVersion() { return updateToVersion; }
-
- public Map<String, String> getParams() { return params; }
-
- public AttributeDefinition[] getAttributeDefinitions() { return attributeDefinitions; }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 95c3dd9..69e8d12 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -20,7 +20,6 @@ package org.apache.atlas.services;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
import org.apache.atlas.ApplicationProperties;
@@ -29,7 +28,6 @@ import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.RequestContext;
-import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
import org.apache.atlas.listener.ActiveStateChangeHandler;
@@ -54,7 +52,6 @@ import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
@@ -83,9 +80,6 @@ import java.util.Map;
import javax.inject.Inject;
import javax.inject.Singleton;
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_INPUTS;
-import static org.apache.atlas.AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS;
-
/**
@@ -103,35 +97,29 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private final TypeSystem typeSystem;
private final MetadataRepository repository;
private final ITypeStore typeStore;
- private IBootstrapTypesRegistrar typesRegistrar;
private final Collection<TypesChangeListener> typeChangeListeners = new LinkedHashSet<>();
private final Collection<EntityChangeListener> entityChangeListeners = new LinkedHashSet<>();
- private boolean wasInitialized = false;
-
@Inject
private EntityAuditRepository auditRepository;
@Inject
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
- final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException {
- this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
+ this(repository, typeStore, typeListenerProviders, entityListenerProviders,
TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
//for testing only
public DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
- final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem,
final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore;
- this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
/**
* Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
@@ -163,70 +151,20 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
private void restoreTypeSystem() throws AtlasException {
LOG.info("Restoring type system from the store");
+
TypesDef typesDef = typeStore.restore();
- if (!wasInitialized) {
- LOG.info("Initializing type system for the first time.");
- typeSystem.defineTypes(typesDef);
-
- // restore types before creating super types
- createSuperTypes();
- typesRegistrar.registerTypes(ReservedTypesRegistrar.getTypesDir(), typeSystem, this);
- wasInitialized = true;
- } else {
- LOG.info("Type system was already initialized, refreshing cache.");
- refreshCache(typesDef);
- }
+
+ refreshCache(typesDef);
+
LOG.info("Restored type system from the store");
}
private void refreshCache(TypesDef typesDef) throws AtlasException {
- TypeSystem.TransientTypeSystem transientTypeSystem
- = typeSystem.createTransientTypeSystem(typesDef, true);
- Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
- LOG.info("Number of types got from transient type system: " + typesAdded.size());
- typeSystem.commitTypes(typesAdded);
- }
-
- @InterfaceAudience.Private
- private void createSuperTypes() throws AtlasException {
- HierarchicalTypeDefinition<ClassType> referenceableType = TypesUtil
- .createClassTypeDef(AtlasClient.REFERENCEABLE_SUPER_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, true, true, null));
- createType(referenceableType);
-
- HierarchicalTypeDefinition<ClassType> assetType = TypesUtil
- .createClassTypeDef(AtlasClient.ASSET_TYPE, ImmutableSet.<String>of(),
- new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- TypesUtil.createOptionalAttrDef(AtlasClient.DESCRIPTION, DataTypes.STRING_TYPE),
- new AttributeDefinition(AtlasClient.OWNER, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null));
- createType(assetType);
-
- HierarchicalTypeDefinition<ClassType> infraType = TypesUtil
- .createClassTypeDef(AtlasClient.INFRASTRUCTURE_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
- createType(infraType);
-
- HierarchicalTypeDefinition<ClassType> datasetType = TypesUtil
- .createClassTypeDef(AtlasClient.DATA_SET_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE));
- createType(datasetType);
-
- HierarchicalTypeDefinition<ClassType> processType = TypesUtil
- .createClassTypeDef(AtlasClient.PROCESS_SUPER_TYPE,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE),
- new AttributeDefinition(PROCESS_ATTRIBUTE_INPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PROCESS_ATTRIBUTE_OUTPUTS, DataTypes.arrayTypeName(AtlasClient.DATA_SET_SUPER_TYPE),
- Multiplicity.OPTIONAL, false, null));
- createType(processType);
- }
-
- private void createType(HierarchicalTypeDefinition<ClassType> type) throws AtlasException {
- if (!typeSystem.isRegistered(type.typeName)) {
- TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- ImmutableList.<HierarchicalTypeDefinition<TraitType>>of(),
- ImmutableList.of(type));
- createType(TypesSerialization.toJson(typesDef));
+ if (typesDef != null && !typesDef.isEmpty()) {
+ TypeSystem.TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(typesDef, true);
+ Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
+ LOG.info("Number of types got from transient type system: " + typesAdded.size());
+ typeSystem.commitTypes(typesAdded);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java b/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
deleted file mode 100644
index fce5cb3..0000000
--- a/repository/src/main/java/org/apache/atlas/services/IBootstrapTypesRegistrar.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.types.TypeSystem;
-
-public interface IBootstrapTypesRegistrar {
- void registerTypes(String typesDirName, TypeSystem typeSystem, MetadataService metadataService)
- throws AtlasException;
-}
[3/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
index b6f55a1..40866fe 100755
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/hook/HiveHookIT.java
@@ -23,10 +23,8 @@ import com.google.common.collect.ImmutableList;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.HiveITBase;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
@@ -45,7 +43,6 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.HiveOperation;
import org.apache.hadoop.hive.ql.session.SessionState;
import org.apache.hadoop.security.UserGroupInformation;
-import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.slf4j.Logger;
@@ -80,7 +77,7 @@ public class HiveHookIT extends HiveITBase {
String dbId = assertDatabaseIsRegistered(dbName);
Referenceable definition = atlasClient.getEntity(dbId);
- Map params = (Map) definition.get(HiveDataModelGenerator.PARAMETERS);
+ Map params = (Map) definition.get(HiveMetaStoreBridge.PARAMETERS);
Assert.assertNotNull(params);
Assert.assertEquals(params.size(), 2);
Assert.assertEquals(params.get("p1"), "v1");
@@ -150,8 +147,8 @@ public class HiveHookIT extends HiveITBase {
Referenceable colEntity = atlasClient.getEntity(colId);
Assert.assertEquals(colEntity.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), String.format("%s.%s.%s@%s", dbName.toLowerCase(),
tableName.toLowerCase(), colName.toLowerCase(), CLUSTER_NAME));
- Assert.assertNotNull(colEntity.get(HiveDataModelGenerator.TABLE));
- Assert.assertEquals(((Id) colEntity.get(HiveDataModelGenerator.TABLE))._getId(), tableId);
+ Assert.assertNotNull(colEntity.get(HiveMetaStoreBridge.TABLE));
+ Assert.assertEquals(((Id) colEntity.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
//assert that column.owner = table.owner
Referenceable tableRef = atlasClient.getEntity(tableId);
@@ -161,8 +158,8 @@ public class HiveHookIT extends HiveITBase {
tableName = createTable();
tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
tableRef = atlasClient.getEntity(tableId);
- Assert.assertEquals(tableRef.get(HiveDataModelGenerator.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name());
- Assert.assertEquals(tableRef.get(HiveDataModelGenerator.COMMENT), "table comment");
+ Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.TABLE_TYPE_ATTR), TableType.MANAGED_TABLE.name());
+ Assert.assertEquals(tableRef.get(HiveMetaStoreBridge.COMMENT), "table comment");
String entityName = HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName);
Assert.assertEquals(tableRef.get(AtlasClient.NAME), tableName.toLowerCase());
Assert.assertEquals(tableRef.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME), entityName);
@@ -170,13 +167,13 @@ public class HiveHookIT extends HiveITBase {
Table t = hiveMetaStoreBridge.hiveClient.getTable(DEFAULT_DB, tableName);
long createTime = Long.parseLong(t.getMetadata().getProperty(hive_metastoreConstants.DDL_TIME)) * HiveMetaStoreBridge.MILLIS_CONVERT_FACTOR;
- verifyTimestamps(tableRef, HiveDataModelGenerator.CREATE_TIME, createTime);
- verifyTimestamps(tableRef, HiveDataModelGenerator.LAST_ACCESS_TIME, createTime);
+ verifyTimestamps(tableRef, HiveMetaStoreBridge.CREATE_TIME, createTime);
+ verifyTimestamps(tableRef, HiveMetaStoreBridge.LAST_ACCESS_TIME, createTime);
- final Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
- Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS), false);
- Assert.assertNotNull(sdRef.get(HiveDataModelGenerator.TABLE));
- Assert.assertEquals(((Id) sdRef.get(HiveDataModelGenerator.TABLE))._getId(), tableId);
+ final Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+ Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_IS_STORED_AS_SUB_DIRS), false);
+ Assert.assertNotNull(sdRef.get(HiveMetaStoreBridge.TABLE));
+ Assert.assertEquals(((Id) sdRef.get(HiveMetaStoreBridge.TABLE))._getId(), tableId);
//Create table where database doesn't exist, will create database instance as well
assertDatabaseIsRegistered(DEFAULT_DB);
@@ -878,7 +875,7 @@ public class HiveHookIT extends HiveITBase {
String tableId = assertTableIsRegistered(DEFAULT_DB, tableName);
Referenceable tableEntity = atlasClient.getEntity(tableId);
- final String createTime = (String)tableEntity.get(HiveDataModelGenerator.CREATE_TIME);
+ final String createTime = (String)tableEntity.get(HiveMetaStoreBridge.CREATE_TIME);
Assert.assertNotNull(createTime);
String columnGuid = assertColumnIsRegistered(HiveMetaStoreBridge.getColumnQualifiedName(HiveMetaStoreBridge.getTableQualifiedName(CLUSTER_NAME, DEFAULT_DB, tableName), NAME));
@@ -916,10 +913,10 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(newDBName, newTableName, new AssertPredicate() {
@Override
public void assertOnEntity(final Referenceable entity) throws Exception {
- Referenceable sd = ((Referenceable) entity.get(HiveDataModelGenerator.STORAGE_DESC));
- String location = (String) sd.get(HiveDataModelGenerator.LOCATION);
+ Referenceable sd = ((Referenceable) entity.get(HiveMetaStoreBridge.STORAGE_DESC));
+ String location = (String) sd.get(HiveMetaStoreBridge.LOCATION);
assertTrue(location.contains(newTableName));
- Assert.assertEquals(entity.get(HiveDataModelGenerator.CREATE_TIME), createTime);
+ Assert.assertEquals(entity.get(HiveMetaStoreBridge.CREATE_TIME), createTime);
}
});
}
@@ -929,7 +926,7 @@ public class HiveHookIT extends HiveITBase {
Referenceable tableRef = atlasClient.getEntity(tableId);
//with soft delete, the deleted columns are returned as well. So, filter the deleted ones
- List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveDataModelGenerator.COLUMNS));
+ List<Referenceable> columns = ((List<Referenceable>) tableRef.get(HiveMetaStoreBridge.COLUMNS));
List<Referenceable> activeColumns = new ArrayList<>();
for (Referenceable col : columns) {
if (col.getId().getState() == Id.EntityState.ACTIVE) {
@@ -1046,7 +1043,7 @@ public class HiveHookIT extends HiveITBase {
assertColumnIsRegistered(newColQualifiedName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable entity) throws Exception {
- assertEquals(entity.get(HiveDataModelGenerator.COMMENT), comment);
+ assertEquals(entity.get(HiveMetaStoreBridge.COMMENT), comment);
}
});
@@ -1071,7 +1068,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable entity) throws Exception {
- List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS);
+ List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
assertEquals(columns.get(0).get(NAME), finalNewColName);
assertEquals(columns.get(1).get(NAME), "id");
}
@@ -1099,7 +1096,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable entity) throws Exception {
- List<Referenceable> columns = (List<Referenceable>) entity.get(HiveDataModelGenerator.COLUMNS);
+ List<Referenceable> columns = (List<Referenceable>) entity.get(HiveMetaStoreBridge.COLUMNS);
assertEquals(columns.get(1).get(NAME), finalNewColName2);
assertEquals(columns.get(0).get(NAME), "id");
}
@@ -1289,8 +1286,8 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable tableRef) throws Exception {
- Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
- Assert.assertEquals(new Path((String)sdRef.get(HiveDataModelGenerator.LOCATION)).toString(), new Path(testPath).toString());
+ Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+ Assert.assertEquals(new Path((String)sdRef.get(HiveMetaStoreBridge.LOCATION)).toString(), new Path(testPath).toString());
}
});
@@ -1313,18 +1310,18 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable tableRef) throws Exception {
- Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
- Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT),
+ Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+ Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT),
"org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
- Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT),
+ Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT),
"org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
Assert.assertNotNull(sdRef.get("serdeInfo"));
Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
Assert.assertEquals(serdeInfo.get("serializationLib"), "org.apache.hadoop.hive.ql.io.orc.OrcSerde");
- Assert.assertNotNull(serdeInfo.get(HiveDataModelGenerator.PARAMETERS));
+ Assert.assertNotNull(serdeInfo.get(HiveMetaStoreBridge.PARAMETERS));
Assert.assertEquals(
- ((Map<String, String>) serdeInfo.get(HiveDataModelGenerator.PARAMETERS))
+ ((Map<String, String>) serdeInfo.get(HiveMetaStoreBridge.PARAMETERS))
.get("serialization.format"),
"1");
}
@@ -1337,10 +1334,10 @@ public class HiveHookIT extends HiveITBase {
* runCommand(query);
* tableRef = atlasClient.getEntity(tableId);
- * sdRef = (Referenceable)tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
- * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
- * Assert.assertEquals(sdRef.get(HiveDataModelGenerator.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
- * Assert.assertEquals(((Map) sdRef.get(HiveDataModelGenerator.PARAMETERS)).get("orc.compress"), "ZLIB");
+ * sdRef = (Referenceable)tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+ * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_INPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat");
+ * Assert.assertEquals(sdRef.get(HiveMetaStoreBridge.STORAGE_DESC_OUTPUT_FMT), "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat");
+ * Assert.assertEquals(((Map) sdRef.get(HiveMetaStoreBridge.PARAMETERS)).get("orc.compress"), "ZLIB");
*/
}
@@ -1375,8 +1372,8 @@ public class HiveHookIT extends HiveITBase {
private void verifyBucketSortingProperties(Referenceable tableRef, int numBuckets,
ImmutableList<String> bucketColNames,
ImmutableList<String> sortcolNames) throws Exception {
- Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
- Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveDataModelGenerator.STORAGE_NUM_BUCKETS)).intValue(),
+ Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
+ Assert.assertEquals(((scala.math.BigInt) sdRef.get(HiveMetaStoreBridge.STORAGE_NUM_BUCKETS)).intValue(),
numBuckets);
Assert.assertEquals(sdRef.get("bucketCols"), bucketColNames);
@@ -1661,7 +1658,7 @@ public class HiveHookIT extends HiveITBase {
assertTableIsRegistered(DEFAULT_DB, tableName, new AssertPredicate() {
@Override
public void assertOnEntity(Referenceable tableRef) throws Exception {
- Referenceable sdRef = (Referenceable) tableRef.get(HiveDataModelGenerator.STORAGE_DESC);
+ Referenceable sdRef = (Referenceable) tableRef.get(HiveMetaStoreBridge.STORAGE_DESC);
Struct serdeInfo = (Struct) sdRef.get("serdeInfo");
Assert.assertEquals(serdeInfo.get("serializationLib"), serdeLib);
verifyProperties(serdeInfo, expectedProps, false);
@@ -1670,7 +1667,7 @@ public class HiveHookIT extends HiveITBase {
}
private void verifyProperties(Struct referenceable, Map<String, String> expectedProps, boolean checkIfNotExists) {
- Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveDataModelGenerator.PARAMETERS);
+ Map<String, String> parameters = (Map<String, String>) referenceable.get(HiveMetaStoreBridge.PARAMETERS);
if (checkIfNotExists == false) {
//Check if properties exist
@@ -1744,7 +1741,7 @@ public class HiveHookIT extends HiveITBase {
}
private String getDSTypeName(Entity entity) {
- return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : FSDataTypes.HDFS_PATH().toString();
+ return Entity.Type.TABLE.equals(entity.getType()) ? HiveDataTypes.HIVE_TABLE.name() : HiveMetaStoreBridge.HDFS_PATH.toString();
}
private <T extends Entity> SortedMap<T, Referenceable> getSortedProcessDataSets(Set<T> inputTbls) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0010-base_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0010-base_model.json b/addons/models/0010-base_model.json
new file mode 100644
index 0000000..7f64d85
--- /dev/null
+++ b/addons/models/0010-base_model.json
@@ -0,0 +1,97 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "Referenceable",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "qualifiedName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ }
+ ]
+ },
+ {
+ "name": "Asset",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "owner",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "DataSet",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Infrastructure",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "Process",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<DataSet>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0020-fs_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0020-fs_model.json b/addons/models/0020-fs_model.json
new file mode 100644
index 0000000..ff17261
--- /dev/null
+++ b/addons/models/0020-fs_model.json
@@ -0,0 +1,191 @@
+{
+ "enumDefs": [
+ {
+ "name": "file_action",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "ordinal": 0,
+ "value": "NONE"
+ },
+ {
+ "ordinal": 1,
+ "value": "EXECUTE"
+ },
+ {
+ "ordinal": 2,
+ "value": "WRITE"
+ },
+ {
+ "ordinal": 3,
+ "value": "WRITE_EXECUTE"
+ },
+ {
+ "ordinal": 4,
+ "value": "READ"
+ },
+ {
+ "ordinal": 5,
+ "value": "READ_EXECUTE"
+ },
+ {
+ "ordinal": 6,
+ "value": "READ_WRITE"
+ },
+ {
+ "ordinal": 7,
+ "value": "ALL"
+ }
+ ]
+ }
+ ],
+ "structDefs": [
+ {
+ "name": "fs_permissions",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "group",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "user",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "sticky",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "others",
+ "typeName": "file_action",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "fs_path",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "path",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isFile",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "isSymlink",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "fileSize",
+ "typeName": "long",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "group",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "posixPermissions",
+ "typeName": "fs_permissions",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hdfs_path",
+ "superTypes": [
+ "fs_path"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "numberOfReplicas",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "extendedAttributes",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0030-hive_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0030-hive_model.json b/addons/models/0030-hive_model.json
new file mode 100644
index 0000000..2c2e9ed
--- /dev/null
+++ b/addons/models/0030-hive_model.json
@@ -0,0 +1,498 @@
+{
+ "enumDefs": [
+ {
+ "name": "hive_principal_type",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "ordinal": 1,
+ "value": "USER"
+ },
+ {
+ "ordinal": 2,
+ "value": "ROLE"
+ },
+ {
+ "ordinal": 3,
+ "value": "GROUP"
+ }
+ ]
+ }
+ ],
+ "structDefs": [
+ {
+ "name": "hive_order",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "order",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "col",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_serde",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "serializationLib",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "hive_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "operationType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryPlan",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "queryId",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "recentQueries",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "queryGraph",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_table",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "db",
+ "typeName": "hive_db",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "lastAccessTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "retention",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "sd",
+ "typeName": "hive_storagedesc",
+ "cardinality": "SINGLE",
+ "constraintDefs": [
+ {
+ "type": "mappedFromRef",
+ "params": {
+ "refAttribute": "table"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "partitionKeys",
+ "typeName": "array<hive_column>",
+ "cardinality": "SINGLE",
+ "constraintDefs": [
+ {
+ "type": "foreignKey"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "aliases",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "columns",
+ "typeName": "array<hive_column>",
+ "cardinality": "SINGLE",
+ "constraintDefs": [
+ {
+ "type": "mappedFromRef",
+ "params": {
+ "refAttribute": "table"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "viewOriginalText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "viewExpandedText",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "tableType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "temporary",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_storagedesc",
+ "superTypes": [
+ "Referenceable"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "cardinality": "SINGLE",
+ "constraintDefs": [
+ {
+ "type": "foreignKey",
+ "params": {
+ "onDelete": "cascade"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "inputFormat",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "outputFormat",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "compressed",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "numBuckets",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "serdeInfo",
+ "typeName": "hive_serde",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "bucketCols",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "sortCols",
+ "typeName": "array<hive_order>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "storedAsSubDirectories",
+ "typeName": "boolean",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_db",
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "ownerType",
+ "typeName": "hive_principal_type",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "hive_column",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "type",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "cardinality": "SINGLE",
+ "constraintDefs": [
+ {
+ "type": "foreignKey",
+ "params": {
+ "onDelete": "cascade"
+ }
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0040-sqoop_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0040-sqoop_model.json b/addons/models/0040-sqoop_model.json
new file mode 100644
index 0000000..f5c7fd9
--- /dev/null
+++ b/addons/models/0040-sqoop_model.json
@@ -0,0 +1,97 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "sqoop_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "operation",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "commandlineOpts",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "sqoop_dbdatastore",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "dbStoreType",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "storeUse",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "storeUri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "source",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0050-falcon_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0050-falcon_model.json b/addons/models/0050-falcon_model.json
new file mode 100644
index 0000000..b7398ef
--- /dev/null
+++ b/addons/models/0050-falcon_model.json
@@ -0,0 +1,147 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "falcon_feed_replication",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": []
+ },
+ {
+ "name": "falcon_cluster",
+ "superTypes": [
+ "Infrastructure"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "colo",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_feed",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "groups",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_process",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "runs-on",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "pipelines",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "workflow-properties",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "falcon_feed_creation",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0060-hbase_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0060-hbase_model.json b/addons/models/0060-hbase_model.json
new file mode 100644
index 0000000..d249d87
--- /dev/null
+++ b/addons/models/0060-hbase_model.json
@@ -0,0 +1,24 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "hbase_table",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0070-kafka_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0070-kafka_model.json b/addons/models/0070-kafka_model.json
new file mode 100644
index 0000000..b7f6e33
--- /dev/null
+++ b/addons/models/0070-kafka_model.json
@@ -0,0 +1,57 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "kafka_topic",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "jms_topic",
+ "superTypes": [
+ "DataSet"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/0080-storm_model.json
----------------------------------------------------------------------
diff --git a/addons/models/0080-storm_model.json b/addons/models/0080-storm_model.json
new file mode 100644
index 0000000..59f3228
--- /dev/null
+++ b/addons/models/0080-storm_model.json
@@ -0,0 +1,150 @@
+{
+ "enumDefs": [],
+ "structDefs": [],
+ "classificationDefs": [],
+ "entityDefs": [
+ {
+ "name": "storm_topology",
+ "superTypes": [
+ "Process"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "id",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": true
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "nodes",
+ "typeName": "array<storm_node>",
+ "cardinality": "LIST",
+ "constraintDefs": [
+ {
+ "type": "foreignKey"
+ }
+ ],
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_node",
+ "superTypes": [],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": true,
+ "isUnique": false
+ },
+ {
+ "name": "driverClass",
+ "typeName": "string",
+ "cardinality": "SINGLE",
+ "isIndexable": true,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_spout",
+ "superTypes": [
+ "storm_node"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "cardinality": "LIST",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ }
+ ]
+ },
+ {
+ "name": "storm_bolt",
+ "superTypes": [
+ "storm_node"
+ ],
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<string>",
+ "cardinality": "LIST",
+ "isIndexable": false,
+ "isOptional": false,
+ "isUnique": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/models/patches/001-hive_column_add_position.json
----------------------------------------------------------------------
diff --git a/addons/models/patches/001-hive_column_add_position.json b/addons/models/patches/001-hive_column_add_position.json
new file mode 100644
index 0000000..eeab74a
--- /dev/null
+++ b/addons/models/patches/001-hive_column_add_position.json
@@ -0,0 +1,21 @@
+{
+ "patches": [
+ {
+ "action": "ADD_ATTRIBUTE",
+ "typeName": "hive_column",
+ "applyToVersion": "1.0",
+ "updateToVersion": "1.1",
+ "params": null,
+ "attributeDefs": [
+ {
+ "name": "position",
+ "typeName": "int",
+ "cardinality": "SINGLE",
+ "isIndexable": false,
+ "isOptional": true,
+ "isUnique": false
+ }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/pom.xml b/addons/sqoop-bridge/pom.xml
index 9ac411e..e71d64a 100644
--- a/addons/sqoop-bridge/pom.xml
+++ b/addons/sqoop-bridge/pom.xml
@@ -411,24 +411,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.sqoop.model.SqoopDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/sqoop_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
index e43fc86..6fb27e5 100644
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
+++ b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/hook/SqoopHook.java
@@ -23,11 +23,9 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.notification.hook.HookNotification;
-import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
import org.apache.atlas.sqoop.model.SqoopDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.configuration.Configuration;
@@ -55,6 +53,20 @@ public class SqoopHook extends SqoopJobDataPublisher {
public static final String ATLAS_CLUSTER_NAME = "atlas.cluster.name";
public static final String DEFAULT_CLUSTER_NAME = "primary";
+ public static final String USER = "userName";
+ public static final String DB_STORE_TYPE = "dbStoreType";
+ public static final String DB_STORE_USAGE = "storeUse";
+ public static final String SOURCE = "source";
+ public static final String DESCRIPTION = "description";
+ public static final String STORE_URI = "storeUri";
+ public static final String OPERATION = "operation";
+ public static final String START_TIME = "startTime";
+ public static final String END_TIME = "endTime";
+ public static final String CMD_LINE_OPTS = "commandlineOpts";
+ // multiple inputs and outputs for process
+ public static final String INPUTS = "inputs";
+ public static final String OUTPUTS = "outputs";
+
static {
org.apache.hadoop.conf.Configuration.addDefaultResource("sqoop-site.xml");
}
@@ -75,7 +87,7 @@ public class SqoopHook extends SqoopJobDataPublisher {
tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
- tableRef.set(HiveDataModelGenerator.DB, dbRef);
+ tableRef.set(HiveMetaStoreBridge.DB, dbRef);
return tableRef;
}
@@ -94,11 +106,11 @@ public class SqoopHook extends SqoopJobDataPublisher {
String name = getSqoopDBStoreName(data);
storeRef.set(AtlasClient.NAME, name);
storeRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, name);
- storeRef.set(SqoopDataModelGenerator.DB_STORE_TYPE, data.getStoreType());
- storeRef.set(SqoopDataModelGenerator.DB_STORE_USAGE, usage);
- storeRef.set(SqoopDataModelGenerator.STORE_URI, data.getUrl());
- storeRef.set(SqoopDataModelGenerator.SOURCE, source);
- storeRef.set(SqoopDataModelGenerator.DESCRIPTION, "");
+ storeRef.set(SqoopHook.DB_STORE_TYPE, data.getStoreType());
+ storeRef.set(SqoopHook.DB_STORE_USAGE, usage);
+ storeRef.set(SqoopHook.STORE_URI, data.getUrl());
+ storeRef.set(SqoopHook.SOURCE, source);
+ storeRef.set(SqoopHook.DESCRIPTION, "");
storeRef.set(AtlasClient.OWNER, data.getUser());
return storeRef;
}
@@ -109,24 +121,24 @@ public class SqoopHook extends SqoopJobDataPublisher {
final String sqoopProcessName = getSqoopProcessName(data, clusterName);
procRef.set(AtlasClient.NAME, sqoopProcessName);
procRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, sqoopProcessName);
- procRef.set(SqoopDataModelGenerator.OPERATION, data.getOperation());
+ procRef.set(SqoopHook.OPERATION, data.getOperation());
if (isImportOperation(data)) {
- procRef.set(SqoopDataModelGenerator.INPUTS, dbStoreRef);
- procRef.set(SqoopDataModelGenerator.OUTPUTS, hiveTableRef);
+ procRef.set(SqoopHook.INPUTS, dbStoreRef);
+ procRef.set(SqoopHook.OUTPUTS, hiveTableRef);
} else {
- procRef.set(SqoopDataModelGenerator.INPUTS, hiveTableRef);
- procRef.set(SqoopDataModelGenerator.OUTPUTS, dbStoreRef);
+ procRef.set(SqoopHook.INPUTS, hiveTableRef);
+ procRef.set(SqoopHook.OUTPUTS, dbStoreRef);
}
- procRef.set(SqoopDataModelGenerator.USER, data.getUser());
- procRef.set(SqoopDataModelGenerator.START_TIME, new Date(data.getStartTime()));
- procRef.set(SqoopDataModelGenerator.END_TIME, new Date(data.getEndTime()));
+ procRef.set(SqoopHook.USER, data.getUser());
+ procRef.set(SqoopHook.START_TIME, new Date(data.getStartTime()));
+ procRef.set(SqoopHook.END_TIME, new Date(data.getEndTime()));
Map<String, String> sqoopOptionsMap = new HashMap<>();
Properties options = data.getOptions();
for (Object k : options.keySet()) {
sqoopOptionsMap.put((String)k, (String) options.get(k));
}
- procRef.set(SqoopDataModelGenerator.CMD_LINE_OPTS, sqoopOptionsMap);
+ procRef.set(SqoopHook.CMD_LINE_OPTS, sqoopOptionsMap);
return procRef;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java b/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
deleted file mode 100644
index 3373246..0000000
--- a/addons/sqoop-bridge/src/main/java/org/apache/atlas/sqoop/model/SqoopDataModelGenerator.java
+++ /dev/null
@@ -1,187 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.sqoop.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates Sqoop data model for both metastore entities and DDL/DML queries.
- */
-public class SqoopDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(SqoopDataModelGenerator.class);
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
- private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
- private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
- private static final DataTypes.MapType STRING_MAP_TYPE =
- new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
- public static final String USER = "userName";
- public static final String DB_STORE_TYPE = "dbStoreType";
- public static final String DB_STORE_USAGE = "storeUse";
- public static final String SOURCE = "source";
- public static final String DESCRIPTION = "description";
- public static final String STORE_URI = "storeUri";
- public static final String OPERATION = "operation";
- public static final String START_TIME = "startTime";
- public static final String END_TIME = "endTime";
- public static final String CMD_LINE_OPTS = "commandlineOpts";
- // multiple inputs and outputs for process
- public static final String INPUTS = "inputs";
- public static final String OUTPUTS = "outputs";
-
- public SqoopDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- enumTypeDefinitionMap = new HashMap<>();
- structTypeDefinitionMap = new HashMap<>();
- }
-
- public void createDataModel() throws AtlasException {
- LOG.info("Generating the Sqoop Data Model....");
-
- // enums
-
- // structs
-
- // classes
- createSqoopDbStoreClass();
-
- // DDL/DML Process
- createSqoopProcessClass();
- }
-
- public TypesDef getTypesDef() {
- return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
- getClassTypeDefinitions());
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
- return ImmutableList.copyOf(enumTypeDefinitionMap.values());
- }
-
- public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
- return ImmutableList.copyOf(structTypeDefinitionMap.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createSqoopDbStoreClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(DB_STORE_TYPE,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- new AttributeDefinition(DB_STORE_USAGE,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(STORE_URI,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(SOURCE,
- DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null)
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_DBDATASTORE.getName(), null,
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(SqoopDataTypes.SQOOP_DBDATASTORE.getName(), definition);
- LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_DBDATASTORE.getName());
- }
-
-
- private void createSqoopProcessClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(OPERATION,
- DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- new AttributeDefinition(CMD_LINE_OPTS, STRING_MAP_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(START_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(END_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(USER,
- DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, SqoopDataTypes.SQOOP_PROCESS.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(SqoopDataTypes.SQOOP_PROCESS.getName(), definition);
- LOG.debug("Created definition for " + SqoopDataTypes.SQOOP_PROCESS.getName());
- }
-
- public String getModelAsJson() throws AtlasException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
- String modelAsJson = dataModelGenerator.getModelAsJson();
-
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
- return;
- }
-
- System.out.println("sqoopDataModelAsJSON = " + modelAsJson);
-
- TypesDef typesDef = dataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
-
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(
- String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
- ClassType.class.getSimpleName(), StringUtils.join(classType.superTypes, ","),
- Arrays.toString(classType.attributeDefinitions)));
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
----------------------------------------------------------------------
diff --git a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
index d4fb3af..b1bd115 100644
--- a/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
+++ b/addons/sqoop-bridge/src/test/java/org/apache/atlas/sqoop/hook/SqoopHookIT.java
@@ -18,17 +18,13 @@
package org.apache.atlas.sqoop.hook;
-import com.sun.jersey.api.client.ClientResponse;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes;
-import org.apache.atlas.sqoop.model.SqoopDataModelGenerator;
import org.apache.atlas.sqoop.model.SqoopDataTypes;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.configuration.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.sqoop.SqoopJobDataPublisher;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
@@ -54,28 +50,6 @@ public class SqoopHookIT {
} else {
atlasClient = new AtlasClient(configuration.getStringArray(HiveMetaStoreBridge.ATLAS_ENDPOINT));
}
- registerDataModels(atlasClient);
- }
-
- private void registerDataModels(AtlasClient client) throws Exception {
- // Make sure hive model exists
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
- SqoopDataModelGenerator dataModelGenerator = new SqoopDataModelGenerator();
-
- //Register sqoop data model if its not already registered
- try {
- client.getType(SqoopDataTypes.SQOOP_PROCESS.getName());
- LOG.info("Sqoop data model is already registered!");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Expected in case types do not exist
- LOG.info("Registering Sqoop data model");
- client.createType(dataModelGenerator.getModelAsJson());
- } else {
- throw ase;
- }
- }
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/pom.xml b/addons/storm-bridge/pom.xml
index fcb6e01..6be55b4 100644
--- a/addons/storm-bridge/pom.xml
+++ b/addons/storm-bridge/pom.xml
@@ -529,24 +529,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.storm.model.StormDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/storm_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
index bb6a476..5193832 100644
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
+++ b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/hook/StormAtlasHook.java
@@ -26,9 +26,7 @@ import org.apache.storm.generated.TopologyInfo;
import org.apache.storm.utils.Utils;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.storm.model.StormDataTypes;
import org.apache.atlas.typesystem.Referenceable;
@@ -213,7 +211,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
break;
case "HdfsBolt":
- dataSetReferenceable = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+ dataSetReferenceable = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
String hdfsUri = config.get("HdfsBolt.rotationActions") == null
? config.get("HdfsBolt.fileNameFormat.path")
: config.get("HdfsBolt.rotationActions");
@@ -241,7 +239,7 @@ public class StormAtlasHook extends AtlasHook implements ISubmitterHook {
final String tableQualifiedName = HiveMetaStoreBridge.getTableQualifiedName(clusterName,
databaseName, hiveTableName);
dataSetReferenceable.set(AtlasClient.NAME, hiveTableName);
- dataSetReferenceable.set(HiveDataModelGenerator.DB, dbReferenceable);
+ dataSetReferenceable.set(HiveMetaStoreBridge.DB, dbReferenceable);
dataSetReferenceable.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, tableQualifiedName);
break;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java b/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
deleted file mode 100644
index 95eead7..0000000
--- a/addons/storm-bridge/src/main/java/org/apache/atlas/storm/model/StormDataModelGenerator.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.storm.model;
-
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-
-import java.io.IOException;
-
-public class StormDataModelGenerator {
-
- public static void main(String[] args) throws IOException {
- StormDataModel.main(new String[]{});
- TypesDef typesDef = StormDataModel.typesDef();
- String stormTypesAsJSON = TypesSerialization.toJson(typesDef);
-
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], stormTypesAsJSON);
- return;
- }
-
- System.out.println("stormTypesAsJSON = " + stormTypesAsJSON);
- }
-}
[4/4] incubator-atlas git commit: ATLAS-1272: updated types bootstrap
to load from new format typedef JSON files
Posted by ma...@apache.org.
ATLAS-1272: updated types bootstrap to load from new format typedef JSON files
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/def9e385
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/def9e385
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/def9e385
Branch: refs/heads/master
Commit: def9e385cc17c2bceb9450c82947370fadc09c3c
Parents: 2ea3a45
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Sun Nov 6 19:32:16 2016 -0800
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Mon Nov 7 14:06:57 2016 -0800
----------------------------------------------------------------------
addons/falcon-bridge/pom.xml | 18 -
.../atlas/falcon/bridge/FalconBridge.java | 46 +-
.../falcon/model/FalconDataModelGenerator.java | 214 --------
.../apache/atlas/falcon/hook/FalconHookIT.java | 22 +-
addons/hdfs-model/pom.xml | 18 -
.../atlas/fs/model/FSDataModelGenerator.java | 38 --
.../org/apache/atlas/fs/model/FSDataModel.scala | 95 ----
.../apache/atlas/fs/model/HDFSModelTest.java | 83 ----
addons/hive-bridge/pom.xml | 18 -
.../atlas/hive/bridge/ColumnLineageUtils.java | 5 +-
.../atlas/hive/bridge/HiveMetaStoreBridge.java | 101 ++--
.../org/apache/atlas/hive/hook/HiveHook.java | 9 +-
.../hive/model/HiveDataModelGenerator.java | 350 -------------
.../java/org/apache/atlas/hive/HiveITBase.java | 4 +-
.../hive/bridge/HiveMetaStoreBridgeTest.java | 6 +-
.../org/apache/atlas/hive/hook/HiveHookIT.java | 73 ++-
addons/models/0010-base_model.json | 97 ++++
addons/models/0020-fs_model.json | 191 +++++++
addons/models/0030-hive_model.json | 498 +++++++++++++++++++
addons/models/0040-sqoop_model.json | 97 ++++
addons/models/0050-falcon_model.json | 147 ++++++
addons/models/0060-hbase_model.json | 24 +
addons/models/0070-kafka_model.json | 57 +++
addons/models/0080-storm_model.json | 150 ++++++
.../patches/001-hive_column_add_position.json | 21 +
addons/sqoop-bridge/pom.xml | 18 -
.../org/apache/atlas/sqoop/hook/SqoopHook.java | 46 +-
.../sqoop/model/SqoopDataModelGenerator.java | 187 -------
.../apache/atlas/sqoop/hook/SqoopHookIT.java | 26 -
addons/storm-bridge/pom.xml | 18 -
.../apache/atlas/storm/hook/StormAtlasHook.java | 6 +-
.../storm/model/StormDataModelGenerator.java | 41 --
.../atlas/storm/model/StormDataModel.scala | 104 ----
.../atlas/storm/hook/StormAtlasHookIT.java | 56 ---
.../src/main/assemblies/standalone-package.xml | 30 +-
.../java/org/apache/atlas/AtlasErrorCode.java | 4 +-
.../atlas/model/typedef/AtlasStructDef.java | 26 +-
.../atlas/model/typedef/AtlasTypesDef.java | 2 +
.../org/apache/atlas/type/AtlasStructType.java | 14 +-
.../apache/atlas/type/AtlasTypeRegistry.java | 8 +
.../apache/atlas/RepositoryMetadataModule.java | 4 -
.../graph/GraphBackedSearchIndexer.java | 4 +-
.../bootstrap/AtlasTypeDefStoreInitializer.java | 364 ++++++++++++++
.../store/graph/AtlasTypeDefGraphStore.java | 13 +
.../store/graph/v1/AtlasStructDefStoreV1.java | 35 +-
.../atlas/services/AtlasPatchHandler.java | 173 -------
.../atlas/services/AtlasTypeAttributePatch.java | 296 -----------
.../apache/atlas/services/AtlasTypePatch.java | 104 ----
.../atlas/services/DefaultMetadataService.java | 82 +--
.../services/IBootstrapTypesRegistrar.java | 27 -
.../atlas/services/ReservedTypesRegistrar.java | 135 -----
.../ReservedTypesRegistrationException.java | 26 -
.../org/apache/atlas/BaseRepositoryTest.java | 42 ++
.../test/java/org/apache/atlas/TestUtils.java | 2 -
.../store/graph/AtlasTypeDefGraphStoreTest.java | 4 +-
.../DefaultMetadataServiceMockTest.java | 151 ------
.../services/ReservedTypesRegistrarTest.java | 103 ----
57 files changed, 1917 insertions(+), 2616 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/pom.xml b/addons/falcon-bridge/pom.xml
index de3e810..96743e7 100644
--- a/addons/falcon-bridge/pom.xml
+++ b/addons/falcon-bridge/pom.xml
@@ -355,24 +355,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.falcon.model.FalconDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/falcon_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
index 05d072f..d1a7e87 100644
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
+++ b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/bridge/FalconBridge.java
@@ -21,11 +21,8 @@ package org.apache.atlas.falcon.bridge;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.falcon.Util.EventUtil;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.commons.collections.CollectionUtils;
@@ -61,6 +58,15 @@ import java.util.Map;
public class FalconBridge {
private static final Logger LOG = LoggerFactory.getLogger(FalconBridge.class);
+ public static final String COLO = "colo";
+ public static final String TAGS = "tags";
+ public static final String GROUPS = "groups";
+ public static final String PIPELINES = "pipelines";
+ public static final String WFPROPERTIES = "workflow-properties";
+ public static final String RUNSON = "runs-on";
+ public static final String STOREDIN = "stored-in";
+ public static final String FREQUENCY = "frequency";
+
/**
* Creates cluster entity
*
@@ -77,14 +83,14 @@ public class FalconBridge {
clusterRef.set(AtlasClient.DESCRIPTION, cluster.getDescription());
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, cluster.getName());
- clusterRef.set(FalconDataModelGenerator.COLO, cluster.getColo());
+ clusterRef.set(FalconBridge.COLO, cluster.getColo());
if (cluster.getACL() != null) {
clusterRef.set(AtlasClient.OWNER, cluster.getACL().getGroup());
}
if (StringUtils.isNotEmpty(cluster.getTags())) {
- clusterRef.set(FalconDataModelGenerator.TAGS,
+ clusterRef.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(cluster.getTags()));
}
@@ -100,19 +106,19 @@ public class FalconBridge {
String feedQualifiedName =
getFeedQualifiedName(feed.getName(), (String) clusterReferenceable.get(AtlasClient.NAME));
feedEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, feedQualifiedName);
- feedEntity.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency().toString());
- feedEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+ feedEntity.set(FalconBridge.FREQUENCY, feed.getFrequency().toString());
+ feedEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
if (feed.getACL() != null) {
feedEntity.set(AtlasClient.OWNER, feed.getACL().getOwner());
}
if (StringUtils.isNotEmpty(feed.getTags())) {
- feedEntity.set(FalconDataModelGenerator.TAGS,
+ feedEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(feed.getTags()));
}
if (feed.getGroups() != null) {
- feedEntity.set(FalconDataModelGenerator.GROUPS, feed.getGroups());
+ feedEntity.set(FalconBridge.GROUPS, feed.getGroups());
}
return feedEntity;
@@ -165,7 +171,7 @@ public class FalconBridge {
feedCreateEntity.set(AtlasClient.PROCESS_ATTRIBUTE_OUTPUTS, outputs);
}
- feedCreateEntity.set(FalconDataModelGenerator.STOREDIN, clusterReferenceable);
+ feedCreateEntity.set(FalconBridge.STOREDIN, clusterReferenceable);
entities.add(feedCreateEntity);
}
@@ -244,7 +250,7 @@ public class FalconBridge {
processEntity.set(AtlasClient.NAME, process.getName());
processEntity.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
getProcessQualifiedName(process.getName(), cluster.getName()));
- processEntity.set(FalconDataModelGenerator.FREQUENCY, process.getFrequency().toString());
+ processEntity.set(FalconBridge.FREQUENCY, process.getFrequency().toString());
if (!inputs.isEmpty()) {
processEntity.set(AtlasClient.PROCESS_ATTRIBUTE_INPUTS, inputs);
@@ -254,7 +260,7 @@ public class FalconBridge {
}
// set cluster
- processEntity.set(FalconDataModelGenerator.RUNSON, clusterReferenceable);
+ processEntity.set(FalconBridge.RUNSON, clusterReferenceable);
// Set user
if (process.getACL() != null) {
@@ -262,15 +268,15 @@ public class FalconBridge {
}
if (StringUtils.isNotEmpty(process.getTags())) {
- processEntity.set(FalconDataModelGenerator.TAGS,
+ processEntity.set(FalconBridge.TAGS,
EventUtil.convertKeyValueStringToMap(process.getTags()));
}
if (process.getPipelines() != null) {
- processEntity.set(FalconDataModelGenerator.PIPELINES, process.getPipelines());
+ processEntity.set(FalconBridge.PIPELINES, process.getPipelines());
}
- processEntity.set(FalconDataModelGenerator.WFPROPERTIES,
+ processEntity.set(FalconBridge.WFPROPERTIES,
getProcessEntityWFProperties(process.getWorkflow(),
process.getName()));
@@ -319,7 +325,7 @@ public class FalconBridge {
private static List<Referenceable> fillHDFSDataSet(final String pathUri, final String clusterName) {
List<Referenceable> entities = new ArrayList<>();
- Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+ Referenceable ref = new Referenceable(HiveMetaStoreBridge.HDFS_PATH);
ref.set("path", pathUri);
// Path path = new Path(pathUri);
// ref.set("name", path.getName());
@@ -352,7 +358,7 @@ public class FalconBridge {
tableRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
HiveMetaStoreBridge.getTableQualifiedName(clusterName, dbName, tableName));
tableRef.set(AtlasClient.NAME, tableName.toLowerCase());
- tableRef.set(HiveDataModelGenerator.DB, dbRef);
+ tableRef.set(HiveMetaStoreBridge.DB, dbRef);
entities.add(tableRef);
return entities;
@@ -364,7 +370,7 @@ public class FalconBridge {
Referenceable clusterRef = new Referenceable(FalconDataTypes.FALCON_CLUSTER.getName());
clusterRef.set(AtlasClient.NAME, String.format("%s", clusterName));
clusterRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, clusterName);
- clusterRef.set(FalconDataModelGenerator.COLO, colo);
+ clusterRef.set(FalconBridge.COLO, colo);
return clusterRef;
}
@@ -375,8 +381,8 @@ public class FalconBridge {
feedDatasetRef.set(AtlasClient.NAME, feed.getName());
feedDatasetRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, getFeedQualifiedName(feed.getName(),
(String) clusterReference.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME)));
- feedDatasetRef.set(FalconDataModelGenerator.STOREDIN, clusterReference);
- feedDatasetRef.set(FalconDataModelGenerator.FREQUENCY, feed.getFrequency());
+ feedDatasetRef.set(FalconBridge.STOREDIN, clusterReference);
+ feedDatasetRef.set(FalconBridge.FREQUENCY, feed.getFrequency());
return feedDatasetRef;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java b/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
deleted file mode 100644
index fc0a9b2..0000000
--- a/addons/falcon-bridge/src/main/java/org/apache/atlas/falcon/model/FalconDataModelGenerator.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.falcon.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates falcon data model.
- */
-public class FalconDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(FalconDataModelGenerator.class);
- public static final String FREQUENCY = "frequency";
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
-
- public static final String COLO = "colo";
- public static final String TAGS = "tags";
- public static final String GROUPS = "groups";
- public static final String PIPELINES = "pipelines";
- public static final String WFPROPERTIES = "workflow-properties";
- public static final String RUNSON = "runs-on";
- public static final String STOREDIN = "stored-in";
-
- public FalconDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- }
-
- public void createDataModel() throws AtlasException {
- LOG.info("Generating the Falcon Data Model");
-
- // classes
- createClusterEntityClass();
- createProcessEntityClass();
- createFeedCreationEntityClass();
- createFeedEntityClass();
- createReplicationFeedEntityClass();
- }
-
- private TypesDef getTypesDef() {
- return TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.<StructTypeDefinition>of(),
- getTraitTypeDefinitions(), getClassTypeDefinitions());
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- private ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- private ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createClusterEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(COLO, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_CLUSTER.getName(), null,
- ImmutableSet.of(AtlasClient.INFRASTRUCTURE_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_CLUSTER.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_CLUSTER.getName());
- }
-
- private void createFeedCreationEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null)
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED_CREATION.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_CREATION.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_CREATION.getName());
- }
-
- private void createFeedEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
- new AttributeDefinition(STOREDIN, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null),
- new AttributeDefinition(GROUPS, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null)
- };
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_FEED.getName(), null,
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED.getName());
- }
-
-
- private void createReplicationFeedEntityClass() throws AtlasException {
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class,
- FalconDataTypes.FALCON_FEED_REPLICATION.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), null);
- classTypeDefinitions.put(FalconDataTypes.FALCON_FEED_REPLICATION.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_FEED_REPLICATION.getName());
- }
-
- private void createProcessEntityClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- TypesUtil.createRequiredAttrDef(FREQUENCY, DataTypes.STRING_TYPE),
- new AttributeDefinition(RUNSON, FalconDataTypes.FALCON_CLUSTER.getName(), Multiplicity.REQUIRED,
- false, false, true, null),
- // map of tags
- new AttributeDefinition(TAGS,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(PIPELINES, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, false, true, null),
- // wf properties
- new AttributeDefinition(WFPROPERTIES,
- DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, FalconDataTypes.FALCON_PROCESS.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(FalconDataTypes.FALCON_PROCESS.getName(), definition);
- LOG.debug("Created definition for {}", FalconDataTypes.FALCON_PROCESS.getName());
- }
-
-
- public String getModelAsJson() throws AtlasException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- FalconDataModelGenerator falconDataModelGenerator = new FalconDataModelGenerator();
- String modelAsJson = falconDataModelGenerator.getModelAsJson();
-
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
- return;
- }
-
- System.out.println("falconDataModelAsJSON = " + modelAsJson);
-
- TypesDef typesDef = falconDataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
- for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
- System.out.println(
- String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
- Arrays.toString(structType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName,
- ClassType.class.getSimpleName(),
- StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
- Arrays.toString(traitType.attributeDefinitions)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
----------------------------------------------------------------------
diff --git a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
index 31b70fd..8d0a47a 100644
--- a/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
+++ b/addons/falcon-bridge/src/test/java/org/apache/atlas/falcon/hook/FalconHookIT.java
@@ -23,9 +23,7 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
import org.apache.atlas.falcon.bridge.FalconBridge;
-import org.apache.atlas.falcon.model.FalconDataModelGenerator;
import org.apache.atlas.falcon.model.FalconDataTypes;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
@@ -47,7 +45,6 @@ import org.apache.falcon.entity.v0.feed.Location;
import org.apache.falcon.entity.v0.feed.LocationType;
import org.apache.falcon.entity.v0.process.Process;
import org.apache.falcon.security.CurrentUser;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.slf4j.Logger;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -84,24 +81,9 @@ public class FalconHookIT {
AtlasService service = new AtlasService();
service.init();
STORE.registerListener(service);
- registerFalconDataModel();
CurrentUser.authenticate(System.getProperty("user.name"));
}
- private void registerFalconDataModel() throws Exception {
- if (isDataModelAlreadyRegistered()) {
- LOG.info("Falcon data model is already registered!");
- return;
- }
-
- HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(ApplicationProperties.get(), new HiveConf(), atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
-
- FalconDataModelGenerator dataModelGenerator = new FalconDataModelGenerator();
- LOG.info("Registering Falcon data model");
- atlasClient.createType(dataModelGenerator.getModelAsJson());
- }
-
private boolean isDataModelAlreadyRegistered() throws Exception {
try {
atlasClient.getType(FalconDataTypes.FALCON_PROCESS.getName());
@@ -196,7 +178,7 @@ public class FalconHookIT {
String inputId = ((List<Id>) processEntity.get("inputs")).get(0).getId()._getId();
Referenceable pathEntity = atlasClient.getEntity(inputId);
- assertEquals(pathEntity.getTypeName(), FSDataTypes.HDFS_PATH().toString());
+ assertEquals(pathEntity.getTypeName(), HiveMetaStoreBridge.HDFS_PATH.toString());
List<Location> locations = FeedHelper.getLocations(feedCluster, feed);
Location dataLocation = FileSystemStorage.getLocation(locations, LocationType.DATA);
@@ -243,7 +225,7 @@ public class FalconHookIT {
private void assertFeedAttributes(String feedId) throws Exception {
Referenceable feedEntity = atlasClient.getEntity(feedId);
assertEquals(feedEntity.get(AtlasClient.OWNER), "testuser");
- assertEquals(feedEntity.get(FalconDataModelGenerator.FREQUENCY), "hours(1)");
+ assertEquals(feedEntity.get(FalconBridge.FREQUENCY), "hours(1)");
assertEquals(feedEntity.get(AtlasClient.DESCRIPTION), "test input");
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/pom.xml b/addons/hdfs-model/pom.xml
index e2ab029..1376fdc 100644
--- a/addons/hdfs-model/pom.xml
+++ b/addons/hdfs-model/pom.xml
@@ -180,24 +180,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.fs.model.FSDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/src/test/resources/</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/fs_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java b/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
deleted file mode 100644
index 555d565..0000000
--- a/addons/hdfs-model/src/main/java/org/apache/atlas/fs/model/FSDataModelGenerator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model;
-
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-
-import java.io.IOException;
-
-public class FSDataModelGenerator {
-
- public static void main(String[] args) throws IOException {
- FSDataModel.main(args);
- TypesDef typesDef = FSDataModel.typesDef();
- String fsTypesAsJSON = TypesSerialization.toJson(typesDef);
- if (args.length == 1) {
- ModelDefinitionDump.dumpModelToFile(args[0], fsTypesAsJSON);
- return;
- }
- System.out.println("FS Data Model as JSON = " + fsTypesAsJSON);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala b/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
deleted file mode 100644
index ad40340..0000000
--- a/addons/hdfs-model/src/main/scala/org/apache/atlas/fs/model/FSDataModel.scala
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.fs.model
-
-import org.apache.atlas.{AtlasConstants, AtlasClient}
-import org.apache.atlas.typesystem.TypesDef
-import org.apache.atlas.typesystem.builders.TypesBuilder
-import org.apache.atlas.typesystem.json.TypesSerialization
-import org.apache.atlas.typesystem.types.DataTypes.MapType
-import org.apache.hadoop.fs.permission.FsAction
-
-import scala.tools.scalap.scalax.rules.scalasig.ClassFileParser.EnumConstValue
-
-/**
- * This represents the data model for a HDFS Path
- */
-object FSDataModel extends App {
-
- val typesBuilder = new TypesBuilder
- import typesBuilder._
-
- val typesDef : TypesDef = types {
-
- // FS DataSet
- _class(FSDataTypes.FS_PATH.toString, List(AtlasClient.DATA_SET_SUPER_TYPE)) {
- //fully qualified path/URI to the filesystem path is stored in 'qualifiedName' and 'path'.
- "path" ~ (string, required, indexed)
- "createTime" ~ (date, optional, indexed)
- "modifiedTime" ~ (date, optional, indexed)
- //Is a regular file or a directory. If true, it is a file else a directory
- "isFile" ~ (boolean, optional, indexed)
- //Is a symlink or not
- "isSymlink" ~ (boolean, optional)
- //Optional and may not be set for a directory
- "fileSize" ~ (long, optional)
- "group" ~ (string, optional, indexed)
- "posixPermissions" ~ (FSDataTypes.FS_PERMISSIONS.toString, optional, indexed)
- }
-
- enum(FSDataTypes.FS_ACTION.toString, FsAction.values().map(x => x.name()) : _*)
-
- struct(FSDataTypes.FS_PERMISSIONS.toString) {
- PosixPermissions.PERM_USER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.PERM_GROUP.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.PERM_OTHER.toString ~ (FSDataTypes.FS_ACTION.toString, required, indexed)
- PosixPermissions.STICKY_BIT.toString ~ (boolean, required, indexed)
- }
-
- //HDFS DataSet
- _class(FSDataTypes.HDFS_PATH.toString, List(FSDataTypes.FS_PATH.toString)) {
- //Making cluster optional since path is already unique containing the namenode URI
- AtlasConstants.CLUSTER_NAME_ATTRIBUTE ~ (string, optional, indexed)
- "numberOfReplicas" ~ (int, optional, indexed)
- "extendedAttributes" ~ (map(string, string), optional)
- }
- //TODO - ACLs - https://hadoop.apache.org/docs/r2.7.1/hadoop-project-dist/hadoop-hdfs/HdfsPermissionsGuide.html#ACLs_Access_Control_Lists
- }
-
- // add the types to atlas
- val typesAsJSON = TypesSerialization.toJson(typesDef)
- println("FS Data Model as JSON: ")
- println(typesAsJSON)
-
-}
-
-object FSDataTypes extends Enumeration {
- type FSDataTypes = Value
- val FS_ACTION = Value("file_action")
- val FS_PATH = Value("fs_path")
- val HDFS_PATH = Value("hdfs_path")
- val FS_PERMISSIONS = Value("fs_permissions")
-}
-
-object PosixPermissions extends Enumeration {
- type PosixPermissions = Value
- val PERM_USER = Value("user")
- val PERM_GROUP = Value("group")
- val PERM_OTHER = Value("others")
- val STICKY_BIT = Value("sticky")
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
----------------------------------------------------------------------
diff --git a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java b/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
deleted file mode 100644
index 5944da8..0000000
--- a/addons/hdfs-model/src/test/java/org/apache/atlas/fs/model/HDFSModelTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.fs.model;
-
-import javax.inject.Inject;
-
-import org.apache.atlas.RepositoryMetadataModule;
-import org.apache.atlas.repository.graph.AtlasGraphProvider;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.services.MetadataService;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import scala.Enumeration;
-import scala.collection.Iterator;
-
-@Test
-@Guice(modules = RepositoryMetadataModule.class)
-public class HDFSModelTest {
-
- public static final Logger LOG = LoggerFactory.getLogger(HDFSModelTest.class);
- private static final String ATLAS_URL = "http://localhost:21000/";
-
- @Inject
- private MetadataService metadataService;
-
- @BeforeClass
- public void setUp() throws Exception {
- }
-
- @AfterClass
- public void tearDown() throws Exception {
- TypeSystem.getInstance().reset();
- AtlasGraphProvider.cleanup();
- }
-
- @Test
- public void testCreateDataModel() throws Exception {
- FSDataModel.main(new String[]{});
- TypesDef fsTypesDef = FSDataModel.typesDef();
-
- String fsTypesAsJSON = TypesSerialization.toJson(fsTypesDef);
- LOG.info("fsTypesAsJSON = {}", fsTypesAsJSON);
-
- metadataService.createType(fsTypesAsJSON);
-
- // verify types are registered
- final Iterator<Enumeration.Value> valueIterator = FSDataTypes.values().iterator();
- while (valueIterator.hasNext() ) {
- final Enumeration.Value typeEnum = valueIterator.next();
- String typeDefStr = metadataService.getTypeDefinition(typeEnum.toString());
- Assert.assertNotNull(typeDefStr);
-
- TypesDef typesDef = TypesSerialization.fromJson(typeDefStr);
- Assert.assertNotNull(typesDef);
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/pom.xml
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/pom.xml b/addons/hive-bridge/pom.xml
index 884791d..7ab47f1 100755
--- a/addons/hive-bridge/pom.xml
+++ b/addons/hive-bridge/pom.xml
@@ -399,24 +399,6 @@
<version>1.2.1</version>
<inherited>false</inherited>
<executions>
- <execution>
- <configuration>
- <mainClass>org.apache.atlas.hive.model.HiveDataModelGenerator</mainClass>
- <systemProperties>
- <systemProperty>
- <key>atlas.conf</key>
- <value>${project.build.directory}/../../../typesystem/target/test-classes</value>
- </systemProperty>
- </systemProperties>
- <arguments>
- <argument>${project.build.directory}/models/hive_model.json</argument>
- </arguments>
- </configuration>
- <phase>package</phase>
- <goals>
- <goal>java</goal>
- </goals>
- </execution>
</executions>
</plugin>
</plugins>
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
index e4a20e1..c54fdb3 100644
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/ColumnLineageUtils.java
@@ -19,8 +19,6 @@
package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.hive.ql.hooks.LineageInfo;
@@ -31,7 +29,6 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
public class ColumnLineageUtils {
public static final Logger LOG = LoggerFactory.getLogger(ColumnLineageUtils.class);
@@ -92,7 +89,7 @@ public class ColumnLineageUtils {
if (r.getTypeName().equals(HiveDataTypes.HIVE_TABLE.getName())) {
String qName = (String) r.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] qNameComps = extractComponents(qName);
- for (Referenceable col : (List<Referenceable>) r.get(HiveDataModelGenerator.COLUMNS)) {
+ for (Referenceable col : (List<Referenceable>) r.get(HiveMetaStoreBridge.COLUMNS)) {
String cName = (String) col.get(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME);
String[] colQNameComps = extractComponents(cName);
String colQName = colQNameComps[0] + "." + colQNameComps[1] + "." + colQNameComps[2];
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
index 2d2039b..cbc51cc 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridge.java
@@ -26,15 +26,11 @@ import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.fs.model.FSDataModel;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.hook.HiveHook;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.json.InstanceSerialization;
-import org.apache.atlas.typesystem.json.TypesSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.utils.AuthenticationUtil;
import org.apache.commons.cli.BasicParser;
@@ -79,6 +75,25 @@ public class HiveMetaStoreBridge {
public static final String ATLAS_ENDPOINT = "atlas.rest.address";
+ public static final String COMMENT = "comment";
+ public static final String PARAMETERS = "parameters";
+ public static final String COLUMNS = "columns";
+ public static final String POSITION = "position";
+ public static final String PART_COLS = "partitionKeys";
+ public static final String TABLE_ALIAS_LIST = "aliases";
+ public static final String STORAGE_NUM_BUCKETS = "numBuckets";
+ public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
+ public static final String TABLE = "table";
+ public static final String DB = "db";
+ public static final String STORAGE_DESC = "sd";
+ public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
+ public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
+ public static final String LOCATION = "location";
+ public static final String TABLE_TYPE_ATTR = "tableType";
+ public static final String CREATE_TIME = "createTime";
+ public static final String LAST_ACCESS_TIME = "lastAccessTime";
+ public static final String HDFS_PATH = "hdfs_path";
+
private static final Logger LOG = LoggerFactory.getLogger(HiveMetaStoreBridge.class);
public final Hive hiveClient;
@@ -174,8 +189,8 @@ public class HiveMetaStoreBridge {
dbRef.set(AtlasClient.NAME, dbName);
dbRef.set(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, clusterName);
dbRef.set(DESCRIPTION_ATTR, hiveDB.getDescription());
- dbRef.set(HiveDataModelGenerator.LOCATION, hiveDB.getLocationUri());
- dbRef.set(HiveDataModelGenerator.PARAMETERS, hiveDB.getParameters());
+ dbRef.set(LOCATION, hiveDB.getLocationUri());
+ dbRef.set(PARAMETERS, hiveDB.getParameters());
dbRef.set(AtlasClient.OWNER, hiveDB.getOwnerName());
if (hiveDB.getOwnerType() != null) {
dbRef.set("ownerType", hiveDB.getOwnerType().getValue());
@@ -431,7 +446,7 @@ public class HiveMetaStoreBridge {
try {
createDate = getTableCreatedTime(hiveTable);
LOG.debug("Setting create time to {} ", createDate);
- tableReference.set(HiveDataModelGenerator.CREATE_TIME, createDate);
+ tableReference.set(CREATE_TIME, createDate);
} catch(Exception ne) {
LOG.error("Error while setting createTime for the table {} ", hiveTable.getCompleteName(), ne);
}
@@ -441,19 +456,19 @@ public class HiveMetaStoreBridge {
if ( hiveTable.getLastAccessTime() > 0) {
lastAccessTime = new Date(hiveTable.getLastAccessTime() * MILLIS_CONVERT_FACTOR);
}
- tableReference.set(HiveDataModelGenerator.LAST_ACCESS_TIME, lastAccessTime);
+ tableReference.set(LAST_ACCESS_TIME, lastAccessTime);
tableReference.set("retention", hiveTable.getRetention());
- tableReference.set(HiveDataModelGenerator.COMMENT, hiveTable.getParameters().get(HiveDataModelGenerator.COMMENT));
+ tableReference.set(COMMENT, hiveTable.getParameters().get(COMMENT));
// add reference to the database
- tableReference.set(HiveDataModelGenerator.DB, dbReference);
+ tableReference.set(DB, dbReference);
// add reference to the StorageDescriptor
Referenceable sdReferenceable = fillStorageDesc(hiveTable.getSd(), tableQualifiedName, getStorageDescQFName(tableQualifiedName), tableReference.getId());
- tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReferenceable);
+ tableReference.set(STORAGE_DESC, sdReferenceable);
- tableReference.set(HiveDataModelGenerator.PARAMETERS, hiveTable.getParameters());
+ tableReference.set(PARAMETERS, hiveTable.getParameters());
if (hiveTable.getViewOriginalText() != null) {
tableReference.set("viewOriginalText", hiveTable.getViewOriginalText());
@@ -463,14 +478,14 @@ public class HiveMetaStoreBridge {
tableReference.set("viewExpandedText", hiveTable.getViewExpandedText());
}
- tableReference.set(HiveDataModelGenerator.TABLE_TYPE_ATTR, hiveTable.getTableType().name());
+ tableReference.set(TABLE_TYPE_ATTR, hiveTable.getTableType().name());
tableReference.set("temporary", hiveTable.isTemporary());
// add reference to the Partition Keys
List<Referenceable> partKeys = getColumns(hiveTable.getPartitionKeys(), tableReference);
tableReference.set("partitionKeys", partKeys);
- tableReference.set(HiveDataModelGenerator.COLUMNS, getColumns(hiveTable.getCols(), tableReference));
+ tableReference.set(COLUMNS, getColumns(hiveTable.getCols(), tableReference));
return tableReference;
}
@@ -523,12 +538,12 @@ public class HiveMetaStoreBridge {
serdeInfoStruct.set(AtlasClient.NAME, serdeInfo.getName());
serdeInfoStruct.set("serializationLib", serdeInfo.getSerializationLib());
- serdeInfoStruct.set(HiveDataModelGenerator.PARAMETERS, serdeInfo.getParameters());
+ serdeInfoStruct.set(PARAMETERS, serdeInfo.getParameters());
sdReferenceable.set("serdeInfo", serdeInfoStruct);
- sdReferenceable.set(HiveDataModelGenerator.STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
+ sdReferenceable.set(STORAGE_NUM_BUCKETS, storageDesc.getNumBuckets());
sdReferenceable
- .set(HiveDataModelGenerator.STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
+ .set(STORAGE_IS_STORED_AS_SUB_DIRS, storageDesc.isStoredAsSubDirectories());
List<Struct> sortColsStruct = new ArrayList<>();
for (Order sortcol : storageDesc.getSortCols()) {
@@ -543,7 +558,7 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("sortCols", sortColsStruct);
}
- sdReferenceable.set(HiveDataModelGenerator.LOCATION, storageDesc.getLocation());
+ sdReferenceable.set(LOCATION, storageDesc.getLocation());
sdReferenceable.set("inputFormat", storageDesc.getInputFormat());
sdReferenceable.set("outputFormat", storageDesc.getOutputFormat());
sdReferenceable.set("compressed", storageDesc.isCompressed());
@@ -552,15 +567,15 @@ public class HiveMetaStoreBridge {
sdReferenceable.set("bucketCols", storageDesc.getBucketCols());
}
- sdReferenceable.set(HiveDataModelGenerator.PARAMETERS, storageDesc.getParameters());
+ sdReferenceable.set(PARAMETERS, storageDesc.getParameters());
sdReferenceable.set("storedAsSubDirectories", storageDesc.isStoredAsSubDirectories());
- sdReferenceable.set(HiveDataModelGenerator.TABLE, tableId);
+ sdReferenceable.set(TABLE, tableId);
return sdReferenceable;
}
public Referenceable fillHDFSDataSet(String pathUri) {
- Referenceable ref = new Referenceable(FSDataTypes.HDFS_PATH().toString());
+ Referenceable ref = new Referenceable(HDFS_PATH.toString());
ref.set("path", pathUri);
Path path = new Path(pathUri);
ref.set(AtlasClient.NAME, Path.getPathWithoutSchemeAndAuthority(path).toString().toLowerCase());
@@ -586,9 +601,9 @@ public class HiveMetaStoreBridge {
colReferenceable.set(AtlasClient.NAME, fs.getName());
colReferenceable.set(AtlasClient.OWNER, tableReference.get(AtlasClient.OWNER));
colReferenceable.set("type", fs.getType());
- colReferenceable.set(HiveDataModelGenerator.POSITION, columnPosition++);
- colReferenceable.set(HiveDataModelGenerator.COMMENT, fs.getComment());
- colReferenceable.set(HiveDataModelGenerator.TABLE, tableReference.getId());
+ colReferenceable.set(POSITION, columnPosition++);
+ colReferenceable.set(COMMENT, fs.getComment());
+ colReferenceable.set(TABLE, tableReference.getId());
colList.add(colReferenceable);
@@ -596,43 +611,6 @@ public class HiveMetaStoreBridge {
return colList;
}
- /**
- * Register the Hive DataModel in Atlas, if not already defined.
- *
- * The method checks for the presence of the type {@link HiveDataTypes#HIVE_PROCESS} with the Atlas server.
- * If this type is defined, then we assume the Hive DataModel is registered.
- * @throws Exception
- */
- public synchronized void registerHiveDataModel() throws Exception {
- HiveDataModelGenerator dataModelGenerator = new HiveDataModelGenerator();
- AtlasClient dgiClient = getAtlasClient();
-
- try {
- dgiClient.getType(FSDataTypes.HDFS_PATH().toString());
- LOG.info("HDFS data model is already registered!");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Trigger val definition
- FSDataModel.main(null);
-
- final String hdfsModelJson = TypesSerialization.toJson(FSDataModel.typesDef());
- //Expected in case types do not exist
- LOG.info("Registering HDFS data model : " + hdfsModelJson);
- dgiClient.createType(hdfsModelJson);
- }
- }
-
- try {
- dgiClient.getType(HiveDataTypes.HIVE_PROCESS.getName());
- LOG.info("Hive data model is already registered!");
- } catch(AtlasServiceException ase) {
- if (ase.getStatus() == ClientResponse.Status.NOT_FOUND) {
- //Expected in case types do not exist
- LOG.info("Registering Hive data model");
- dgiClient.createType(dataModelGenerator.getModelAsJson());
- }
- }
- }
public static void main(String[] args) throws Exception {
@@ -661,7 +639,6 @@ public class HiveMetaStoreBridge {
}
HiveMetaStoreBridge hiveMetaStoreBridge = new HiveMetaStoreBridge(atlasConf, new HiveConf(), atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
hiveMetaStoreBridge.importHiveMetadata(failOnError);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
index eaef337..0c6582b 100755
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
+++ b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/hook/HiveHook.java
@@ -25,7 +25,6 @@ import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasConstants;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.bridge.ColumnLineageUtils;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.hook.AtlasHook;
import org.apache.atlas.notification.hook.HookNotification;
@@ -411,10 +410,10 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
Referenceable tableEntity = tables.get(Type.TABLE);
//Reset regular column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
- replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.COLUMNS), oldQualifiedName, newQualifiedName);
+ replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.COLUMNS), oldQualifiedName, newQualifiedName);
//Reset partition key column QF Name to old Name and create a new partial notification request to replace old column QFName to newName to retain any existing traits
- replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveDataModelGenerator.PART_COLS), oldQualifiedName, newQualifiedName);
+ replaceColumnQFName(event, (List<Referenceable>) tableEntity.get(HiveMetaStoreBridge.PART_COLS), oldQualifiedName, newQualifiedName);
//Reset SD QF Name to old Name and create a new partial notification request to replace old SD QFName to newName to retain any existing traits
replaceSDQFName(event, tableEntity, oldQualifiedName, newQualifiedName);
@@ -437,7 +436,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
ArrayList<String> alias_list = new ArrayList<>();
alias_list.add(oldTable.getTableName().toLowerCase());
- newEntity.set(HiveDataModelGenerator.TABLE_ALIAS_LIST, alias_list);
+ newEntity.set(HiveMetaStoreBridge.TABLE_ALIAS_LIST, alias_list);
event.addMessage(new HookNotification.EntityPartialUpdateRequest(event.getUser(),
HiveDataTypes.HIVE_TABLE.getName(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME,
oldTableQFName, newEntity));
@@ -466,7 +465,7 @@ public class HiveHook extends AtlasHook implements ExecuteWithHookContext {
private Referenceable replaceSDQFName(final HiveEventContext event, Referenceable tableEntity, final String oldTblQFName, final String newTblQFName) {
//Reset storage desc QF Name to old Name
- final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveDataModelGenerator.STORAGE_DESC));
+ final Referenceable sdRef = ((Referenceable) tableEntity.get(HiveMetaStoreBridge.STORAGE_DESC));
sdRef.set(AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, HiveMetaStoreBridge.getStorageDescQFName(oldTblQFName));
//Replace SD QF name first to retain tags
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java b/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
deleted file mode 100755
index d0fc43f..0000000
--- a/addons/hive-bridge/src/main/java/org/apache/atlas/hive/model/HiveDataModelGenerator.java
+++ /dev/null
@@ -1,350 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.hive.model;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.apache.atlas.AtlasClient;
-import org.apache.atlas.AtlasConstants;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.addons.ModelDefinitionDump;
-import org.apache.atlas.typesystem.TypesDef;
-import org.apache.atlas.typesystem.json.TypesSerialization;
-import org.apache.atlas.typesystem.types.AttributeDefinition;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.EnumTypeDefinition;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.StructTypeDefinition;
-import org.apache.atlas.typesystem.types.TraitType;
-import org.apache.atlas.typesystem.types.utils.TypesUtil;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.Map;
-
-/**
- * Utility that generates hive data model for both metastore entities and DDL/DML queries.
- */
-public class HiveDataModelGenerator {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveDataModelGenerator.class);
-
- private static final DataTypes.MapType STRING_MAP_TYPE =
- new DataTypes.MapType(DataTypes.STRING_TYPE, DataTypes.STRING_TYPE);
-
- private final Map<String, HierarchicalTypeDefinition<ClassType>> classTypeDefinitions;
- private final Map<String, EnumTypeDefinition> enumTypeDefinitionMap;
- private final Map<String, StructTypeDefinition> structTypeDefinitionMap;
-
- public static final String COMMENT = "comment";
- public static final String PARAMETERS = "parameters";
- public static final String COLUMNS = "columns";
- public static final String POSITION = "position";
- public static final String PART_COLS = "partitionKeys";
- public static final String TABLE_ALIAS_LIST = "aliases";
-
- public static final String STORAGE_NUM_BUCKETS = "numBuckets";
- public static final String STORAGE_IS_STORED_AS_SUB_DIRS = "storedAsSubDirectories";
-
- public static final String TABLE = "table";
- public static final String DB = "db";
-
- public static final String STORAGE_DESC = "sd";
- public static final String STORAGE_DESC_INPUT_FMT = "inputFormat";
- public static final String STORAGE_DESC_OUTPUT_FMT = "outputFormat";
- public static final String LOCATION = "location";
-
- public static final String TABLE_TYPE_ATTR = "tableType";
-
- public static final String CREATE_TIME = "createTime";
- public static final String LAST_ACCESS_TIME = "lastAccessTime";
-
- public HiveDataModelGenerator() {
- classTypeDefinitions = new HashMap<>();
- enumTypeDefinitionMap = new HashMap<>();
- structTypeDefinitionMap = new HashMap<>();
- }
-
- public void createDataModel() throws AtlasException {
- LOG.info("Generating the Hive Data Model....");
-
- // enums
- createHivePrincipalTypeEnum();
- // structs
- createSerDeStruct();
- createOrderStruct();
- createStorageDescClass();
-
- // classes
- createDBClass();
- createColumnClass();
- createTableClass();
-
- // DDL/DML Process
- createProcessClass();
- createColumnLineageClass();
- }
-
- public TypesDef getTypesDef() {
- return TypesUtil.getTypesDef(getEnumTypeDefinitions(), getStructTypeDefinitions(), getTraitTypeDefinitions(),
- getClassTypeDefinitions());
- }
-
- public String getDataModelAsJSON() {
- return TypesSerialization.toJson(getTypesDef());
- }
-
- public ImmutableList<EnumTypeDefinition> getEnumTypeDefinitions() {
- return ImmutableList.copyOf(enumTypeDefinitionMap.values());
- }
-
- public ImmutableList<StructTypeDefinition> getStructTypeDefinitions() {
- return ImmutableList.copyOf(structTypeDefinitionMap.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<ClassType>> getClassTypeDefinitions() {
- return ImmutableList.copyOf(classTypeDefinitions.values());
- }
-
- public ImmutableList<HierarchicalTypeDefinition<TraitType>> getTraitTypeDefinitions() {
- return ImmutableList.of();
- }
-
- private void createHivePrincipalTypeEnum() throws AtlasException {
- EnumValue values[] = {new EnumValue("USER", 1), new EnumValue("ROLE", 2), new EnumValue("GROUP", 3),};
-
- EnumTypeDefinition definition = new EnumTypeDefinition(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), values);
-
- enumTypeDefinitionMap.put(HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName());
- }
-
- private void createSerDeStruct() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(AtlasClient.NAME, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("serializationLib", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),};
- StructTypeDefinition definition =
- new StructTypeDefinition(HiveDataTypes.HIVE_SERDE.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_SERDE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_SERDE.getName());
- }
-
- private void createOrderStruct() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("col", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("order", DataTypes.INT_TYPE.getName(), Multiplicity.REQUIRED, false, null),};
-
- StructTypeDefinition definition =
- new StructTypeDefinition(HiveDataTypes.HIVE_ORDER.getName(), attributeDefinitions);
- structTypeDefinitionMap.put(HiveDataTypes.HIVE_ORDER.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_ORDER.getName());
- }
-
- private void createStorageDescClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- //Optional to keep it backward-compatible
- new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false,
- STORAGE_DESC),
- new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("inputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("outputFormat", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("compressed", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition(STORAGE_NUM_BUCKETS, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("serdeInfo", HiveDataTypes.HIVE_SERDE.getName(), Multiplicity.OPTIONAL, true,
- null),
- new AttributeDefinition("bucketCols", String.format("array<%s>", DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("sortCols", String.format("array<%s>", HiveDataTypes.HIVE_ORDER.getName()),
- Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("parameters", STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- //new AttributeDefinition("skewedInfo", DefinedTypes.HIVE_SKEWEDINFO.getName(),
- // Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(STORAGE_IS_STORED_AS_SUB_DIRS, DataTypes.BOOLEAN_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_STORAGEDESC.getName(), null,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_STORAGEDESC.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_STORAGEDESC.getName());
- }
-
- /** Revisit later after nested array types are handled by the typesystem **/
-
- private void createDBClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition(LOCATION, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("ownerType", HiveDataTypes.HIVE_PRINCIPAL_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_DB.getName(), null,
- ImmutableSet.of(AtlasClient.REFERENCEABLE_SUPER_TYPE, AtlasClient.ASSET_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_DB.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_DB.getName());
- }
-
- private void createColumnClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("type", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, false, true, null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- //Making this optional since this is an incompatible change
- //Reverse attribute to 'columns' in Table
- new AttributeDefinition(TABLE, HiveDataTypes.HIVE_TABLE.getName(), Multiplicity.OPTIONAL, false, COLUMNS),
- new AttributeDefinition(POSITION, DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, false, false, null)};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN.getName(), null, "1.1",
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN.getName());
- }
-
- private void createTableClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition(DB, HiveDataTypes.HIVE_DB.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition(CREATE_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(LAST_ACCESS_TIME, DataTypes.DATE_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition(COMMENT, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("retention", DataTypes.INT_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(STORAGE_DESC, HiveDataTypes.HIVE_STORAGEDESC.getName(), Multiplicity.OPTIONAL, true,
- null),
- new AttributeDefinition(PART_COLS, DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition(TABLE_ALIAS_LIST, DataTypes.arrayTypeName(DataTypes.STRING_TYPE.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition("columns", DataTypes.arrayTypeName(HiveDataTypes.HIVE_COLUMN.getName()),
- Multiplicity.OPTIONAL, true, null),
- new AttributeDefinition(HiveDataModelGenerator.PARAMETERS, STRING_MAP_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("viewOriginalText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition("viewExpandedText", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL,
- false, null),
- new AttributeDefinition(HiveDataModelGenerator.TABLE_TYPE_ATTR, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),
- new AttributeDefinition("temporary", DataTypes.BOOLEAN_TYPE.getName(), Multiplicity.OPTIONAL, false,
- false, true, null),};
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_TABLE.getName(), null,
- ImmutableSet.of(AtlasClient.DATA_SET_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_TABLE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_TABLE.getName());
- }
-
- private void createProcessClass() throws AtlasException {
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("startTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("endTime", DataTypes.DATE_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("userName", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition("operationType", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- false, true, null),
- new AttributeDefinition("queryText", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition("queryPlan", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false,
- null),
- new AttributeDefinition("queryId", DataTypes.STRING_TYPE.getName(), Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("recentQueries", String.format("array<%s>", DataTypes.STRING_TYPE.getName()), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition(AtlasConstants.CLUSTER_NAME_ATTRIBUTE, DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false, null),
- new AttributeDefinition("queryGraph", DataTypes.STRING_TYPE.getName(), Multiplicity.OPTIONAL, false,
- null),};
-
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_PROCESS.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_PROCESS.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_PROCESS.getName());
- }
-
- public String getModelAsJson() throws AtlasException {
- createDataModel();
- return getDataModelAsJSON();
- }
-
- public static void main(String[] args) throws Exception {
- HiveDataModelGenerator hiveDataModelGenerator = new HiveDataModelGenerator();
- String modelAsJson = hiveDataModelGenerator.getModelAsJson();
-
- if (args.length==1) {
- ModelDefinitionDump.dumpModelToFile(args[0], modelAsJson);
- return;
- }
-
- System.out.println("hiveDataModelAsJSON = " + modelAsJson);
-
- TypesDef typesDef = hiveDataModelGenerator.getTypesDef();
- for (EnumTypeDefinition enumType : typesDef.enumTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - values %s", enumType.name, EnumType.class.getSimpleName(),
- Arrays.toString(enumType.enumValues)));
- }
- for (StructTypeDefinition structType : typesDef.structTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - attributes %s", structType.typeName, StructType.class.getSimpleName(),
- Arrays.toString(structType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<ClassType> classType : typesDef.classTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - super types [%s] - attributes %s", classType.typeName, ClassType.class.getSimpleName(),
- StringUtils.join(classType.superTypes, ","), Arrays.toString(classType.attributeDefinitions)));
- }
- for (HierarchicalTypeDefinition<TraitType> traitType : typesDef.traitTypesAsJavaList()) {
- System.out.println(String.format("%s(%s) - %s", traitType.typeName, TraitType.class.getSimpleName(),
- Arrays.toString(traitType.attributeDefinitions)));
- }
- }
-
- private void createColumnLineageClass() throws AtlasException {
-
- AttributeDefinition[] attributeDefinitions = new AttributeDefinition[]{
- new AttributeDefinition("query", HiveDataTypes.HIVE_PROCESS.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("depenendencyType",DataTypes.STRING_TYPE.getName(),
- Multiplicity.REQUIRED, false, null),
- new AttributeDefinition("expression",DataTypes.STRING_TYPE.getName(),
- Multiplicity.OPTIONAL, false, null)
- };
- HierarchicalTypeDefinition<ClassType> definition =
- new HierarchicalTypeDefinition<>(ClassType.class, HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), null,
- ImmutableSet.of(AtlasClient.PROCESS_SUPER_TYPE), attributeDefinitions);
- classTypeDefinitions.put(HiveDataTypes.HIVE_COLUMN_LINEAGE.getName(), definition);
- LOG.debug("Created definition for " + HiveDataTypes.HIVE_COLUMN_LINEAGE.getName());
-
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
index bdf0d2d..5abf2df 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/HiveITBase.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
-import org.apache.atlas.fs.model.FSDataTypes;
import org.apache.atlas.hive.bridge.HiveMetaStoreBridge;
import org.apache.atlas.hive.hook.HiveHookIT;
import org.apache.atlas.hive.model.HiveDataTypes;
@@ -93,7 +92,6 @@ public class HiveITBase {
}
hiveMetaStoreBridge = new HiveMetaStoreBridge(configuration, conf, atlasClient);
- hiveMetaStoreBridge.registerHiveDataModel();
HiveConf conf = new HiveConf();
conf.set("hive.exec.post.hooks", "");
@@ -232,7 +230,7 @@ public class HiveITBase {
private String assertHDFSPathIsRegistered(String path) throws Exception {
LOG.debug("Searching for hdfs path {}", path);
- return assertEntityIsRegistered(FSDataTypes.HDFS_PATH().toString(), AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
+ return assertEntityIsRegistered(HiveMetaStoreBridge.HDFS_PATH, AtlasClient.REFERENCEABLE_ATTRIBUTE_NAME, path, null);
}
protected String assertDatabaseIsRegistered(String dbName) throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/def9e385/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
----------------------------------------------------------------------
diff --git a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
index e488f93..0cba27e 100644
--- a/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
+++ b/addons/hive-bridge/src/test/java/org/apache/atlas/hive/bridge/HiveMetaStoreBridgeTest.java
@@ -20,7 +20,6 @@ package org.apache.atlas.hive.bridge;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasServiceException;
-import org.apache.atlas.hive.model.HiveDataModelGenerator;
import org.apache.atlas.hive.model.HiveDataTypes;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.hadoop.fs.Path;
@@ -32,7 +31,6 @@ import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.mapred.TextInputFormat;
-import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONException;
import org.mockito.ArgumentMatcher;
import org.mockito.Mock;
@@ -110,7 +108,7 @@ public class HiveMetaStoreBridgeTest {
// verify update is called on table
verify(atlasClient).updateEntity(eq("82e06b34-9151-4023-aa9d-b82103a50e77"),
- (Referenceable) argThat(new MatchesReferenceableProperty(HiveDataModelGenerator.TABLE_TYPE_ATTR,
+ (Referenceable) argThat(new MatchesReferenceableProperty(HiveMetaStoreBridge.TABLE_TYPE_ATTR,
TableType.EXTERNAL_TABLE.name())));
}
@@ -228,7 +226,7 @@ public class HiveMetaStoreBridgeTest {
private Referenceable createTableReference() {
Referenceable tableReference = new Referenceable(HiveDataTypes.HIVE_TABLE.getName());
Referenceable sdReference = new Referenceable(HiveDataTypes.HIVE_STORAGEDESC.getName());
- tableReference.set(HiveDataModelGenerator.STORAGE_DESC, sdReference);
+ tableReference.set(HiveMetaStoreBridge.STORAGE_DESC, sdReference);
return tableReference;
}