You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sh...@apache.org on 2016/06/22 06:02:04 UTC
[2/3] incubator-atlas git commit: ATLAS-856 Lazy-load type cache
provider (dkantor via shwethags)
ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/ee8a2153
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/ee8a2153
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/ee8a2153
Branch: refs/heads/0.7-incubating
Commit: ee8a2153e8ea19cfb8cb48d3b5b6b8aec6fa0a9f
Parents: bfb60aa
Author: Shwetha GS <ss...@hortonworks.com>
Authored: Wed Jun 22 11:27:30 2016 +0530
Committer: Shwetha GS <ss...@hortonworks.com>
Committed: Wed Jun 22 11:27:30 2016 +0530
----------------------------------------------------------------------
.../org/apache/atlas/ApplicationProperties.java | 5 +-
release-log.txt | 1 +
.../apache/atlas/RepositoryMetadataModule.java | 49 ++-
.../typestore/GraphBackedTypeStore.java | 14 +
.../atlas/repository/typestore/ITypeStore.java | 9 +
.../typestore/StoreBackedTypeCache.java | 250 ++++++++++++
.../atlas/services/DefaultMetadataService.java | 20 +-
.../typestore/GraphBackedTypeStoreTest.java | 44 ++-
.../StoreBackedTypeCacheConfigurationTest.java | 43 +++
.../typestore/StoreBackedTypeCacheTest.java | 212 ++++++++++
.../StoreBackedTypeCacheTestModule.java | 42 ++
...StoreBackedTypeCacheMetadataServiceTest.java | 81 ++++
.../DefaultMetadataServiceMockTest.java | 16 +-
.../atlas/typesystem/types/TypeSystem.java | 42 +-
.../types/cache/DefaultTypeCache.java | 229 +++++++++++
.../types/cache/DefaultTypeCacheProvider.java | 229 -----------
.../types/cache/ITypeCacheProvider.java | 137 -------
.../atlas/typesystem/types/cache/TypeCache.java | 137 +++++++
.../apache/atlas/ApplicationPropertiesTest.java | 11 +-
.../cache/DefaultTypeCacheProviderTest.java | 383 -------------------
.../types/cache/DefaultTypeCacheTest.java | 383 +++++++++++++++++++
.../apache/atlas/web/listeners/TestModule.java | 4 +-
22 files changed, 1513 insertions(+), 828 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/common/src/main/java/org/apache/atlas/ApplicationProperties.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/ApplicationProperties.java b/common/src/main/java/org/apache/atlas/ApplicationProperties.java
index 741e1a6..9b1e9cd 100644
--- a/common/src/main/java/org/apache/atlas/ApplicationProperties.java
+++ b/common/src/main/java/org/apache/atlas/ApplicationProperties.java
@@ -108,10 +108,9 @@ public final class ApplicationProperties extends PropertiesConfiguration {
return inConf.subset(prefix);
}
- public static Class getClass(String propertyName, String defaultValue, Class assignableClass)
- throws AtlasException {
+ public static Class getClass(Configuration configuration, String propertyName, String defaultValue,
+ Class assignableClass) throws AtlasException {
try {
- Configuration configuration = get();
String propertyValue = configuration.getString(propertyName, defaultValue);
Class<?> clazz = Class.forName(propertyValue);
if (assignableClass == null || assignableClass.isAssignableFrom(clazz)) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 2f10a70..54a733c 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -31,6 +31,7 @@ ATLAS-409 Atlas will not import avro tables with schema read from a file (dosset
ATLAS-379 Create sqoop and falcon metadata addons (venkatnrangan,bvellanki,sowmyaramesh via shwethags)
ALL CHANGES:
+ATLAS-856 Lazy-load type cache provider (dkantor via shwethags)
ATLAS-931 Delete entities fails when hard delete is configured (dkantor via sumasai)
ATLAS-932 UI: 'create tag' button does not work (mneethiraj via sumasai)
ATLAS-928 UI is not showing the name column for hive tables in the schema tab (yhemanth via sumasai)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
index c4c7678..3486436 100755
--- a/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
+++ b/repository/src/main/java/org/apache/atlas/RepositoryMetadataModule.java
@@ -24,6 +24,7 @@ import com.google.inject.matcher.Matchers;
import com.google.inject.multibindings.Multibinder;
import com.google.inject.throwingproviders.ThrowingProviderBinder;
import com.thinkaurelius.titan.core.TitanGraph;
+
import org.aopalliance.intercept.MethodInterceptor;
import org.apache.atlas.discovery.DiscoveryService;
import org.apache.atlas.discovery.DataSetLineageService;
@@ -50,6 +51,9 @@ import org.apache.atlas.services.MetadataService;
import org.apache.atlas.services.ReservedTypesRegistrar;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.TypeSystemProvider;
+import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.apache.commons.configuration.Configuration;
/**
* Guice module for Repository module.
@@ -85,9 +89,12 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bind(LineageService.class).to(DataSetLineageService.class).asEagerSingleton();
- bindAuditRepository(binder());
+ Configuration configuration = getConfiguration();
+ bindAuditRepository(binder(), configuration);
+
+ bind(DeleteHandler.class).to(getDeleteHandlerImpl(configuration)).asEagerSingleton();
- bind(DeleteHandler.class).to(getDeleteHandlerImpl()).asEagerSingleton();
+ bind(TypeCache.class).to(getTypeCache(configuration)).asEagerSingleton();
//Add EntityAuditListener as EntityChangeListener
Multibinder<EntityChangeListener> entityChangeListenerBinder =
@@ -99,9 +106,17 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
bindInterceptor(Matchers.any(), Matchers.annotatedWith(GraphTransaction.class), interceptor);
}
- protected void bindAuditRepository(Binder binder) {
+ protected Configuration getConfiguration() {
+ try {
+ return ApplicationProperties.get();
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ protected void bindAuditRepository(Binder binder, Configuration configuration) {
- Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl();
+ Class<? extends EntityAuditRepository> auditRepoImpl = getAuditRepositoryImpl(getConfiguration());
//Map EntityAuditRepository interface to configured implementation
binder.bind(EntityAuditRepository.class).to(auditRepoImpl).asEagerSingleton();
@@ -117,10 +132,10 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY = "atlas.EntityAuditRepository.impl";
- private Class<? extends EntityAuditRepository> getAuditRepositoryImpl() {
+ private Class<? extends EntityAuditRepository> getAuditRepositoryImpl(Configuration configuration) {
try {
- return ApplicationProperties.getClass(AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY,
- HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
+ return ApplicationProperties.getClass(configuration,
+ AUDIT_REPOSITORY_IMPLEMENTATION_PROPERTY, HBaseBasedAuditRepository.class.getName(), EntityAuditRepository.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
@@ -128,12 +143,26 @@ public class RepositoryMetadataModule extends com.google.inject.AbstractModule {
private static final String DELETE_HANDLER_IMPLEMENTATION_PROPERTY = "atlas.DeleteHandler.impl";
- private Class<? extends DeleteHandler> getDeleteHandlerImpl() {
+ private Class<? extends DeleteHandler> getDeleteHandlerImpl(Configuration configuration) {
try {
- return ApplicationProperties.getClass(DELETE_HANDLER_IMPLEMENTATION_PROPERTY,
- SoftDeleteHandler.class.getName(), DeleteHandler.class);
+ return ApplicationProperties.getClass(configuration,
+ DELETE_HANDLER_IMPLEMENTATION_PROPERTY, SoftDeleteHandler.class.getName(), DeleteHandler.class);
} catch (AtlasException e) {
throw new RuntimeException(e);
}
}
+
+ public static final String TYPE_CACHE_IMPLEMENTATION_PROPERTY = "atlas.TypeCache.impl";
+
+ protected Class<? extends TypeCache> getTypeCache(Configuration configuration) {
+
+ // Get the type cache implementation class from Atlas configuration.
+ try {
+ return ApplicationProperties.getClass(configuration, TYPE_CACHE_IMPLEMENTATION_PROPERTY,
+ DefaultTypeCache.class.getName(), TypeCache.class);
+ } catch (AtlasException e) {
+ throw new RuntimeException("Error getting TypeCache implementation class", e);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
index 5ed9e02..4503899 100755
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -214,6 +214,20 @@ public class GraphBackedTypeStore implements ITypeStore {
Iterator vertices =
titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).vertices().iterator();
+ return getTypesFromVertices(vertices);
+ }
+
+ @Override
+ @GraphTransaction
+ public TypesDef restoreType(String typeName) throws AtlasException {
+ // Get vertex for the specified type name.
+ Iterator vertices =
+ titanGraph.query().has(Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE).has(Constants.TYPENAME_PROPERTY_KEY, typeName).vertices().iterator();
+
+ return getTypesFromVertices(vertices);
+ }
+
+ private TypesDef getTypesFromVertices(Iterator vertices) throws AtlasException {
ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
index 790c4b3..25f5f8b 100755
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/ITypeStore.java
@@ -40,4 +40,13 @@ public interface ITypeStore {
* @throws AtlasException
*/
TypesDef restore() throws AtlasException;
+
+ /**
+ * Restore the specified type definition
+ *
+ * @param typeName name of requested type
+ * @return persisted type definition
+ * @throws AtlasException
+ */
+ TypesDef restoreType(String typeName) throws AtlasException;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
new file mode 100644
index 0000000..d0c6f6a
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/StoreBackedTypeCache.java
@@ -0,0 +1,250 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.TypesDef;
+import org.apache.atlas.typesystem.types.AttributeDefinition;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.EnumTypeDefinition;
+import org.apache.atlas.typesystem.types.HierarchicalTypeDefinition;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.StructTypeDefinition;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeSystem.TransientTypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
+import org.apache.atlas.typesystem.types.utils.TypesUtil;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+import com.google.inject.Singleton;
+
+
+/**
+ * An extension of {@link DefaultTypeCache} which loads
+ * the requested type from the type store if it is not found in the cache,
+ * and adds it to the cache if it's found in the store.
+ * Any attribute and super types that are required by the requested type
+ * are also loaded from the store if they are not already in the cache.
+ */
+@Singleton
+public class StoreBackedTypeCache extends DefaultTypeCache {
+
+ private ITypeStore typeStore;
+
+ private ImmutableList<String> coreTypes;
+ private TypeSystem typeSystem;
+
+ @Inject
+ public StoreBackedTypeCache(final ITypeStore typeStore) {
+ this.typeStore = typeStore;
+ typeSystem = TypeSystem.getInstance();
+ coreTypes = typeSystem.getCoreTypes();
+ }
+
+ private static class Context {
+ ImmutableList.Builder<EnumTypeDefinition> enums = ImmutableList.builder();
+ ImmutableList.Builder<StructTypeDefinition> structs = ImmutableList.builder();
+ ImmutableList.Builder<HierarchicalTypeDefinition<ClassType>> classTypes = ImmutableList.builder();
+ ImmutableList.Builder<HierarchicalTypeDefinition<TraitType>> traits = ImmutableList.builder();
+ Set<String> loadedFromStore = new HashSet<>();
+
+ public void addTypesDefToLists(TypesDef typesDef) {
+
+ List<EnumTypeDefinition> enumTypesAsJavaList = typesDef.enumTypesAsJavaList();
+ enums.addAll(enumTypesAsJavaList);
+ for (EnumTypeDefinition etd : enumTypesAsJavaList) {
+ loadedFromStore.add(etd.name);
+ }
+ List<StructTypeDefinition> structTypesAsJavaList = typesDef.structTypesAsJavaList();
+ structs.addAll(structTypesAsJavaList);
+ for (StructTypeDefinition std : structTypesAsJavaList) {
+ loadedFromStore.add(std.typeName);
+ }
+ List<HierarchicalTypeDefinition<ClassType>> classTypesAsJavaList = typesDef.classTypesAsJavaList();
+ classTypes.addAll(classTypesAsJavaList);
+ for (HierarchicalTypeDefinition<ClassType> classTypeDef : classTypesAsJavaList) {
+ loadedFromStore.add(classTypeDef.typeName);
+ }
+ List<HierarchicalTypeDefinition<TraitType>> traitTypesAsJavaList = typesDef.traitTypesAsJavaList();
+ traits.addAll(traitTypesAsJavaList);
+ for (HierarchicalTypeDefinition<TraitType> traitTypeDef : traitTypesAsJavaList) {
+ loadedFromStore.add(traitTypeDef.typeName);
+ }
+ }
+
+ public boolean isLoadedFromStore(String typeName) {
+ return loadedFromStore.contains(typeName);
+ }
+
+ public TypesDef getTypesDef() {
+ return TypesUtil.getTypesDef(enums.build(), structs.build(), traits.build(), classTypes.build());
+ }
+ }
+
+ @Override
+ public boolean has(String typeName) throws AtlasException {
+
+ return (get(typeName) != null);
+ }
+
+ /**
+ * Checks whether the specified type is cached in memory and does *not*
+ * access the type store. Used for testing.
+ *
+ * @param typeName
+ * @return
+ */
+ public boolean isCachedInMemory(String typeName) throws AtlasException {
+ return super.has(typeName);
+ }
+
+ /**
+ * Gets the requested type from the cache.
+ * This implementation will check the type store if the type is
+ * not already cached. If found in the type store, the type and
+ * any required super and attribute types are loaded from the type store, and
+ * added to the cache.
+ *
+ * @see org.apache.atlas.typesystem.types.cache.DefaultTypeCache#get(java.lang.String)
+ */
+ @Override
+ public IDataType get(String typeName) throws AtlasException {
+
+ IDataType type = super.get(typeName);
+ if (type != null) {
+ return type;
+ }
+
+ // Type is not cached - check the type store.
+ // Any super and attribute types needed by the requested type
+ // which are not cached will also be loaded from the store.
+ Context context = new Context();
+ TypesDef typesDef = getTypeFromStore(typeName, context);
+ if (typesDef.isEmpty()) {
+ // Type not found in the type store.
+ return null;
+ }
+
+ // Add all types that were loaded from the store to the cache.
+ TransientTypeSystem transientTypeSystem = typeSystem.createTransientTypeSystem(context.getTypesDef(), false);
+ Map<String, IDataType> typesAdded = transientTypeSystem.getTypesAdded();
+ putAll(typesAdded.values());
+ return typesAdded.get(typeName);
+ }
+
+ private void getTypeFromCacheOrStore(String typeName, Context context)
+ throws AtlasException {
+
+ if (coreTypes.contains(typeName) || super.has(typeName)) {
+ return;
+ }
+
+ if (context.isLoadedFromStore(typeName)) {
+ return;
+ }
+
+ // Type not cached and hasn't been loaded during this operation, so check the store.
+ TypesDef typesDef = getTypeFromStore(typeName, context);
+ if (typesDef.isEmpty()) {
+ // Attribute type not found in cache or store.
+ throw new AtlasException(typeName + " not found in type store");
+ }
+ }
+
+ private TypesDef getTypeFromStore(String typeName, Context context)
+ throws AtlasException {
+
+ TypesDef typesDef = typeStore.restoreType(typeName);
+ if (!typesDef.isEmpty()) {
+ // Type found in store, add it to lists.
+ context.addTypesDefToLists(typesDef);
+
+ // Check the attribute and super types that are
+ // used by the requested type, and restore them
+ // as needed.
+ checkAttributeAndSuperTypes(typesDef, context);
+ }
+ return typesDef;
+ }
+
+ private void checkAttributeAndSuperTypes(TypesDef typesDef, Context context)
+ throws AtlasException {
+
+ // Check the cache and store for attribute types and super types.
+ for (HierarchicalTypeDefinition<ClassType> classTypeDef : typesDef.classTypesAsJavaList()) {
+ checkAttributeTypes(classTypeDef.attributeDefinitions, context);
+ for (String superTypeName : classTypeDef.superTypes) {
+ getTypeFromCacheOrStore(superTypeName, context);
+ }
+ }
+ for (HierarchicalTypeDefinition<TraitType> traitTypeDef : typesDef.traitTypesAsJavaList()) {
+ checkAttributeTypes(traitTypeDef.attributeDefinitions, context);
+ for (String superTypeName : traitTypeDef.superTypes) {
+ getTypeFromCacheOrStore(superTypeName, context);
+ }
+ }
+ for (StructTypeDefinition structTypeDef : typesDef.structTypesAsJavaList()) {
+ checkAttributeTypes(structTypeDef.attributeDefinitions, context);
+ }
+ }
+
+ private void checkAttributeTypes(AttributeDefinition[] attributeDefinitions,
+ Context context) throws AtlasException {
+
+ for (AttributeDefinition attrDef : attributeDefinitions) {
+ checkAttributeType(attrDef, context);
+ }
+ }
+
+ private void checkAttributeType(AttributeDefinition attrDef, Context context) throws AtlasException {
+
+ List<String> typeNamesToLookup = new ArrayList<>(2);
+
+ // Get the attribute type(s).
+ String elementTypeName = TypeUtils.parseAsArrayType(attrDef.dataTypeName);
+ if (elementTypeName != null) {
+ // Array attribute, lookup the element type.
+ typeNamesToLookup.add(elementTypeName);
+ }
+ else {
+ String[] mapTypeNames = TypeUtils.parseAsMapType(attrDef.dataTypeName);
+ if (mapTypeNames != null) {
+ // Map attribute, lookup the key and value types.
+ typeNamesToLookup.addAll(Arrays.asList(mapTypeNames));
+ }
+ else {
+ // Not an array or map, lookup the attribute type.
+ typeNamesToLookup.add(attrDef.dataTypeName);
+ }
+ }
+
+ for (String typeName : typeNamesToLookup) {
+ getTypeFromCacheOrStore(typeName, context);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 37e7b66..99f0036 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -22,6 +22,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import com.google.inject.Provider;
+
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
@@ -60,6 +61,7 @@ import org.apache.atlas.typesystem.types.StructTypeDefinition;
import org.apache.atlas.typesystem.types.TraitType;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.configuration.Configuration;
@@ -71,6 +73,7 @@ import org.slf4j.LoggerFactory;
import javax.inject.Inject;
import javax.inject.Singleton;
+
import java.util.ArrayList;
import java.util.Collection;
import java.util.LinkedHashSet;
@@ -109,10 +112,10 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
final IBootstrapTypesRegistrar typesRegistrar,
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
- final Collection<Provider<EntityChangeListener>> entityListenerProviders)
+ final Collection<Provider<EntityChangeListener>> entityListenerProviders, TypeCache typeCache)
throws AtlasException {
this(repository, typeStore, typesRegistrar, typeListenerProviders, entityListenerProviders,
- TypeSystem.getInstance(), ApplicationProperties.get());
+ TypeSystem.getInstance(), ApplicationProperties.get(), typeCache);
}
DefaultMetadataService(final MetadataRepository repository, final ITypeStore typeStore,
@@ -120,10 +123,21 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
final Collection<Provider<TypesChangeListener>> typeListenerProviders,
final Collection<Provider<EntityChangeListener>> entityListenerProviders,
final TypeSystem typeSystem,
- final Configuration configuration) throws AtlasException {
+ final Configuration configuration, TypeCache typeCache) throws AtlasException {
this.typeStore = typeStore;
this.typesRegistrar = typesRegistrar;
this.typeSystem = typeSystem;
+ /**
+ * Ideally a TypeCache implementation should have been injected in the TypeSystemProvider,
+ * but a singleton of TypeSystem is constructed privately within the class so that
+ * clients of TypeSystem would never instantiate a TypeSystem object directly in
+ * their code. As soon as a client makes a call to TypeSystem.getInstance(), they
+ * should have the singleton ready for consumption. Manually inject TypeSystem with
+ * the Guice-instantiated type cache here, before types are restored.
+ * This allows cache implementations to participate in Guice dependency injection.
+ */
+ this.typeSystem.setTypeCache(typeCache);
+
this.repository = repository;
for (Provider<TypesChangeListener> provider : typeListenerProviders) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
index dd530a7..da47dc1 100755
--- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
@@ -25,6 +25,7 @@ import com.thinkaurelius.titan.core.util.TitanCleanup;
import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.Vertex;
+
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
@@ -53,6 +54,7 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
+
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -64,6 +66,8 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructType
@Guice(modules = RepositoryMetadataModule.class)
public class GraphBackedTypeStoreTest {
+ private static final String DESCRIPTION = "_description";
+
@Inject
private GraphProvider<TitanGraph> graphProvider;
@@ -97,6 +101,12 @@ public class GraphBackedTypeStoreTest {
dumpGraph();
}
+ @Test(dependsOnMethods = "testStore")
+ public void testRestoreType() throws Exception {
+ TypesDef typesDef = ((GraphBackedTypeStore)typeStore).restoreType("Manager");
+ verifyRestoredClassType(typesDef, "Manager");
+ }
+
private void dumpGraph() {
TitanGraph graph = graphProvider.get();
for (Vertex v : graph.getVertices()) {
@@ -109,7 +119,6 @@ public class GraphBackedTypeStoreTest {
@Test(dependsOnMethods = "testStore")
public void testRestore() throws Exception {
- String description = "_description";
TypesDef types = typeStore.restore();
//validate enum
@@ -117,7 +126,7 @@ public class GraphBackedTypeStoreTest {
Assert.assertEquals(1, enumTypes.size());
EnumTypeDefinition orgLevel = enumTypes.get(0);
Assert.assertEquals(orgLevel.name, "OrgLevel");
- Assert.assertEquals(orgLevel.description, "OrgLevel"+description);
+ Assert.assertEquals(orgLevel.description, "OrgLevel"+DESCRIPTION);
Assert.assertEquals(orgLevel.enumValues.length, 2);
EnumValue enumValue = orgLevel.enumValues[0];
Assert.assertEquals(enumValue.value, "L1");
@@ -127,25 +136,14 @@ public class GraphBackedTypeStoreTest {
List<StructTypeDefinition> structTypes = types.structTypesAsJavaList();
Assert.assertEquals(1, structTypes.size());
- boolean clsTypeFound = false;
- List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
- for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
- if (classType.typeName.equals("Manager")) {
- ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
- Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
- Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
- Assert.assertEquals(classType.typeDescription, classType.typeName+description);
- clsTypeFound = true;
- }
- }
- Assert.assertTrue(clsTypeFound, "Manager type not restored");
+ verifyRestoredClassType(types, "Manager");
//validate trait
List<HierarchicalTypeDefinition<TraitType>> traitTypes = types.traitTypesAsJavaList();
Assert.assertEquals(1, traitTypes.size());
HierarchicalTypeDefinition<TraitType> trait = traitTypes.get(0);
Assert.assertEquals("SecurityClearance", trait.typeName);
- Assert.assertEquals(trait.typeName+description, trait.typeDescription);
+ Assert.assertEquals(trait.typeName+DESCRIPTION, trait.typeDescription);
Assert.assertEquals(1, trait.attributeDefinitions.length);
AttributeDefinition attribute = trait.attributeDefinitions[0];
Assert.assertEquals("level", attribute.name);
@@ -229,4 +227,20 @@ public class GraphBackedTypeStoreTest {
}
return edgeCount;
}
+
+ private void verifyRestoredClassType(TypesDef types, String typeName) throws AtlasException {
+ boolean clsTypeFound = false;
+ List<HierarchicalTypeDefinition<ClassType>> classTypes = types.classTypesAsJavaList();
+ for (HierarchicalTypeDefinition<ClassType> classType : classTypes) {
+ if (classType.typeName.equals(typeName)) {
+ ClassType expectedType = ts.getDataType(ClassType.class, classType.typeName);
+ Assert.assertEquals(expectedType.immediateAttrs.size(), classType.attributeDefinitions.length);
+ Assert.assertEquals(expectedType.superTypes.size(), classType.superTypes.size());
+ Assert.assertEquals(classType.typeDescription, classType.typeName+DESCRIPTION);
+ clsTypeFound = true;
+ }
+ }
+ Assert.assertTrue(clsTypeFound, typeName + " type not restored");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java
new file mode 100644
index 0000000..e62af85
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheConfigurationTest.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.testng.Assert;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.inject.Inject;
+
+/**
+ * Verify Guice can successfully instantiate and inject StoreBackTypeCache.
+ * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
+ * which has type cache implementation class set to {@link StoreBackedTypeCache}.
+ */
+@Guice(modules = StoreBackedTypeCacheTestModule.class)
+public class StoreBackedTypeCacheConfigurationTest {
+
+ @Inject
+ private TypeCache typeCache;
+
+ @Test
+ public void testConfigureAsTypeCache() throws Exception {
+ // Verify Guice successfully instantiated and injected StoreBackTypeCache
+ Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
new file mode 100644
index 0000000..ee4cc28
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTest.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.inject.Inject;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.graph.GraphProvider;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.TypeUtils;
+import org.testng.Assert;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.thinkaurelius.titan.core.util.TitanCleanup;
+
+
+/**
+ * Unit test for {@link StoreBackedTypeCache}
+ */
+@Guice(modules = RepositoryMetadataModule.class)
+public class StoreBackedTypeCacheTest {
+
+ @Inject
+ private GraphProvider<TitanGraph> graphProvider;
+
+ @Inject
+ private ITypeStore typeStore;
+
+ @Inject
+ private StoreBackedTypeCache typeCache;
+
+ private TypeSystem ts;
+
+ private Map<String, ClassType> classTypesToTest = new HashMap<>();
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ ts = TypeSystem.getInstance();
+ ts.reset();
+
+ // Populate the type store for testing.
+ TestUtils.defineDeptEmployeeTypes(ts);
+ TestUtils.createHiveTypes(ts);
+ ImmutableList<String> typeNames = ts.getTypeNames();
+ typeStore.store(ts, typeNames);
+
+ ClassType type = ts.getDataType(ClassType.class, "Manager");
+ classTypesToTest.put("Manager", type);
+ type = ts.getDataType(ClassType.class, TestUtils.TABLE_TYPE);
+ classTypesToTest.put(TestUtils.TABLE_TYPE, type);
+ }
+
+ @AfterClass
+ public void tearDown() throws Exception {
+ ts.reset();
+ try {
+ graphProvider.get().shutdown();
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+
+ try {
+ TitanCleanup.clear(graphProvider.get());
+ }
+ catch(Exception e) {
+ e.printStackTrace();
+ }
+ }
+
+ @BeforeMethod
+ public void setupTestMethod() throws Exception {
+ typeCache.clear();
+ }
+
+ @Test
+ public void testGetClassType() throws Exception {
+ for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
+ // Not cached yet
+ Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+ IDataType dataType = typeCache.get(typeEntry.getKey());
+ // Verify the type is now cached.
+ Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+ Assert.assertTrue(dataType instanceof ClassType);
+ ClassType cachedType = (ClassType)dataType;
+ // Verify that get() also loaded and cached any dependencies of this type from the type store.
+ verifyHierarchicalType(cachedType, typeEntry.getValue());
+ }
+ }
+
+ @Test
+ public void testHasClassType() throws Exception {
+ for (Map.Entry<String, ClassType> typeEntry : classTypesToTest.entrySet()) {
+ // Not cached yet
+ Assert.assertFalse(typeCache.isCachedInMemory(typeEntry.getKey()));
+
+ // Calling has() should result in type and its dependencies
+ // loaded from the type store and added to the cache.
+ Assert.assertTrue(typeCache.has(typeEntry.getKey()));
+
+ // Verify the type is now cached in memory.
+ Assert.assertTrue(typeCache.isCachedInMemory(typeEntry.getKey()));
+ }
+ }
+
+ @Test
+ public void testGetTraitType() throws Exception {
+ ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
+ for (String traitTypeName : traitNames) {
+ // Not cached yet
+ Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
+
+ IDataType dataType = typeCache.get(traitTypeName);
+ // Verify the type is now cached.
+ Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
+
+ Assert.assertTrue(dataType instanceof TraitType);
+ TraitType cachedType = (TraitType)dataType;
+ // Verify that get() also loaded and cached any dependencies of this type from the type store.
+ verifyHierarchicalType(cachedType, ts.getDataType(TraitType.class, traitTypeName));
+ }
+ }
+
+ @Test
+ public void testHasTraitType() throws Exception {
+ ImmutableList<String> traitNames = ts.getTypeNamesByCategory(TypeCategory.TRAIT);
+ for (String traitTypeName : traitNames) {
+ // Not cached yet
+ Assert.assertFalse(typeCache.isCachedInMemory(traitTypeName));
+
+ // Calling has() should result in type and its dependencies
+ // loaded from the type store and added to the cache.
+ Assert.assertTrue(typeCache.has(traitTypeName));
+
+ // Verify the type is now cached.
+ Assert.assertTrue(typeCache.isCachedInMemory(traitTypeName));
+ }
+ }
+
+ private <T extends HierarchicalType> void verifyHierarchicalType(T dataType, T expectedDataType) throws AtlasException {
+ Assert.assertEquals(dataType.numFields, expectedDataType.numFields);
+ Assert.assertEquals(dataType.immediateAttrs.size(), expectedDataType.immediateAttrs.size());
+ Assert.assertEquals(dataType.fieldMapping().fields.size(), expectedDataType.fieldMapping().fields.size());
+ ImmutableSet<String> superTypes = dataType.superTypes;
+ Assert.assertEquals(superTypes.size(), expectedDataType.superTypes.size());
+
+ // Verify that any attribute and super types were also cached.
+ for (String superTypeName : superTypes) {
+ Assert.assertTrue(typeCache.has(superTypeName));
+ }
+ for (AttributeInfo attrInfo : dataType.fieldMapping().fields.values()) {
+ switch (attrInfo.dataType().getTypeCategory()) {
+ case CLASS:
+ case STRUCT:
+ case ENUM:
+ Assert.assertTrue(typeCache.has(attrInfo.dataType().getName()), attrInfo.dataType().getName() + " should be cached");
+ break;
+ case ARRAY:
+ String elementTypeName = TypeUtils.parseAsArrayType(attrInfo.dataType().getName());
+ if (!ts.getCoreTypes().contains(elementTypeName)) {
+ Assert.assertTrue(typeCache.has(elementTypeName), elementTypeName + " should be cached");
+ }
+ break;
+ case MAP:
+ String[] mapTypeNames = TypeUtils.parseAsMapType(attrInfo.dataType().getName());
+ for (String typeName : mapTypeNames) {
+ if (!ts.getCoreTypes().contains(typeName)) {
+ Assert.assertTrue(typeCache.has(typeName), typeName + " should be cached");
+ }
+ }
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java
new file mode 100644
index 0000000..058ed4d
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/StoreBackedTypeCacheTestModule.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.RepositoryMetadataModule;
+import org.apache.commons.configuration.Configuration;
+
+
+/**
+ * Guice module which sets TypeCache implementation class configuration property to {@link StoreBackedTypeCache}.
+ *
+ */
+public class StoreBackedTypeCacheTestModule extends RepositoryMetadataModule {
+ @Override
+ protected Configuration getConfiguration() {
+ try {
+ Configuration configuration = ApplicationProperties.get();
+ configuration.setProperty(RepositoryMetadataModule.TYPE_CACHE_IMPLEMENTATION_PROPERTY,
+ StoreBackedTypeCache.class.getName());
+ return configuration;
+ } catch (AtlasException e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
new file mode 100644
index 0000000..4cb5c67
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/service/StoreBackedTypeCacheMetadataServiceTest.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.service;
+
+import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.typestore.ITypeStore;
+import org.apache.atlas.repository.typestore.StoreBackedTypeCache;
+import org.apache.atlas.repository.typestore.StoreBackedTypeCacheTestModule;
+import org.apache.atlas.services.MetadataService;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
+import org.testng.Assert;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.ImmutableList;
+import com.google.inject.Inject;
+
+
+/**
+ * Verify MetadataService type lookup triggers StoreBackedTypeCache to load type from the store.
+ * StoreBackedTypeCacheTestModule Guice module uses Atlas configuration
+ * which has type cache implementation class set to {@link StoreBackedTypeCache}.
+ */
+@Guice(modules = StoreBackedTypeCacheTestModule.class)
+public class StoreBackedTypeCacheMetadataServiceTest
+{
+ @Inject
+ private MetadataService metadataService;
+
+ @Inject
+ private ITypeStore typeStore;
+
+ @Inject
+ TypeCache typeCache;
+
+ private TypeSystem ts;
+
+ @BeforeClass
+ public void setUp() throws Exception {
+ ts = TypeSystem.getInstance();
+ ts.reset();
+
+ // Populate the type store for testing.
+ TestUtils.defineDeptEmployeeTypes(ts);
+ TestUtils.createHiveTypes(ts);
+ ImmutableList<String> typeNames = ts.getTypeNames();
+ typeStore.store(ts, typeNames);
+ ts.reset();
+ }
+
+ @Test
+ public void testIt() throws Exception {
+ Assert.assertTrue(typeCache instanceof StoreBackedTypeCache);
+ StoreBackedTypeCache storeBackedCache = (StoreBackedTypeCache) typeCache;
+
+ // Cache should be empty
+ Assert.assertFalse(storeBackedCache.isCachedInMemory("Manager"));
+
+ // Type lookup on MetadataService should cause Manager type to be loaded from the type store
+ // and cached.
+ Assert.assertNotNull(metadataService.getTypeDefinition("Manager"));
+ Assert.assertTrue(storeBackedCache.isCachedInMemory("Manager"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
index effee2a..393b539 100644
--- a/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
+++ b/repository/src/test/java/org/apache/atlas/services/DefaultMetadataServiceMockTest.java
@@ -19,6 +19,7 @@
package org.apache.atlas.services;
import com.google.inject.Provider;
+
import org.apache.atlas.AtlasException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
@@ -26,12 +27,8 @@ import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.ha.HAConfiguration;
-import org.apache.atlas.listener.TypesChangeListener;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.typestore.ITypeStore;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.commons.configuration.Configuration;
import org.mockito.Matchers;
import org.mockito.Mock;
@@ -44,6 +41,7 @@ import java.util.HashMap;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyZeroInteractions;
@@ -78,7 +76,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(mock(MetadataRepository.class),
mock(ITypeStore.class),
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+ new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verify(typesRegistrar).registerTypes(ReservedTypesRegistrar.getTypesDir(),
typeSystem, defaultMetadataService);
@@ -91,10 +89,10 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+ new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
verifyZeroInteractions(typeStore);
- verifyZeroInteractions(typeSystem);
+ verify(typeSystem, never()).defineTypes(Matchers.<TypesDef>any());
verifyZeroInteractions(typesRegistrar);
}
@@ -109,7 +107,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+ new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
verify(typeStore).restore();
@@ -134,7 +132,7 @@ public class DefaultMetadataServiceMockTest {
DefaultMetadataService defaultMetadataService = new DefaultMetadataService(metadataRepository,
typeStore,
typesRegistrar, new ArrayList<Provider<TypesChangeListener>>(),
- new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration);
+ new ArrayList<Provider<EntityChangeListener>>(), typeSystem, configuration, null);
defaultMetadataService.instanceIsActive();
defaultMetadataService.instanceIsPassive();
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
index 14f1968..90b5815 100755
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/TypeSystem.java
@@ -30,14 +30,13 @@ import java.util.concurrent.ConcurrentHashMap;
import javax.inject.Singleton;
-import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.exception.TypeExistsException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
-import org.apache.atlas.typesystem.types.cache.DefaultTypeCacheProvider;
-import org.apache.atlas.typesystem.types.cache.ITypeCacheProvider;
+import org.apache.atlas.typesystem.types.cache.DefaultTypeCache;
+import org.apache.atlas.typesystem.types.cache.TypeCache;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +47,6 @@ import com.google.common.collect.ImmutableSet;
@InterfaceAudience.Private
public class TypeSystem {
private static final Logger LOG = LoggerFactory.getLogger(TypeSystem.class);
- private static final String CACHE_PROVIDER_CLASS_PROPERTY = "atlas.typesystem.cache.provider";
private static final TypeSystem INSTANCE = new TypeSystem();
private static ThreadLocal<SimpleDateFormat> dateFormat = new ThreadLocal<SimpleDateFormat>() {
@@ -60,7 +58,7 @@ public class TypeSystem {
}
};
- private ITypeCacheProvider typeCache;
+ private TypeCache typeCache = new DefaultTypeCache();
private IdType idType;
private Map<String, IDataType> coreTypes;
@@ -84,44 +82,18 @@ public class TypeSystem {
return this;
}
+ public void setTypeCache(TypeCache typeCache) {
+ this.typeCache = typeCache;
+ }
+
private void initialize() {
- initCacheProvider();
coreTypes = new ConcurrentHashMap<>();
registerPrimitiveTypes();
registerCoreTypes();
}
- /**
- * Ideally a cache provider should have been injected in the TypeSystemProvider,
- * but a singleton of TypeSystem is constructed privately within the class so that
- * clients of TypeSystem would never instantiate a TypeSystem object directly in
- * their code. As soon as a client makes a call to TypeSystem.getInstance(), they
- * should have the singleton ready for consumption. To enable such an access pattern,
- * it kind of becomes imperative to initialize the cache provider within the
- * TypeSystem constructor (bypassing the GUICE way of injecting a cache provider)
- */
- private void initCacheProvider() {
-
- // read the pluggable cache provider from Atlas configuration
- final String defaultCacheProvider = DefaultTypeCacheProvider.class.getName();
- Class cacheProviderClass;
- try {
- cacheProviderClass = ApplicationProperties.getClass(CACHE_PROVIDER_CLASS_PROPERTY,
- defaultCacheProvider, ITypeCacheProvider.class);
- } catch (AtlasException e) {
- throw new RuntimeException("Error getting type cache provider implementation class", e);
- }
-
- try {
- typeCache = (ITypeCacheProvider)cacheProviderClass.newInstance();
- }
- catch (Exception e) {
- throw new RuntimeException("Error creating instance of type cache provider implementation class " + cacheProviderClass.getName(), e);
- }
- }
-
public ImmutableList<String> getCoreTypes() {
return ImmutableList.copyOf(coreTypes.keySet());
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
new file mode 100644
index 0000000..f683ed0
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCache.java
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.typesystem.types.cache;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+
+import com.google.inject.Singleton;
+
+/**
+ * Caches the types in-memory within the same process space.
+ */
+@Singleton
+@SuppressWarnings("rawtypes")
+public class DefaultTypeCache implements TypeCache {
+
+ private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#has(java.lang
+ * .String)
+ */
+ @Override
+ public boolean has(String typeName) throws AtlasException {
+
+ return types_.containsKey(typeName);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.typesystem.types.cache.TypeCache#has(org.
+ * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+ */
+ @Override
+ public boolean has(TypeCategory typeCategory, String typeName)
+ throws AtlasException {
+
+ assertValidTypeCategory(typeCategory);
+ return has(typeName);
+ }
+
+ private void assertValidTypeCategory(TypeCategory typeCategory) throws
+ AtlasException {
+
+ // there might no need of 'typeCategory' in this implementation for
+ // certain API, but for a distributed cache, it might help for the
+ // implementers to partition the types per their category
+ // while persisting so that look can be efficient
+
+ if (typeCategory == null) {
+ throw new AtlasException("Category of the types to be filtered is null.");
+ }
+
+ boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) ||
+ typeCategory.equals(TypeCategory.TRAIT) ||
+ typeCategory.equals(TypeCategory.ENUM) ||
+ typeCategory.equals(TypeCategory.STRUCT);
+
+ if (!validTypeCategory) {
+ throw new AtlasException("Category of the types should be one of CLASS "
+ + "| TRAIT | ENUM | STRUCT.");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#get(java.lang
+ * .String)
+ */
+ @Override
+ public IDataType get(String typeName) throws AtlasException {
+
+ return types_.get(typeName);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.typesystem.types.cache.TypeCache#get(org.apache.
+ * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+ */
+ @Override
+ public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException {
+
+ assertValidTypeCategory(typeCategory);
+ return get(typeName);
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#getNames(org
+ * .apache.atlas.typesystem.types.DataTypes.TypeCategory)
+ */
+ @Override
+ public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException {
+
+ assertValidTypeCategory(typeCategory);
+
+ List<String> typeNames = new ArrayList<>();
+ for (Entry<String, IDataType> typeEntry : types_.entrySet()) {
+ String name = typeEntry.getKey();
+ IDataType type = typeEntry.getValue();
+
+ if (type.getTypeCategory().equals(typeCategory)) {
+ typeNames.add(name);
+ }
+ }
+ return typeNames;
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#getAllNames()
+ */
+ @Override
+ public Collection<String> getAllTypeNames() throws AtlasException {
+
+ return types_.keySet();
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#put(org.apache
+ * .atlas.typesystem.types.IDataType)
+ */
+ @Override
+ public void put(IDataType type) throws AtlasException {
+
+ assertValidType(type);
+ types_.put(type.getName(), type);
+ }
+
+ private void assertValidType(IDataType type) throws
+ AtlasException {
+
+ if (type == null) {
+ throw new AtlasException("type is null.");
+ }
+
+ boolean validTypeCategory = (type instanceof ClassType) ||
+ (type instanceof TraitType) ||
+ (type instanceof EnumType) ||
+ (type instanceof StructType);
+
+ if (!validTypeCategory) {
+ throw new AtlasException("Category of the types should be one of ClassType | "
+ + "TraitType | EnumType | StructType.");
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#putAll(java
+ * .util.Collection)
+ */
+ @Override
+ public void putAll(Collection<IDataType> types) throws AtlasException {
+
+ for (IDataType type : types) {
+ assertValidType(type);
+ types_.put(type.getName(), type);
+ }
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see
+ * org.apache.atlas.typesystem.types.cache.TypeCache#remove(java
+ * .lang.String)
+ */
+ @Override
+ public void remove(String typeName) throws AtlasException {
+
+ types_.remove(typeName);
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.atlas.typesystem.types.cache.TypeCache#remove(org.
+ * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
+ */
+ @Override
+ public void remove(TypeCategory typeCategory, String typeName)
+ throws AtlasException {
+
+ assertValidTypeCategory(typeCategory);
+ remove(typeName);
+ }
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.atlas.typesystem.types.cache.TypeCache#clear()
+ */
+ @Override
+ public void clear() {
+
+ types_.clear();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java
deleted file mode 100644
index 700bda6..0000000
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/DefaultTypeCacheProvider.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.typesystem.types.cache;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
-import org.apache.atlas.typesystem.types.EnumType;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.TraitType;
-
-import com.google.inject.Singleton;
-
-/**
- * Caches the types in-memory within the same process space.
- */
-@Singleton
-@SuppressWarnings("rawtypes")
-public class DefaultTypeCacheProvider implements ITypeCacheProvider {
-
- private Map<String, IDataType> types_ = new ConcurrentHashMap<>();
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(java.lang
- * .String)
- */
- @Override
- public boolean has(String typeName) throws AtlasException {
-
- return types_.containsKey(typeName);
- }
-
- /* (non-Javadoc)
- * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#has(org.
- * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
- */
- @Override
- public boolean has(TypeCategory typeCategory, String typeName)
- throws AtlasException {
-
- assertValidTypeCategory(typeCategory);
- return has(typeName);
- }
-
- private void assertValidTypeCategory(TypeCategory typeCategory) throws
- AtlasException {
-
- // there might no need of 'typeCategory' in this implementation for
- // certain API, but for a distributed cache, it might help for the
- // implementers to partition the types per their category
- // while persisting so that look can be efficient
-
- if (typeCategory == null) {
- throw new AtlasException("Category of the types to be filtered is null.");
- }
-
- boolean validTypeCategory = typeCategory.equals(TypeCategory.CLASS) ||
- typeCategory.equals(TypeCategory.TRAIT) ||
- typeCategory.equals(TypeCategory.ENUM) ||
- typeCategory.equals(TypeCategory.STRUCT);
-
- if (!validTypeCategory) {
- throw new AtlasException("Category of the types should be one of CLASS "
- + "| TRAIT | ENUM | STRUCT.");
- }
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(java.lang
- * .String)
- */
- @Override
- public IDataType get(String typeName) throws AtlasException {
-
- return types_.get(typeName);
- }
-
- /* (non-Javadoc)
- * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#get(org.apache.
- * atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
- */
- @Override
- public IDataType get(TypeCategory typeCategory, String typeName) throws AtlasException {
-
- assertValidTypeCategory(typeCategory);
- return get(typeName);
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getNames(org
- * .apache.atlas.typesystem.types.DataTypes.TypeCategory)
- */
- @Override
- public Collection<String> getTypeNames(TypeCategory typeCategory) throws AtlasException {
-
- assertValidTypeCategory(typeCategory);
-
- List<String> typeNames = new ArrayList<>();
- for (Entry<String, IDataType> typeEntry : types_.entrySet()) {
- String name = typeEntry.getKey();
- IDataType type = typeEntry.getValue();
-
- if (type.getTypeCategory().equals(typeCategory)) {
- typeNames.add(name);
- }
- }
- return typeNames;
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#getAllNames()
- */
- @Override
- public Collection<String> getAllTypeNames() throws AtlasException {
-
- return types_.keySet();
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#put(org.apache
- * .atlas.typesystem.types.IDataType)
- */
- @Override
- public void put(IDataType type) throws AtlasException {
-
- assertValidType(type);
- types_.put(type.getName(), type);
- }
-
- private void assertValidType(IDataType type) throws
- AtlasException {
-
- if (type == null) {
- throw new AtlasException("type is null.");
- }
-
- boolean validTypeCategory = (type instanceof ClassType) ||
- (type instanceof TraitType) ||
- (type instanceof EnumType) ||
- (type instanceof StructType);
-
- if (!validTypeCategory) {
- throw new AtlasException("Category of the types should be one of ClassType | "
- + "TraitType | EnumType | StructType.");
- }
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#putAll(java
- * .util.Collection)
- */
- @Override
- public void putAll(Collection<IDataType> types) throws AtlasException {
-
- for (IDataType type : types) {
- assertValidType(type);
- types_.put(type.getName(), type);
- }
- }
-
- /*
- * (non-Javadoc)
- * @see
- * org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(java
- * .lang.String)
- */
- @Override
- public void remove(String typeName) throws AtlasException {
-
- types_.remove(typeName);
- }
-
- /* (non-Javadoc)
- * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#remove(org.
- * apache.atlas.typesystem.types.DataTypes.TypeCategory, java.lang.String)
- */
- @Override
- public void remove(TypeCategory typeCategory, String typeName)
- throws AtlasException {
-
- assertValidTypeCategory(typeCategory);
- remove(typeName);
- }
-
- /*
- * (non-Javadoc)
- * @see org.apache.atlas.typesystem.types.cache.ITypeCacheProvider#clear()
- */
- @Override
- public void clear() {
-
- types_.clear();
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java
deleted file mode 100644
index 55e9ddc..0000000
--- a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/ITypeCacheProvider.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.typesystem.types.cache;
-
-import java.util.Collection;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.IDataType;
-
-/**
- * The types are cached to allow faster lookup when type info is needed during
- * creation/updation of entities, DSL query translation/execution.
- * Implementations of this can chose to plugin a distributed cache provider
- * or an in-memory cache synched across nodes in an Altas cluster. <br>
- * <br>
- * Type entries in the cache can be one of ... <br>
- * {@link org.apache.atlas.typesystem.types.ClassType} <br>
- * {@link org.apache.atlas.typesystem.types.TraitType} <br>
- * {@link org.apache.atlas.typesystem.types.StructType} <br>
- * {@link org.apache.atlas.typesystem.types.EnumType}
- */
-@SuppressWarnings("rawtypes")
-public interface ITypeCacheProvider {
-
- /**
- * @param typeName
- * @return true if the type exists in cache, false otherwise.
- * @throws AtlasException
- */
- boolean has(String typeName) throws AtlasException;
-
- /**
- * @param typeCategory Non-null category of type. The category can be one of
- * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
- * @param typeName
- * @return true if the type of given category exists in cache, false otherwise.
- * @throws AtlasException
- */
- boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
-
- /**
- * @param name The name of the type.
- * @return returns non-null type if cached, otherwise null
- * @throws AtlasException
- */
- public IDataType get(String typeName) throws AtlasException;
-
- /**
- * @param typeCategory Non-null category of type. The category can be one of
- * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
- * @param typeName
- * @return returns non-null type (of the specified category) if cached, otherwise null
- * @throws AtlasException
- */
- public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
-
- /**
- * @param typeCategory The category of types to filter the returned types. Cannot be null.
- * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT |
- * TypeCategory.STRUCT | TypeCategory.ENUM.
- * @return
- * @throws AtlasException
- */
- Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException;
-
- /**
- * This is a convenience API to get the names of all types.
- *
- * @see ITypeCacheProvider#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
- * @return
- * @throws AtlasException
- */
- Collection<String> getAllTypeNames() throws AtlasException;
-
- /**
- * @param type The type to be added to the cache. The type should not be
- * null, otherwise throws NullPointerException. <br>
- * Type entries in the cache can be one of ... <br>
- * {@link org.apache.atlas.typesystem.types.ClassType} <br>
- * {@link org.apache.atlas.typesystem.types.TraitType} <br>
- * {@link org.apache.atlas.typesystem.types.StructType} <br>
- * {@link org.apache.atlas.typesystem.types.EnumType}
- * @throws AtlasException
- */
- void put(IDataType type) throws AtlasException;
-
- /**
- * @param types The types to be added to the cache. The type should not be
- * null, otherwise throws NullPointerException. <br>
- * Type entries in the cache can be one of ... <br>
- * {@link org.apache.atlas.typesystem.types.ClassType} <br>
- * {@link org.apache.atlas.typesystem.types.TraitType} <br>
- * {@link org.apache.atlas.typesystem.types.StructType} <br>
- * {@link org.apache.atlas.typesystem.types.EnumType}
- * @throws AtlasException
- */
- void putAll(Collection<IDataType> types) throws AtlasException;
-
- /**
- * @param typeName Name of the type to be removed from the cache. If type
- * exists, it will be removed, otherwise does nothing.
- * @throws AtlasException
- */
- void remove(String typeName) throws AtlasException;
-
- /**
- * @param typeCategory Non-null category of type. The category can be one of
- * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
- * @param typeName Name of the type to be removed from the cache. If type
- * exists, it will be removed, otherwise does nothing.
- * @throws AtlasException
- */
- void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
-
- /**
- * Clear the type cache
- *
- */
- void clear();
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
----------------------------------------------------------------------
diff --git a/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
new file mode 100644
index 0000000..27622c2
--- /dev/null
+++ b/typesystem/src/main/java/org/apache/atlas/typesystem/types/cache/TypeCache.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.typesystem.types.cache;
+
+import java.util.Collection;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+
+/**
+ * The types are cached to allow faster lookup when type info is needed during
+ * creation/updation of entities, DSL query translation/execution.
+ * Implementations of this can chose to plugin a distributed cache
+ * or an in-memory cache synched across nodes in an Altas cluster. <br>
+ * <br>
+ * Type entries in the cache can be one of ... <br>
+ * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+ * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+ * {@link org.apache.atlas.typesystem.types.StructType} <br>
+ * {@link org.apache.atlas.typesystem.types.EnumType}
+ */
+@SuppressWarnings("rawtypes")
+public interface TypeCache {
+
+ /**
+ * @param typeName
+ * @return true if the type exists in cache, false otherwise.
+ * @throws AtlasException
+ */
+ boolean has(String typeName) throws AtlasException;
+
+ /**
+ * @param typeCategory Non-null category of type. The category can be one of
+ * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+ * @param typeName
+ * @return true if the type of given category exists in cache, false otherwise.
+ * @throws AtlasException
+ */
+ boolean has(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+ /**
+ * @param name The name of the type.
+ * @return returns non-null type if cached, otherwise null
+ * @throws AtlasException
+ */
+ public IDataType get(String typeName) throws AtlasException;
+
+ /**
+ * @param typeCategory Non-null category of type. The category can be one of
+ * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+ * @param typeName
+ * @return returns non-null type (of the specified category) if cached, otherwise null
+ * @throws AtlasException
+ */
+ public IDataType get(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+ /**
+ * @param typeCategory The category of types to filter the returned types. Cannot be null.
+ * The category can be one of TypeCategory.CLASS | TypeCategory.TRAIT |
+ * TypeCategory.STRUCT | TypeCategory.ENUM.
+ * @return
+ * @throws AtlasException
+ */
+ Collection<String> getTypeNames(DataTypes.TypeCategory typeCategory) throws AtlasException;
+
+ /**
+ * This is a convenience API to get the names of all types.
+ *
+ * @see TypeCache#getTypeNames(org.apache.atlas.typesystem.types.DataTypes.TypeCategory)
+ * @return
+ * @throws AtlasException
+ */
+ Collection<String> getAllTypeNames() throws AtlasException;
+
+ /**
+ * @param type The type to be added to the cache. The type should not be
+ * null, otherwise throws NullPointerException. <br>
+ * Type entries in the cache can be one of ... <br>
+ * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+ * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+ * {@link org.apache.atlas.typesystem.types.StructType} <br>
+ * {@link org.apache.atlas.typesystem.types.EnumType}
+ * @throws AtlasException
+ */
+ void put(IDataType type) throws AtlasException;
+
+ /**
+ * @param types The types to be added to the cache. The type should not be
+ * null, otherwise throws NullPointerException. <br>
+ * Type entries in the cache can be one of ... <br>
+ * {@link org.apache.atlas.typesystem.types.ClassType} <br>
+ * {@link org.apache.atlas.typesystem.types.TraitType} <br>
+ * {@link org.apache.atlas.typesystem.types.StructType} <br>
+ * {@link org.apache.atlas.typesystem.types.EnumType}
+ * @throws AtlasException
+ */
+ void putAll(Collection<IDataType> types) throws AtlasException;
+
+ /**
+ * @param typeName Name of the type to be removed from the cache. If type
+ * exists, it will be removed, otherwise does nothing.
+ * @throws AtlasException
+ */
+ void remove(String typeName) throws AtlasException;
+
+ /**
+ * @param typeCategory Non-null category of type. The category can be one of
+ * TypeCategory.CLASS | TypeCategory.TRAIT | TypeCategory.STRUCT | TypeCategory.ENUM.
+ * @param typeName Name of the type to be removed from the cache. If type
+ * exists, it will be removed, otherwise does nothing.
+ * @throws AtlasException
+ */
+ void remove(DataTypes.TypeCategory typeCategory, String typeName) throws AtlasException;
+
+ /**
+ * Clear the type cache
+ *
+ */
+ void clear();
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/ee8a2153/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
----------------------------------------------------------------------
diff --git a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
index 0679642..6d43359 100644
--- a/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
+++ b/typesystem/src/test/java/org/apache/atlas/ApplicationPropertiesTest.java
@@ -58,17 +58,22 @@ public class ApplicationPropertiesTest {
@Test
public void testGetClass() throws Exception {
+ Configuration configuration = ApplicationProperties.get();
+
//read from atlas-application.properties
- Class cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), TypeSystem.class);
+ Class cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
+ ApplicationProperties.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//default value
- cls = ApplicationProperties.getClass("atlas.TypeSystem2.impl", TypeSystem.class.getName(), TypeSystem.class);
+ cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem2.impl",
+ TypeSystem.class.getName(), TypeSystem.class);
assertEquals(cls.getName(), TypeSystem.class.getName());
//incompatible assignTo class, should throw AtlasException
try {
- cls = ApplicationProperties.getClass("atlas.TypeSystem.impl", ApplicationProperties.class.getName(), ApplicationProperties.class);
+ cls = ApplicationProperties.getClass(configuration, "atlas.TypeSystem.impl",
+ ApplicationProperties.class.getName(), ApplicationProperties.class);
Assert.fail(AtlasException.class.getSimpleName() + " was expected but none thrown.");
}
catch (AtlasException e) {