You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by jn...@apache.org on 2017/02/02 22:35:39 UTC
incubator-atlas git commit: ATLAS-1388: Cache entities that are
created/updated
Repository: incubator-atlas
Updated Branches:
refs/heads/master 62a05c97c -> 09089e09f
ATLAS-1388: Cache entities that are created/updated
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/09089e09
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/09089e09
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/09089e09
Branch: refs/heads/master
Commit: 09089e09f80c1f10123d4132438f7c426087acb1
Parents: 62a05c9
Author: Jeff Hagelberg <jn...@us.ibm.com>
Authored: Thu Feb 2 17:35:25 2017 -0500
Committer: Jeff Hagelberg <jn...@us.ibm.com>
Committed: Thu Feb 2 17:35:25 2017 -0500
----------------------------------------------------------------------
release-log.txt | 1 +
.../graph/DefaultGraphPersistenceStrategy.java | 20 ++-
.../atlas/repository/graph/FullTextMapper.java | 11 +-
.../graph/GraphBackedMetadataRepository.java | 4 +
.../graph/GraphToTypedInstanceMapper.java | 39 ++++--
.../graph/TypedInstanceToGraphMapper.java | 22 +++
.../test/java/org/apache/atlas/TestUtils.java | 140 +++++++++++++++----
.../GraphBackedDiscoveryServiceTest.java | 1 +
...hBackedMetadataRepositoryDeleteTestBase.java | 62 ++------
.../GraphBackedMetadataRepositoryTest.java | 7 +-
.../store/graph/v1/AtlasEntityStoreV1Test.java | 2 +
.../service/DefaultMetadataServiceTest.java | 68 +++++----
.../java/org/apache/atlas/RequestContext.java | 37 ++++-
13 files changed, 273 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 9dd7d6f..991557b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
+ATLAS-1388 Cache entities that are created/updated (jnhagelb)
ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
ATLAS-1517: updated hive_model to include schema related attributes (sarath.kum4r@gmail.com via mneethiraj)
ATLAS-1514 Remove duplicates from class array attribute when target is deleted (dkantor)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
index f4d8f00..1f2a754 100755
--- a/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/graph/DefaultGraphPersistenceStrategy.java
@@ -18,9 +18,12 @@
package org.apache.atlas.discovery.graph;
-import com.google.common.collect.ImmutableCollection;
-import com.google.common.collect.ImmutableList;
+import java.util.List;
+
+import javax.inject.Inject;
+
import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.query.GraphPersistenceStrategies;
import org.apache.atlas.query.GraphPersistenceStrategies$class;
@@ -48,8 +51,8 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.inject.Inject;
-import java.util.List;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
/**
* Default implementation of GraphPersistenceStrategy.
@@ -178,9 +181,12 @@ public class DefaultGraphPersistenceStrategy implements GraphPersistenceStrategi
case CLASS:
AtlasVertex classVertex = (AtlasVertex) value;
- ITypedReferenceableInstance classInstance = metadataRepository.getGraphToInstanceMapper()
- .mapGraphToTypedInstance(GraphHelper.getSingleValuedProperty(classVertex, Constants.GUID_PROPERTY_KEY, String.class),
- classVertex);
+ String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class);
+ // Check if the instance we need was previously loaded.
+ ITypedReferenceableInstance classInstance = RequestContext.get().getInstance(guid);
+ if (classInstance == null) {
+ classInstance = metadataRepository.getGraphToInstanceMapper().mapGraphToTypedInstance(guid, classVertex);
+ }
return dataType.convert(classInstance, Multiplicity.OPTIONAL);
default:
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
index 5be8d0b..b988b42 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
@@ -17,11 +17,11 @@
*/
package org.apache.atlas.repository.graph;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
@@ -44,20 +44,19 @@ public class FullTextMapper {
private static final GraphHelper graphHelper = GraphHelper.getInstance();
private static final String FULL_TEXT_DELIMITER = " ";
- private final Map<String, ITypedReferenceableInstance> instanceCache;
FullTextMapper(TypedInstanceToGraphMapper typedInstanceToGraphMapper,
GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
this.typedInstanceToGraphMapper = typedInstanceToGraphMapper;
- instanceCache = new HashMap<>();
}
public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException {
String guid = GraphHelper.getGuid(instanceVertex);
ITypedReferenceableInstance typedReference;
- if (instanceCache.containsKey(guid)) {
- typedReference = instanceCache.get(guid);
+ RequestContext context = RequestContext.get();
+ typedReference = context.getInstance(guid);
+ if (typedReference != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Cache hit: guid = {}, entityId = {}", guid, typedReference.getId()._getId());
@@ -65,7 +64,7 @@ public class FullTextMapper {
} else {
typedReference =
graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- instanceCache.put(guid, typedReference);
+ context.cache(typedReference);
if (LOG.isDebugEnabled()) {
LOG.debug("Cache miss: guid = {}, entityId = {}", guid, typedReference.getId().getId());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index 6608551..27bf6d7 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -191,6 +191,10 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
String guid = GraphHelper.getGuid(instanceVertex);
+ ITypedReferenceableInstance cached = RequestContext.get().getInstance(guid);
+ if(cached != null) {
+ return cached;
+ }
return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
index 38a553a..dfa4407 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
@@ -17,8 +17,18 @@
*/
package org.apache.atlas.repository.graph;
-import com.google.inject.Singleton;
+import static org.apache.atlas.repository.graph.GraphHelper.string;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
import org.apache.atlas.AtlasException;
+import org.apache.atlas.RequestContext;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -28,10 +38,8 @@ import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.ITypedStruct;
-import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.persistence.AtlasSystemAttributes;
import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
@@ -43,15 +51,7 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.atlas.repository.graph.GraphHelper.string;
+import com.google.inject.Singleton;
@Singleton
public final class GraphToTypedInstanceMapper {
@@ -69,7 +69,14 @@ public final class GraphToTypedInstanceMapper {
public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, AtlasVertex instanceVertex)
throws AtlasException {
- if (LOG.isDebugEnabled()) {
+ if(LOG.isDebugEnabled()) {
+ //We don't do a cache check here since we want that to be at a higher level
+ //where the vertex lookup can also be avoided. However, this is a convenient
+ //place to add a check to see if there are any places that were missed.
+ if(RequestContext.get().getInstance(guid) != null) {
+ LOG.warn("Looking up previously cached guid at: ", new Exception());
+ }
+
LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
}
@@ -99,7 +106,7 @@ public final class GraphToTypedInstanceMapper {
mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
-
+ RequestContext.get().cache(typedInstance);
return typedInstance;
}
@@ -209,6 +216,10 @@ public final class GraphToTypedInstanceMapper {
if (attributeInfo.isComposite) {
//Also, when you retrieve a type's instance, you get the complete object graph of the composites
LOG.debug("Found composite, mapping vertex to instance");
+ ITypedReferenceableInstance cached = RequestContext.get().getInstance(guid);
+ if(cached != null) {
+ return cached;
+ }
return mapGraphToTypedInstance(guid, referenceVertex);
} else {
String state = GraphHelper.getStateAsString(referenceVertex);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
index bae8b2a..1a88251 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -125,6 +125,9 @@ public final class TypedInstanceToGraphMapper {
throw new UnsupportedOperationException("Not handled - " + operation);
}
+ for(ITypedReferenceableInstance instance : typedInstances) {
+ addToEntityCache(requestContext, instance);
+ }
}
private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
@@ -825,4 +828,23 @@ public final class TypedInstanceToGraphMapper {
public AtlasVertex lookupVertex(Id refId) {
return idToVertexMap.get(refId);
}
+
+ private void addToEntityCache(RequestContext context, ITypedReferenceableInstance instance)
+ throws EntityNotFoundException {
+
+ Id instanceId = instance.getId();
+ if(instanceId.isUnassigned()) {
+ if(instance instanceof ReferenceableInstance) {
+ //When the id is unassigned, we can only cache the instance of it is
+ //an instance of ReferenceableInstance, since replaceWithNewId is not
+ //currently in the ITypedReferenceableInstance interface.
+ Id id = getId(instance);
+ ((ReferenceableInstance)instance).replaceWithNewId(id);
+ context.cache(instance);
+ }
+ }
+ else {
+ context.cache(instance);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index cda9eac..1d1a5e0 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -18,14 +18,33 @@
package org.apache.atlas;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Provider;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Proxy;
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
+import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasGraph;
@@ -59,23 +78,9 @@ import org.apache.commons.lang.RandomStringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.testng.Assert;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.math.BigDecimal;
-import java.math.BigInteger;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Date;
-import java.util.List;
-
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createTraitTypeDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Provider;
/**
* Test utility class.
@@ -505,11 +510,14 @@ public final class TestUtils {
}
return null;
}
-
- public static void resetRequestContext() {
+
+ public static void resetRequestContext() {
+ //reset the context while preserving the user
+ String user = RequestContext.get().getUser();
RequestContext.createContext();
+ RequestContext.get().setUser(user);
}
-
+
public static void setupGraphProvider(MetadataRepository repo) throws AtlasException {
TypeCache typeCache = null;
try {
@@ -538,10 +546,92 @@ public final class TestUtils {
getGraph().commit();
}
-
+
public static AtlasGraph getGraph() {
return AtlasGraphProvider.getGraphInstance();
-
+
+ }
+
+ /**
+ * Adds a proxy wrapper around the specified MetadataService that automatically
+ * resets the request context before every call.
+ *
+ * @param delegate
+ * @return
+ */
+ public static MetadataService addSessionCleanupWrapper(final MetadataService delegate) {
+
+ return (MetadataService)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
+ new Class[]{MetadataService.class}, new InvocationHandler() {
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+
+ try {
+ resetRequestContext();
+ Object result = method.invoke(delegate, args);
+
+ return result;
+ }
+ catch(InvocationTargetException e) {
+ e.getCause().printStackTrace();
+ throw e.getCause();
+ }
+ catch(Throwable t) {
+ t.printStackTrace();
+ throw t;
+ }
+ }
+
+ });
+ }
+
+ /**
+ * Adds a proxy wrapper around the specified MetadataRepository that automatically
+ * resets the request context before every call and either commits or rolls
+ * back the graph transaction after every call.
+ *
+ * @param delegate
+ * @return
+ */
+ public static MetadataRepository addTransactionWrapper(final MetadataRepository delegate) {
+ return (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
+ new Class[]{MetadataRepository.class}, new InvocationHandler() {
+
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ boolean useTransaction = GraphBackedMetadataRepository.class.getMethod(
+ method.getName(), method.getParameterTypes())
+ .isAnnotationPresent(GraphTransaction.class);
+ try {
+ resetRequestContext();
+ Object result = method.invoke(delegate, args);
+ if(useTransaction) {
+ System.out.println("Committing changes");
+ getGraph().commit();
+ System.out.println("Commit succeeded.");
+ }
+ return result;
+ }
+ catch(InvocationTargetException e) {
+ e.getCause().printStackTrace();
+ if(useTransaction) {
+ System.out.println("Rolling back changes due to exception.");
+ getGraph().rollback();
+ }
+ throw e.getCause();
+ }
+ catch(Throwable t) {
+ t.printStackTrace();
+ if(useTransaction) {
+ System.out.println("Rolling back changes due to exception.");
+ getGraph().rollback();
+ }
+ throw t;
+ }
+ }
+
+ });
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index d447c2d..ce87c9e 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -87,6 +87,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseRepositoryTest {
public void setUp() throws Exception {
super.setUp();
+ repositoryService = TestUtils.addTransactionWrapper(repositoryService);
final TypeSystem typeSystem = TypeSystem.getInstance();
Collection<String> oldTypeNames = new HashSet<>();
oldTypeNames.addAll(typeSystem.getTypeNames());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
index 4919b08..d6136ed 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryDeleteTestBase.java
@@ -114,43 +114,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
new GraphBackedSearchIndexer(new AtlasTypeRegistry());
final GraphBackedMetadataRepository delegate = new GraphBackedMetadataRepository(getDeleteHandler(typeSystem));
- repositoryService = (MetadataRepository)Proxy.newProxyInstance(Thread.currentThread().getContextClassLoader(),
- new Class[]{MetadataRepository.class}, new InvocationHandler() {
-
- @Override
- public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
- boolean useTransaction = GraphBackedMetadataRepository.class.getMethod(
- method.getName(), method.getParameterTypes())
- .isAnnotationPresent(GraphTransaction.class);
- try {
-
- Object result = method.invoke(delegate, args);
- if(useTransaction) {
- System.out.println("Committing changes");
- TestUtils.getGraph().commit();
- System.out.println("Commit succeeded.");
- }
- return result;
- }
- catch(InvocationTargetException e) {
- e.getCause().printStackTrace();
- if(useTransaction) {
- System.out.println("Rolling back changes due to exception.");
- TestUtils.getGraph().rollback();
- }
- throw e.getCause();
- }
- catch(Throwable t) {
- t.printStackTrace();
- if(useTransaction) {
- System.out.println("Rolling back changes due to exception.");
- TestUtils.getGraph().rollback();
- }
- throw t;
- }
- }
-
- });
+ repositoryService = TestUtils.addTransactionWrapper(delegate);
TestUtils.defineDeptEmployeeTypes(typeSystem);
TestUtils.createHiveTypes(typeSystem);
@@ -531,7 +495,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
/**
* Verify deleting an entity which is contained by another
* entity through a bi-directional composite reference.
- *
+ *
* @throws Exception
*/
@Test
@@ -633,21 +597,21 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
@Test
public void testDisconnectUnidirectionalArrayReferenceFromStructAndTraitTypes() throws Exception {
// Define class types.
- HierarchicalTypeDefinition<ClassType> structTargetDef = TypesUtil.createClassTypeDef("StructTarget",
+ HierarchicalTypeDefinition<ClassType> structTargetDef = TypesUtil.createClassTypeDef("StructTarget",
ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<ClassType> traitTargetDef = TypesUtil.createClassTypeDef("TraitTarget",
+ HierarchicalTypeDefinition<ClassType> traitTargetDef = TypesUtil.createClassTypeDef("TraitTarget",
ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<ClassType> structContainerDef = TypesUtil.createClassTypeDef("StructContainer",
+ HierarchicalTypeDefinition<ClassType> structContainerDef = TypesUtil.createClassTypeDef("StructContainer",
ImmutableSet.<String>of(), TypesUtil.createOptionalAttrDef("struct", "TestStruct"));
// Define struct and trait types which have a unidirectional array reference
// to a class type.
- StructTypeDefinition structDef = TypesUtil.createStructTypeDef("TestStruct",
+ StructTypeDefinition structDef = TypesUtil.createStructTypeDef("TestStruct",
new AttributeDefinition("target", DataTypes.arrayTypeName("StructTarget"), Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("nestedStructs", DataTypes.arrayTypeName("NestedStruct"), Multiplicity.OPTIONAL, false, null));
- StructTypeDefinition nestedStructDef = TypesUtil.createStructTypeDef("NestedStruct",
+ StructTypeDefinition nestedStructDef = TypesUtil.createStructTypeDef("NestedStruct",
TypesUtil.createOptionalAttrDef("attr1", DataTypes.STRING_TYPE));
- HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableSet.<String>of(),
+ HierarchicalTypeDefinition<TraitType> traitDef = TypesUtil.createTraitTypeDef("TestTrait", ImmutableSet.<String>of(),
new AttributeDefinition("target", DataTypes.arrayTypeName("TraitTarget"), Multiplicity.OPTIONAL, false, null));
TypesDef typesDef = TypesUtil.getTypesDef(ImmutableList.<EnumTypeDefinition>of(), ImmutableList.of(structDef, nestedStructDef),
@@ -669,9 +633,9 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
ClassType traitTargetType = typeSystem.getDataType(ClassType.class, "TraitTarget");
ClassType structContainerType = typeSystem.getDataType(ClassType.class, "StructContainer");
- ITypedReferenceableInstance structTargetConvertedEntity =
+ ITypedReferenceableInstance structTargetConvertedEntity =
structTargetType.convert(structTargetEntity, Multiplicity.REQUIRED);
- ITypedReferenceableInstance traitTargetConvertedEntity =
+ ITypedReferenceableInstance traitTargetConvertedEntity =
traitTargetType.convert(traitTargetEntity, Multiplicity.REQUIRED);
ITypedReferenceableInstance structContainerConvertedEntity =
structContainerType.convert(structContainerEntity, Multiplicity.REQUIRED);
@@ -755,13 +719,13 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
@Test
public void testDisconnectMapReferenceFromClassType() throws Exception {
// Define type for map value.
- HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("MapValue",
+ HierarchicalTypeDefinition<ClassType> mapValueDef = TypesUtil.createClassTypeDef("MapValue",
ImmutableSet.<String>of(),
new AttributeDefinition("biMapOwner", "MapOwner", Multiplicity.OPTIONAL, false, "biMap"));
// Define type with unidirectional and bidirectional map references,
// where the map value is a class reference to MapValue.
- HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("MapOwner",
+ HierarchicalTypeDefinition<ClassType> mapOwnerDef = TypesUtil.createClassTypeDef("MapOwner",
ImmutableSet.<String>of(),
new AttributeDefinition("map", DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
"MapValue"), Multiplicity.OPTIONAL, false, null),
@@ -811,7 +775,7 @@ public abstract class GraphBackedMetadataRepositoryDeleteTestBase {
}
// Delete the map value instance.
- // This should disconnect the references from the map owner instance.
+ // This should disconnect the references from the map owner instance.
deleteEntities(mapValueGuid);
assertEntityDeleted(mapValueGuid);
assertTestDisconnectMapReferenceFromClassType(mapOwnerGuid);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
index 7444bf3..96b0173 100755
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepositoryTest.java
@@ -28,6 +28,7 @@ import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
@@ -76,7 +77,7 @@ import java.util.concurrent.Future;
import javax.inject.Inject;
-import scala.actors.threadpool.Arrays;
+import java.util.Arrays;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createUniqueRequiredAttrDef;
@@ -95,7 +96,7 @@ import static org.testng.Assert.assertTrue;
public class GraphBackedMetadataRepositoryTest {
@Inject
- private GraphBackedMetadataRepository repositoryService;
+ private MetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
@@ -109,6 +110,8 @@ public class GraphBackedMetadataRepositoryTest {
typeSystem = TypeSystem.getInstance();
typeSystem.reset();
+ assertTrue(repositoryService instanceof GraphBackedMetadataRepository);
+ repositoryService = TestUtils.addTransactionWrapper(repositoryService);
new GraphBackedSearchIndexer(new AtlasTypeRegistry());
TestUtils.defineDeptEmployeeTypes(typeSystem);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
index 31a619e..546cd0c 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityStoreV1Test.java
@@ -97,6 +97,7 @@ public class AtlasEntityStoreV1Test {
@BeforeClass
public void setUp() throws Exception {
+ metadataService = TestUtils.addSessionCleanupWrapper(metadataService);
new GraphBackedSearchIndexer(typeRegistry);
final AtlasTypesDef deptTypesDef = TestUtilsV2.defineDeptEmployeeTypes();
typeDefStore.createTypesDef(deptTypesDef);
@@ -112,6 +113,7 @@ public class AtlasEntityStoreV1Test {
@AfterClass
public void clear() {
AtlasGraphProvider.cleanup();
+ TestUtils.resetRequestContext();
}
@BeforeTest
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
index 03ef4fe..1ad79b1 100644
--- a/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/service/DefaultMetadataServiceTest.java
@@ -18,9 +18,30 @@
package org.apache.atlas.service;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import com.google.inject.Inject;
+import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
+import static org.apache.atlas.TestUtils.COLUMN_TYPE;
+import static org.apache.atlas.TestUtils.PII;
+import static org.apache.atlas.TestUtils.TABLE_TYPE;
+import static org.apache.atlas.TestUtils.createColumnEntity;
+import static org.apache.atlas.TestUtils.createDBEntity;
+import static org.apache.atlas.TestUtils.createInstance;
+import static org.apache.atlas.TestUtils.createTableEntity;
+import static org.apache.atlas.TestUtils.randomString;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
+import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
+import static org.testng.Assert.assertTrue;
+import static org.testng.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
@@ -32,6 +53,7 @@ import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.ChangedTypeDefs;
import org.apache.atlas.listener.EntityChangeListener;
+import org.apache.atlas.listener.TypeDefChangeListener;
import org.apache.atlas.query.QueryParams;
import org.apache.atlas.repository.audit.EntityAuditRepository;
import org.apache.atlas.repository.audit.HBaseBasedAuditRepository;
@@ -72,37 +94,17 @@ import org.testng.annotations.BeforeTest;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Collections;
-
-import static org.apache.atlas.TestUtils.COLUMNS_ATTR_NAME;
-import static org.apache.atlas.TestUtils.COLUMN_TYPE;
-import static org.apache.atlas.TestUtils.DATABASE_TYPE;
-import static org.apache.atlas.TestUtils.PII;
-import static org.apache.atlas.TestUtils.TABLE_TYPE;
-import static org.apache.atlas.TestUtils.createColumnEntity;
-import static org.apache.atlas.TestUtils.createDBEntity;
-import static org.apache.atlas.TestUtils.createInstance;
-import static org.apache.atlas.TestUtils.createTableEntity;
-import static org.apache.atlas.TestUtils.randomString;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createClassTypeDef;
-import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAttrDef;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.inject.Inject;
@Guice(modules = RepositoryMetadataModule.class)
public class DefaultMetadataServiceTest {
@Inject
private MetadataService metadataService;
+ private TypeDefChangeListener typeDefChangeListener;
+
@Inject
private EntityAuditRepository auditRepository;
@@ -114,12 +116,16 @@ public class DefaultMetadataServiceTest {
private Referenceable table;
private Id tableId;
-
+
private final String NAME = "name";
@BeforeTest
public void setUp() throws Exception {
+
+ typeDefChangeListener = (DefaultMetadataService)metadataService;
+ metadataService = TestUtils.addSessionCleanupWrapper(metadataService);
+
if (auditRepository instanceof HBaseBasedAuditRepository) {
HBaseTestUtils.startCluster();
((HBaseBasedAuditRepository) auditRepository).start();
@@ -1218,7 +1224,7 @@ public class DefaultMetadataServiceTest {
List<String> beforeChangeTypeNames = new ArrayList<>();
beforeChangeTypeNames.addAll(metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>()));
- ((DefaultMetadataService)metadataService).onChange(new ChangedTypeDefs());
+ typeDefChangeListener.onChange(new ChangedTypeDefs());
List<String> afterChangeTypeNames = new ArrayList<>();
afterChangeTypeNames.addAll(metadataService.getTypeNames(new HashMap<TypeCache.TYPE_FILTER, String>()));
@@ -1269,7 +1275,7 @@ public class DefaultMetadataServiceTest {
deletedEntities.add(entity.getId()._getId());
}
}
-
+
public List<String> getDeletedEntities() {
return deletedEntities;
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/09089e09/server-api/src/main/java/org/apache/atlas/RequestContext.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 651a71d..09cdc37 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -18,6 +18,14 @@
package org.apache.atlas;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.atlas.metrics.Metrics;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.persistence.Id;
@@ -26,12 +34,6 @@ import org.apache.atlas.typesystem.types.TypeSystem;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Set;
-
public class RequestContext {
private static final Logger LOG = LoggerFactory.getLogger(RequestContext.class);
@@ -41,6 +43,7 @@ public class RequestContext {
private Set<String> updatedEntityIds = new LinkedHashSet<>();
private Set<String> deletedEntityIds = new LinkedHashSet<>();
private List<ITypedReferenceableInstance> deletedEntities = new ArrayList<>();
+ private Map<String,ITypedReferenceableInstance> entityCache = new HashMap<>();
private String user;
private long requestTime;
@@ -71,7 +74,27 @@ public class RequestContext {
return context;
}
+ /**
+ * Adds the specified instance to the cache
+ *
+ */
+ public void cache(ITypedReferenceableInstance instance) {
+ entityCache.put(instance.getId()._getId(), instance);
+ }
+
+ /**
+ * Checks if an instance with the given guid is in the cache for this request. Either returns the instance
+ * or null if it is not in the cache.
+ *
+ * @param guid the guid to find
+ * @return Either the instance or null if it is not in the cache.
+ */
+ public ITypedReferenceableInstance getInstance(String guid) {
+ return entityCache.get(guid);
+ }
+
public static void clear() {
+ CURRENT_CONTEXT.get().entityCache.clear();
CURRENT_CONTEXT.remove();
}
@@ -122,7 +145,7 @@ public class RequestContext {
public long getRequestTime() {
return requestTime;
}
-
+
public boolean isDeletedEntity(String entityGuid) {
return deletedEntityIds.contains(entityGuid);
}