You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by jn...@apache.org on 2017/02/09 14:23:54 UTC
incubator-atlas git commit: ATLAS-1527 : Batch entity retrievals -
DefaultMetadataService.loadEntities
Repository: incubator-atlas
Updated Branches:
refs/heads/master a67e89625 -> 1e105174c
ATLAS-1527 : Batch entity retrievals - DefaultMetadataService.loadEntities
Signed-off-by: Jeff Hagelberg <jn...@us.ibm.com>
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/1e105174
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/1e105174
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/1e105174
Branch: refs/heads/master
Commit: 1e105174cfc0ba82ad254efaeb666ba880956dea
Parents: a67e896
Author: Wojciech Wojcik <wo...@pl.ibm.com>
Authored: Thu Feb 9 09:22:53 2017 -0500
Committer: Jeff Hagelberg <jn...@us.ibm.com>
Committed: Thu Feb 9 09:23:38 2017 -0500
----------------------------------------------------------------------
release-log.txt | 1 +
.../atlas/repository/MetadataRepository.java | 11 ++++
.../graph/GraphBackedMetadataRepository.java | 69 ++++++++++++++++++--
.../atlas/services/DefaultMetadataService.java | 9 +--
.../GraphBackedRepositoryHardDeleteTest.java | 2 +-
5 files changed, 77 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index c907e67..a444c7a 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
+ATLAS-1527 Batch entity retrievals - DefaultMetadataService.loadEntities
ATLAS-1385 Add configuration property to disable full text mapper (wwojcik via jnhagelb)
ATLAS-746 After updating a set of entities, response contains only the first entity definition (jnhagelb)
ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
old mode 100755
new mode 100644
index 3f31f25..fab1859
--- a/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/MetadataRepository.java
@@ -104,10 +104,21 @@ public interface MetadataRepository {
* @param guid globally unique identifier for the entity
* @return entity (typed instance) definition
* @throws RepositoryException
+ * @throws EntityNotFoundException
*/
ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException;
/**
+ * Fetch the complete entity definitions for the entities with the given GUIDs
+ *
+ * @param guids globally unique identifiers for the entities
+ * @return entity (typed instance) definitions list
+ * @throws RepositoryException
+ * @throws EntityNotFoundException
+ */
+ List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException;
+
+ /**
* Gets the list of entities for a given entity type.
*
* @param entityType name of a type which is unique
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
index 943edef..4461447 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
@@ -19,11 +19,15 @@
package org.apache.atlas.repository.graph;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
@@ -169,19 +173,70 @@ public class GraphBackedMetadataRepository implements MetadataRepository {
}
@Override
- @GraphTransaction
public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
+ return getEntityDefinitions(guid).get(0);
+ }
+
+ @Override
+ @GraphTransaction
+ public List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException {
if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving entity with guid={}", guid);
+ LOG.debug("Retrieving entities with guids={}", Arrays.toString(guids));
}
- AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
+ RequestContext context = RequestContext.get();
+ ITypedReferenceableInstance[] result = new ITypedReferenceableInstance[guids.length];
+
+ // Map of the guids of instances not in the cache to their index(es) in the result.
+ // This is used to put the loaded instances into the location(s) corresponding
+ // to their guid in the result. Note that a set is needed since guids can
+ // appear more than once in the list.
+ Map<String, Set<Integer>> uncachedGuids = new HashMap<>();
+
+ for (int i = 0; i < guids.length; i++) {
+ String guid = guids[i];
+
+ // First, check the cache.
+ ITypedReferenceableInstance cached = context.getInstance(guid);
+ if (cached != null) {
+ result[i] = cached;
+ } else {
+ Set<Integer> indices = uncachedGuids.get(guid);
+ if (indices == null) {
+ indices = new HashSet<>(1);
+ uncachedGuids.put(guid, indices);
+ }
+ indices.add(i);
+ }
+ }
- try {
- return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- } catch (AtlasException e) {
- throw new RepositoryException(e);
+ List<String> guidsToFetch = new ArrayList<>(uncachedGuids.keySet());
+ Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToFetch);
+
+ // search for missing entities
+ if (instanceVertices.size() != guidsToFetch.size()) {
+ Set<String> missingGuids = new HashSet<String>(guidsToFetch);
+ missingGuids.removeAll(instanceVertices.keySet());
+ if (!missingGuids.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Failed to find guids={}", missingGuids);
+ }
+ throw new EntityNotFoundException(
+ "Could not find entities in the repository with guids: " + missingGuids.toString());
+ }
+ }
+
+ for (String guid : guidsToFetch) {
+ try {
+ ITypedReferenceableInstance entity = graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertices.get(guid));
+ for(int index : uncachedGuids.get(guid)) {
+ result[index] = entity;
+ }
+ } catch (AtlasException e) {
+ throw new RepositoryException(e);
+ }
}
+ return Arrays.asList(result);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index c6cd93c..5127f74 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -705,13 +705,8 @@ public class DefaultMetadataService implements MetadataService, ActiveStateChang
}
}
- private List<ITypedReferenceableInstance> loadEntities(List<String> guids) throws EntityNotFoundException,
- RepositoryException {
- List<ITypedReferenceableInstance> entities = new ArrayList<>();
- for (String guid : guids) {
- entities.add(repository.getEntityDefinition(guid));
- }
- return entities;
+ private List<ITypedReferenceableInstance> loadEntities(List<String> guids) throws RepositoryException, EntityNotFoundException {
+ return repository.getEntityDefinitions(guids.toArray(new String[guids.size()]));
}
private void onTypesUpdated(Map<String, IDataType> typesUpdated) throws AtlasException {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/1e105174/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
index 43b08c4..174b8cb 100644
--- a/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/graph/GraphBackedRepositoryHardDeleteTest.java
@@ -80,7 +80,7 @@ public class GraphBackedRepositoryHardDeleteTest extends GraphBackedMetadataRepo
repositoryService.getEntityDefinition(id);
fail("Expected EntityNotFoundException");
} catch(EntityNotFoundException e) {
- //expected
+ // expected
}
}