You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/07/01 22:32:16 UTC
[1/2] atlas git commit: ATLAS-2770: entity-delete fails when Atlas is
configured for hard-delete
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 51dcce487 -> a7a9c0ef9
ATLAS-2770: entity-delete fails when Atlas is configured for hard-delete
(cherry picked from commit eb22be8c36769d88c7ec342bf7f6195849dc13b2)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/db152cf4
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/db152cf4
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/db152cf4
Branch: refs/heads/branch-1.0
Commit: db152cf40c1c9d5f3114aef999490e6d4c85f8bd
Parents: 51dcce4
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Sun Jul 1 09:49:22 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Jul 1 15:31:42 2018 -0700
----------------------------------------------------------------------
.../graph/v2/AtlasEntityChangeNotifier.java | 33 ++++++++++++++------
1 file changed, 24 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/db152cf4/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
index 7ed99a4..deb79e5 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityChangeNotifier.java
@@ -302,7 +302,7 @@ public class AtlasEntityChangeNotifier {
}
private void notifyV2Listeners(List<AtlasEntityHeader> entityHeaders, EntityOperation operation, boolean isImport) throws AtlasBaseException {
- List<AtlasEntity> entities = toAtlasEntities(entityHeaders);
+ List<AtlasEntity> entities = toAtlasEntities(entityHeaders, operation);
for (EntityChangeListenerV2 listener : entityChangeListenersV2) {
switch (operation) {
@@ -377,27 +377,42 @@ public class AtlasEntityChangeNotifier {
return ret;
}
- private List<AtlasEntity> toAtlasEntities(List<AtlasEntityHeader> entityHeaders) throws AtlasBaseException {
+ private List<AtlasEntity> toAtlasEntities(List<AtlasEntityHeader> entityHeaders, EntityOperation operation) throws AtlasBaseException {
List<AtlasEntity> ret = new ArrayList<>();
if (CollectionUtils.isNotEmpty(entityHeaders)) {
for (AtlasEntityHeader entityHeader : entityHeaders) {
- String entityGuid = entityHeader.getGuid();
- String typeName = entityHeader.getTypeName();
+ String entityGuid = entityHeader.getGuid();
+ String typeName = entityHeader.getTypeName();
+ AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName);
+
+ if (entityType == null) {
+ continue;
+ }
// Skip all internal types as the HARD DELETE will cause lookup errors
- AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName(typeName);
- if (Objects.nonNull(entityType) && entityType.isInternalType()) {
+ if (entityType.isInternalType()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skipping internal type = {}", typeName);
}
continue;
}
- AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
+ final AtlasEntity entity;
+
+ // delete notifications don't need all attributes. Hence the special handling for delete operation
+ if (operation == EntityOperation.DELETE) {
+ entity = new AtlasEntity(typeName, entityHeader.getAttributes());
+
+ entity.setGuid(entityGuid);
+ } else {
+ AtlasEntityWithExtInfo entityWithExtInfo = instanceConverter.getAndCacheEntity(entityGuid);
+
+ entity = (entityWithExtInfo != null) ? entityWithExtInfo.getEntity() : null;
+ }
- if (entityWithExtInfo != null) {
- ret.add(entityWithExtInfo.getEntity());
+ if (entity != null) {
+ ret.add(entity);
}
}
}
[2/2] atlas git commit: ATLAS-2769: Atlas start just after the
upgrade fails with 'TableNotFoundException: atlas_janus' exception
Posted by ma...@apache.org.
ATLAS-2769: Atlas start just after the upgrade fails with 'TableNotFoundException: atlas_janus' exception
(cherry picked from commit 78cfd718444e81426334ef2e3fdae9b467e60cd1)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/a7a9c0ef
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/a7a9c0ef
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/a7a9c0ef
Branch: refs/heads/branch-1.0
Commit: a7a9c0ef9c69dd827e34cb2ff4e82e76a21c1a59
Parents: db152cf
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Jun 28 14:43:11 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Jul 1 15:32:09 2018 -0700
----------------------------------------------------------------------
.../repository/graph/AtlasGraphProvider.java | 70 +++++++++++++++++++-
1 file changed, 67 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/a7a9c0ef/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
index 55934c3..211d2ac 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java
@@ -19,10 +19,14 @@
package org.apache.atlas.repository.graph;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@@ -33,7 +37,15 @@ import org.springframework.context.annotation.Configuration;
@Configuration
public class AtlasGraphProvider implements IAtlasGraphProvider {
- private static volatile GraphDatabase<?,?> graphDb_;
+ private static volatile GraphDatabase<?,?> graphDb_;
+
+ private static final Logger LOG = LoggerFactory.getLogger(AtlasGraphProvider.class);
+ private static final Integer MAX_RETRY_COUNT = getMaxRetryCount();
+ private static final Long RETRY_SLEEP_TIME_MS = getRetrySleepTime();
+ private static final String GRAPH_REPOSITORY_MAX_RETRIES = "atlas.graph.repository.max.retries";
+ private static final String GRAPH_REPOSITORY_RETRY_SLEEPTIME = "atlas.graph.repository.retry.sleeptime.ms";
+
+ private static org.apache.commons.configuration.Configuration APPLICATION_PROPERTIES = null;
public static <V, E> AtlasGraph<V, E> getGraphInstance() {
GraphDatabase<?,?> db = getGraphDatabase();
@@ -67,7 +79,59 @@ public class AtlasGraphProvider implements IAtlasGraphProvider {
@Override
@Bean(destroyMethod = "")
- public AtlasGraph get() throws RepositoryException {
- return getGraphInstance();
+ public AtlasGraph get() throws RepositoryException{
+ try {
+ return getGraphInstance();
+ } catch (Exception ex) {
+ LOG.info("Failed to obtain graph instance, retrying " + MAX_RETRY_COUNT + " times, error: " + ex);
+
+ return retry();
+ }
+ }
+
+ private AtlasGraph retry() throws RepositoryException {
+ int retryCounter = 0;
+
+ while (retryCounter < MAX_RETRY_COUNT) {
+ try {
+ // Retry after 30 sec to get graph instance
+ Thread.sleep(RETRY_SLEEP_TIME_MS);
+
+ return getGraphInstance();
+ } catch (Exception ex) {
+ retryCounter++;
+
+ LOG.info("Failed to obtain graph instance on retry " + retryCounter + " of " + MAX_RETRY_COUNT + " error: " + ex);
+
+ if (retryCounter >= MAX_RETRY_COUNT) {
+ LOG.info("Max retries exceeded.");
+ break;
+ }
+ }
+ }
+
+ throw new RepositoryException("Max retries exceeded. Failed to obtain graph instance after " + MAX_RETRY_COUNT + " retries");
+ }
+
+ private static Integer getMaxRetryCount() {
+ initApplicationProperties();
+
+ return (APPLICATION_PROPERTIES == null) ? 3 : APPLICATION_PROPERTIES.getInt(GRAPH_REPOSITORY_MAX_RETRIES, 3);
+ }
+
+ private static Long getRetrySleepTime() {
+ initApplicationProperties();
+
+ return (APPLICATION_PROPERTIES == null) ? 30000 : APPLICATION_PROPERTIES.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000);
+ }
+
+ private static void initApplicationProperties() {
+ if (APPLICATION_PROPERTIES == null) {
+ try {
+ APPLICATION_PROPERTIES = ApplicationProperties.get();
+ } catch (AtlasException ex) {
+ // ignore
+ }
+ }
}
}