You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/09/07 07:11:00 UTC
atlas git commit: ATLAS-2827: fix to handle failure in saving
indexable string property of large size - #3
Repository: atlas
Updated Branches:
refs/heads/master 4c6f1d167 -> eae976187
ATLAS-2827: fix to handle failure in saving indexable string property of large size - #3
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/eae97618
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/eae97618
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/eae97618
Branch: refs/heads/master
Commit: eae9761871af9aeecc0fdc2248b7c8555407d0a6
Parents: 4c6f1d1
Author: Madhan Neethiraj <ma...@apache.org>
Authored: Thu Sep 6 19:23:30 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Thu Sep 6 23:44:49 2018 -0700
----------------------------------------------------------------------
.../store/graph/v2/AtlasEntityStoreV2.java | 5 +++
.../store/graph/v2/EntityGraphMapper.java | 4 ++-
.../store/graph/v2/InstanceGraphMapper.java | 32 -----------------
.../java/org/apache/atlas/RequestContext.java | 36 ++++++++++++++++++++
.../notification/NotificationHookConsumer.java | 2 ++
5 files changed, 46 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/eae97618/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index 6e2a03f..bddbf71 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -715,6 +715,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
EntityGraphDiscovery graphDiscoverer = new AtlasEntityGraphDiscoveryV2(typeRegistry, entityStream);
EntityGraphDiscoveryContext discoveryContext = graphDiscoverer.discoverEntities();
EntityMutationContext context = new EntityMutationContext(discoveryContext);
+ RequestContext requestContext = RequestContext.get();
for (String guid : discoveryContext.getReferencedGuids()) {
AtlasVertex vertex = discoveryContext.getResolvedEntityVertex(guid);
@@ -734,6 +735,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
if (!StringUtils.equals(guidVertex, guid)) { // if entity was found by unique attribute
entity.setGuid(guidVertex);
+
+ requestContext.recordEntityGuidUpdate(entity, guid);
}
context.addUpdated(guid, entity, entityType, vertex);
@@ -756,6 +759,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
entity.setGuid(generatedGuid);
+ requestContext.recordEntityGuidUpdate(entity, guid);
+
context.addCreated(guid, entity, entityType, vertex);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/eae97618/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
index 4cd9909..d1e95f0 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
@@ -640,11 +640,13 @@ public class EntityGraphMapper {
}
if (trimmedLength < value.length()) {
- LOG.warn("Indexed-String-Attribute: {} length is {} characters, trimming to {}", ctx.getAttribute().getQualifiedName(), value.length(), trimmedLength);
+ LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}; trimming to {} - attempt #{}", ctx.getAttribute().getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN, trimmedLength, requestContext.getAttemptCount());
String checksumSuffix = ":" + DigestUtils.shaHex(value); // Storing SHA checksum in case verification is needed after retrieval
ret = value.substring(0, trimmedLength - checksumSuffix.length()) + checksumSuffix;
+ } else {
+ LOG.warn("Length of indexed attribute {} is {} characters, longer than safe-limit {}", ctx.getAttribute().getQualifiedName(), value.length(), INDEXED_STR_SAFE_LEN);
}
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/eae97618/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/InstanceGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/InstanceGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/InstanceGraphMapper.java
deleted file mode 100644
index 8d40500..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/InstanceGraphMapper.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.store.graph.v2;
-
-import org.apache.atlas.exception.AtlasBaseException;
-
-public interface InstanceGraphMapper<T> {
-
- /**
- * Map the given type instance to the graph
- *
- * @param ctx
- * @return the value that was mapped to the vertex
- * @throws AtlasBaseException
- */
- T toGraph(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException;
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/eae97618/server-api/src/main/java/org/apache/atlas/RequestContext.java
----------------------------------------------------------------------
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
index 891dcc8..e23eaec 100644
--- a/server-api/src/main/java/org/apache/atlas/RequestContext.java
+++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -19,6 +19,7 @@
package org.apache.atlas;
import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
import org.apache.atlas.model.instance.AtlasObjectId;
import org.apache.commons.lang.StringUtils;
@@ -38,6 +39,7 @@ public class RequestContext {
private final Map<String, List<AtlasClassification>> addedPropagations = new HashMap<>();
private final Map<String, List<AtlasClassification>> removedPropagations = new HashMap<>();
private final long requestTime = System.currentTimeMillis();
+ private List<EntityGuidPair> entityGuidInRequest = null;
private String user;
private Set<String> userGroups;
@@ -71,6 +73,10 @@ public class RequestContext {
instance.entityCacheV2.clear();
instance.addedPropagations.clear();
instance.removedPropagations.clear();
+
+ if (instance.entityGuidInRequest != null) {
+ instance.entityGuidInRequest.clear();
+ }
}
CURRENT_CONTEXT.remove();
@@ -202,4 +208,34 @@ public class RequestContext {
public boolean isDeletedEntity(String guid) {
return deletedEntities.containsKey(guid);
}
+
+ public void recordEntityGuidUpdate(AtlasEntity entity, String guidInRequest) {
+ if (entityGuidInRequest == null) {
+ entityGuidInRequest = new ArrayList<>();
+ }
+
+ entityGuidInRequest.add(new EntityGuidPair(entity, guidInRequest));
+ }
+
+ public void resetEntityGuidUpdates() {
+ if (entityGuidInRequest != null) {
+ for (EntityGuidPair entityGuidPair : entityGuidInRequest) {
+ entityGuidPair.resetEntityGuid();
+ }
+ }
+ }
+
+ public class EntityGuidPair {
+ private final AtlasEntity entity;
+ private final String guid;
+
+ public EntityGuidPair(AtlasEntity entity, String guid) {
+ this.entity = entity;
+ this.guid = guid;
+ }
+
+ public void resetEntityGuid() {
+ entity.setGuid(guid);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/eae97618/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
index 0ac3ebb..d680e4e 100644
--- a/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
+++ b/webapp/src/main/java/org/apache/atlas/notification/NotificationHookConsumer.java
@@ -521,6 +521,8 @@ public class NotificationHookConsumer implements Service, ActiveStateChangeHandl
break;
} catch (Throwable e) {
+ RequestContext.get().resetEntityGuidUpdates();
+
LOG.warn("Error handling message", e);
try {
LOG.info("Sleeping for {} ms before retry", consumerRetryInterval);