You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2021/04/26 18:36:38 UTC
[atlas] branch branch-2.0 updated: ATLAS-4195: Entity comparator
doesn't account for inactive relations
This is an automated email from the ASF dual-hosted git repository.
sarath pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/atlas.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new c108ed4 ATLAS-4195: Entity comparator doesn't account for inactive relations
c108ed4 is described below
commit c108ed49dab88d7270158fc763239c128d9b6794
Author: Deep Singh <de...@gmail.com>
AuthorDate: Mon Apr 26 11:26:28 2021 -0500
ATLAS-4195: Entity comparator doesn't account for inactive relations
Signed-off-by: Sarath Subramanian <sa...@apache.org>
(cherry picked from commit 683e2a91db50850014aa92a2abfa668d14624f1a)
---
.../store/graph/v2/AtlasEntityStoreV2.java | 8 +-
.../store/graph/v2/EntityGraphRetriever.java | 19 +++-
.../store/graph/v2/DifferentialAuditsTest.java | 90 ++++++++++++++++
.../src/test/resources/json/entities/db01.json | 23 ++++
.../test/resources/json/entities/tbl01-2cols.json | 116 +++++++++++++++++++++
.../src/test/resources/json/entities/tbl01.json | 94 +++++++++++++++++
6 files changed, 346 insertions(+), 4 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
index c133920..65b92ab 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java
@@ -18,6 +18,7 @@
package org.apache.atlas.repository.store.graph.v2;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.AtlasErrorCode;
import org.apache.atlas.DeleteType;
import org.apache.atlas.GraphTransactionInterceptor;
@@ -109,7 +110,7 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
private final IAtlasEntityChangeNotifier entityChangeNotifier;
private final EntityGraphMapper entityGraphMapper;
private final EntityGraphRetriever entityRetriever;
- private final boolean storeDifferentialAudits;
+ private boolean storeDifferentialAudits;
@Inject
public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, AtlasTypeRegistry typeRegistry,
@@ -123,6 +124,11 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore {
this.storeDifferentialAudits = STORE_DIFFERENTIAL_AUDITS.getBoolean();
}
+ @VisibleForTesting
+ public void setStoreDifferentialAudits(boolean val) {
+ this.storeDifferentialAudits = val;
+ }
+
@Override
@GraphTransaction
public List<String> getEntityGUIDS(final String typename) throws AtlasBaseException {
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
index b790023..b6f1ef7 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java
@@ -271,7 +271,7 @@ public class EntityGraphRetriever {
Object ret = null;
try {
- ret = getVertexAttribute(entityVertex, attribute);
+ ret = getVertexAttributeIgnoreInactive(entityVertex, attribute);
} catch (AtlasBaseException excp) {
// ignore
}
@@ -966,6 +966,10 @@ public class EntityGraphRetriever {
}
private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
+ return mapVertexToAttribute(entityVertex, attribute, entityExtInfo, isMinExtInfo, includeReferences, false);
+ }
+
+ private Object mapVertexToAttribute(AtlasVertex entityVertex, AtlasAttribute attribute, AtlasEntityExtInfo entityExtInfo, final boolean isMinExtInfo, boolean includeReferences, boolean ignoreInactive) throws AtlasBaseException {
Object ret = null;
AtlasType attrType = attribute.getAttributeType();
String edgeLabel = attribute.getRelationshipEdgeLabel();
@@ -1011,7 +1015,7 @@ public class EntityGraphRetriever {
if (attribute.getAttributeDef().isSoftReferenced()) {
ret = mapVertexToArrayForSoftRef(entityVertex, attribute, entityExtInfo, isMinExtInfo);
} else {
- ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences);
+ ret = mapVertexToArray(entityVertex, entityExtInfo, isOwnedAttribute, attribute, isMinExtInfo, includeReferences, ignoreInactive);
}
}
}
@@ -1158,7 +1162,8 @@ public class EntityGraphRetriever {
}
private List<Object> mapVertexToArray(AtlasVertex entityVertex, AtlasEntityExtInfo entityExtInfo,
- boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo, boolean includeReferences) throws AtlasBaseException {
+ boolean isOwnedAttribute, AtlasAttribute attribute, final boolean isMinExtInfo,
+ boolean includeReferences, boolean ignoreInactive) throws AtlasBaseException {
AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType();
AtlasType arrayElementType = arrayType.getElementType();
@@ -1184,6 +1189,10 @@ public class EntityGraphRetriever {
continue;
}
+ if (ignoreInactive && GraphHelper.getStatus((AtlasEdge) element) != AtlasEntity.Status.ACTIVE) {
+ continue;
+ }
+
Object arrValue = mapVertexToCollectionEntry(entityVertex, arrayElementType, element, edgeLabel,
entityExtInfo, isOwnedAttribute, edgeDirection, isMinExtInfo, includeReferences);
@@ -1333,6 +1342,10 @@ public class EntityGraphRetriever {
return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null;
}
+ private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException {
+ return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false, true, true) : null;
+ }
+
private void mapRelationshipAttributes(AtlasVertex entityVertex, AtlasEntity entity, AtlasEntityExtInfo entityExtInfo, boolean isMinExtInfo) throws AtlasBaseException {
AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName());
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/DifferentialAuditsTest.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/DifferentialAuditsTest.java
new file mode 100644
index 0000000..d10774f
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v2/DifferentialAuditsTest.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.store.graph.v2;
+
+import org.apache.atlas.ApplicationProperties;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.BasicTestSetup;
+import org.apache.atlas.RequestContext;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntityHeader;
+import org.apache.atlas.model.instance.EntityMutationResponse;
+import org.apache.atlas.query.DSLQueriesTest;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.utils.TestResourceFileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import static org.apache.atlas.AtlasConfiguration.STORE_DIFFERENTIAL_AUDITS;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class DifferentialAuditsTest extends BasicTestSetup {
+ private static final Logger LOG = LoggerFactory.getLogger(DifferentialAuditsTest.class);
+
+
+ @BeforeClass
+ public void setup() throws Exception {
+ super.initialize();
+
+ setupTestData();
+ }
+
+ @Test
+ public void t1() throws IOException, AtlasException, AtlasBaseException {
+ String db01 = TestResourceFileUtils.getJson("entities", "db01");
+ String tbl01 = TestResourceFileUtils.getJson("entities", "tbl01");
+ String tbl012Col = TestResourceFileUtils.getJson("entities", "tbl01-2cols");
+
+ AtlasEntity.AtlasEntitiesWithExtInfo db = AtlasType.fromJson(db01, AtlasEntity.AtlasEntitiesWithExtInfo.class);
+ AtlasEntity.AtlasEntitiesWithExtInfo tbl = AtlasType.fromJson(tbl01, AtlasEntity.AtlasEntitiesWithExtInfo.class);
+ AtlasEntity.AtlasEntitiesWithExtInfo tbl2Cols = AtlasType.fromJson(tbl012Col, AtlasEntity.AtlasEntitiesWithExtInfo.class);
+
+ ((AtlasEntityStoreV2) entityStore).setStoreDifferentialAudits(true);
+ EntityMutationResponse response = entityStore.createOrUpdate(new AtlasEntityStream(db), false);
+
+ assertNotNull(response);
+
+ response = entityStore.createOrUpdate(new AtlasEntityStream(tbl2Cols), false);
+ Collection<AtlasEntity> diffEntities = RequestContext.get().getDifferentialEntities();
+ assertNotNull(response);
+ assertEquals(diffEntities.size(), 0);
+
+ RequestContext.get().clearCache();
+ response = entityStore.createOrUpdate(new AtlasEntityStream(tbl), false);
+ assertNotNull(response);
+ diffEntities = RequestContext.get().getDifferentialEntities();
+ assertEquals(diffEntities.size(), 1);
+
+ RequestContext.get().clearCache();
+ response = entityStore.createOrUpdate(new AtlasEntityStream(tbl), false);
+ assertNotNull(response);
+ diffEntities = RequestContext.get().getDifferentialEntities();
+ assertEquals(diffEntities.size(), 0);
+ }
+}
+
diff --git a/repository/src/test/resources/json/entities/db01.json b/repository/src/test/resources/json/entities/db01.json
new file mode 100644
index 0000000..470d180
--- /dev/null
+++ b/repository/src/test/resources/json/entities/db01.json
@@ -0,0 +1,23 @@
+{
+ "referredEntities": {
+ },
+ "entities": [
+ {
+ "typeName": "hive_db",
+ "attributes": {
+ "owner": "hive",
+ "ownerType": "USER",
+ "qualifiedName": "db01x@cm",
+ "clusterName": "cm",
+ "name": "db01x",
+ "parameters": {}
+ },
+ "guid": "-1",
+ "status": "ACTIVE",
+ "createdBy": "hive",
+ "updatedBy": "hive",
+ "createTime": 1619036192377,
+ "updateTime": 1619036195986
+ }
+ ]
+}
diff --git a/repository/src/test/resources/json/entities/tbl01-2cols.json b/repository/src/test/resources/json/entities/tbl01-2cols.json
new file mode 100644
index 0000000..52f34cd
--- /dev/null
+++ b/repository/src/test/resources/json/entities/tbl01-2cols.json
@@ -0,0 +1,116 @@
+{
+ "referredEntities": {
+ "-3": {
+ "typeName": "hive_column",
+ "attributes": {
+ "owner": "hive",
+ "qualifiedName": "db01x.tbl01.col01@cm",
+ "name": "col01",
+ "position": 0,
+ "type": "string",
+ "table": {
+ "guid": "-1",
+ "typeName": "hive_table"
+ }
+ },
+ "guid": "-3",
+ "status": "ACTIVE",
+ "createTime": 1619036195986,
+ "updateTime": 1619036195986
+ },
+ "-4": {
+ "typeName": "hive_column",
+ "attributes": {
+ "owner": "hive",
+ "qualifiedName": "db01x.tbl01.col02@cm",
+ "name": "col02",
+ "position": 1,
+ "type": "string",
+ "table": {
+ "guid": "-1",
+ "typeName": "hive_table"
+ }
+ },
+ "guid": "-4",
+ "status": "ACTIVE",
+ "createTime": 1619036195986,
+ "updateTime": 1619036195986
+ },
+ "-2": {
+ "typeName": "hive_storagedesc",
+ "attributes": {
+ "qualifiedName": "db01x.tbl01@cm_storage",
+ "inputFormat": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "location": "hdfs://localhost:8020/warehouse/tablespace/managed/hive/db01x.db/tbl01",
+ "outputFormat": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ "compressed": false,
+ "serdeInfo": {
+ "typeName": "hive_serde",
+ "attributes": {
+ "serializationLib": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+ "parameters": {
+ "serialization.format": "1"
+ }
+ }
+ },
+ "table": {
+ "guid": "-1",
+ "typeName": "hive_table"
+ },
+ "numBuckets": -1
+ },
+ "guid": "-2",
+ "status": "ACTIVE",
+ "createdBy": "hive",
+ "updatedBy": "hive",
+ "createTime": 1619036195986,
+ "updateTime": 1619036195986,
+ "version": 0
+ }
+ },
+ "entities": [
+ {
+ "typeName": "hive_table",
+ "attributes": {
+ "owner": "hive",
+ "lastAccessTime": 1619036193000,
+ "qualifiedName": "db01x.tbl01@cm",
+ "displayName": null,
+ "columns": [
+ {
+ "guid": "-3",
+ "typeName": "hive_column"
+ },
+ {
+ "guid": "-4",
+ "typeName": "hive_column"
+ }
+ ],
+ "db": {
+ "typeName": "hive_db",
+ "uniqueAttributes": {
+ "qualifiedName": "db01x@cm"
+ }
+ },
+ "description": null,
+ "viewExpandedText": null,
+ "tableType": "MANAGED_TABLE",
+ "sd": {
+ "guid": "-2",
+ "typeName": "hive_storagedesc"
+ },
+ "rewriteEnable": null,
+ "createTime": 1619036193000,
+ "name": "tbl01"
+ },
+ "guid": "-1",
+ "isIncomplete": false,
+ "status": "ACTIVE",
+ "createdBy": "hive",
+ "updatedBy": "hive",
+ "createTime": 1619036195986,
+ "updateTime": 1619036197378,
+ "version": 0
+ }
+ ]
+}
diff --git a/repository/src/test/resources/json/entities/tbl01.json b/repository/src/test/resources/json/entities/tbl01.json
new file mode 100644
index 0000000..50251f0
--- /dev/null
+++ b/repository/src/test/resources/json/entities/tbl01.json
@@ -0,0 +1,94 @@
+{
+ "referredEntities": {
+ "-3": {
+ "typeName": "hive_column",
+ "attributes": {
+ "owner": "hive",
+ "qualifiedName": "db01x.tbl01.col01@cm",
+ "name": "col01",
+ "position": 0,
+ "type": "string",
+ "table": {
+ "guid": "-1",
+ "typeName": "hive_table"
+ }
+ },
+ "guid": "-3",
+ "status": "ACTIVE",
+ "createTime": 1619036195986,
+ "updateTime": 1619036195986
+ },
+ "-2": {
+ "typeName": "hive_storagedesc",
+ "attributes": {
+ "qualifiedName": "db01x.tbl01@cm_storage",
+ "inputFormat": "org.apache.hadoop.hive.ql.io.orc.OrcInputFormat",
+ "location": "hdfs://localhost:8020/warehouse/tablespace/managed/hive/db01x.db/tbl01",
+ "outputFormat": "org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat",
+ "compressed": false,
+ "serdeInfo": {
+ "typeName": "hive_serde",
+ "attributes": {
+ "serializationLib": "org.apache.hadoop.hive.ql.io.orc.OrcSerde",
+ "parameters": {
+ "serialization.format": "1"
+ }
+ }
+ },
+ "table": {
+ "guid": "-1",
+ "typeName": "hive_table"
+ },
+ "numBuckets": -1
+ },
+ "guid": "-2",
+ "status": "ACTIVE",
+ "createdBy": "hive",
+ "updatedBy": "hive",
+ "createTime": 1619036195986,
+ "updateTime": 1619036195986,
+ "version": 0
+ }
+ },
+ "entities": [
+ {
+ "typeName": "hive_table",
+ "attributes": {
+ "owner": "hive",
+ "lastAccessTime": 1619036193000,
+ "qualifiedName": "db01x.tbl01@cm",
+ "displayName": null,
+ "columns": [
+ {
+ "guid": "-3",
+ "typeName": "hive_column"
+ }
+ ],
+ "db": {
+ "typeName": "hive_db",
+ "uniqueAttributes": {
+ "qualifiedName": "db01x@cm"
+ }
+ },
+ "description": null,
+ "viewExpandedText": null,
+ "tableType": "MANAGED_TABLE",
+ "sd": {
+ "guid": "-2",
+ "typeName": "hive_storagedesc"
+ },
+ "rewriteEnable": null,
+ "createTime": 1619036193000,
+ "name": "tbl01"
+ },
+ "guid": "-1",
+ "isIncomplete": false,
+ "status": "ACTIVE",
+ "createdBy": "hive",
+ "updatedBy": "hive",
+ "createTime": 1619036195986,
+ "updateTime": 1619036197378,
+ "version": 0
+ }
+ ]
+}