You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/06/10 15:17:42 UTC
[2/3] atlas git commit: ATLAS-2748, ATLAS-2849,
ATLAS-2750: Edges' state preserved during import. Classification
dissociation fails on imported entities with classifications. Processing
classifications with attributes containing maps and arrays.
ATLAS-2748, ATLAS-2849, ATLAS-2750: Edges' state preserved during import. Classification dissociation fails on imported entities with classifications. Processing classifications with attributes containing maps and arrays.
(cherry picked from commit 7174257bf131f531a5f86ccb574ddb512af69623)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/6acbb026
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/6acbb026
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/6acbb026
Branch: refs/heads/branch-1.0
Commit: 6acbb0264926bcdf26b354fbd11132b0d6410a1d
Parents: 0e43879
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Fri Jun 8 10:35:27 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Sun Jun 10 08:05:58 2018 -0700
----------------------------------------------------------------------
.../janus/migration/ElementProcessors.java | 4 +-
.../migration/TypesWithCollectionsFinder.java | 1 +
.../janus/migration/GraphSONUtilityTest.java | 46 +++++++++-----------
.../TypesWithCollectionsFinderTest.java | 9 +++-
.../json/typesDef-classification-with-map.json | 30 +++++++++++++
5 files changed, 62 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
index 4017aaa..18082d7 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
@@ -34,6 +34,7 @@ import java.util.UUID;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
@@ -354,9 +355,8 @@ public class ElementProcessors {
private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) {
props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
-
props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label)));
- props.put(STATE_PROPERTY_KEY, "ACTIVE");
+ props.put(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY, false);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
index 55aa9c9..9b4499b 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
@@ -47,6 +47,7 @@ public class TypesWithCollectionsFinder {
addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret);
addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret);
+ addVertexPropertiesForCollectionAttributes(typeRegistry.getAllClassificationTypes(), ret);
displayInfo("types with properties: ", ret);
http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
index 0c9b620..3184236 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
@@ -38,6 +38,7 @@ import java.util.Set;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
@@ -229,7 +230,6 @@ public class GraphSONUtilityTest extends BaseUtils {
@Test
public void tagAssociated_NewAttributesAdded() {
-
ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap(), new HashMap<>());
TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(elementProcessors);
@@ -238,22 +238,8 @@ public class GraphSONUtilityTest extends BaseUtils {
addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
Iterator<Vertex> vertices = tg.vertices();
- while(vertices.hasNext()) {
- Vertex v = vertices.next();
- if(v.id().toString() != "16752") continue;
-
- assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
- assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
- assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
- }
-
- Iterator<Edge> edges = tg.edges();
- while(edges.hasNext()) {
- Edge e = edges.next();
- assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
- assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
- assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
- }
+ assertVertex(vertices, "16752");
+ assertEdge(tg);
}
@Test
@@ -266,21 +252,31 @@ public class GraphSONUtilityTest extends BaseUtils {
addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
Iterator<Vertex> vertices = tg.vertices();
- while(vertices.hasNext()) {
- Vertex v = vertices.next();
- if(v.id().toString() != "16752") continue;
-
- assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
- assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
- assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
- }
+ assertVertex(vertices, "16752");
+ assertEdge(tg);
+ }
+ private void assertEdge(TinkerGraph tg) {
Iterator<Edge> edges = tg.edges();
while(edges.hasNext()) {
Edge e = edges.next();
assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertTrue(e.property(Constants.CLASSIFICATION_EDGE_NAME_PROPERTY_KEY).isPresent());
assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ assertTrue(e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).isPresent());
+ assertFalse((boolean) e.property(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY).value());
+ }
+ }
+
+ private void assertVertex(Iterator<Vertex> vertices, String vertexId) {
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.id().toString() != vertexId) continue;
+
+ assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+ assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+ assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
index b4d76db..dad5e4a 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
@@ -20,10 +20,13 @@ package org.apache.atlas.repository.migration;
import com.google.inject.Inject;
import org.apache.atlas.TestModules;
+import org.apache.atlas.TestUtilsV2;
import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDBMigrator;
import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder;
+import org.apache.atlas.utils.TestResourceFileUtils;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
@@ -45,13 +48,15 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
@BeforeClass
public void setup() throws IOException, AtlasBaseException {
loadTypesFromJson();
+
+ typeDefStore.createTypesDef(TestResourceFileUtils.readObjectFromJson(".", "typesDef-classification-with-map", AtlasTypesDef.class));
}
@Test
public void fetchAll() {
Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry);
- assertEquals(typeAttrMap.size(), 9);
+ assertEquals(typeAttrMap.size(), 10);
assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches");
@@ -75,6 +80,8 @@ public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys");
assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns");
assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters");
+
+ assertProperties(typeAttrMap, "tag_with_map_of_map", "MAP_PRIMITIVE", "tag_with_map_of_map.tag_with_map_of_map");
}
private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/6acbb026/repository/src/test/resources/json/typesDef-classification-with-map.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/json/typesDef-classification-with-map.json b/repository/src/test/resources/json/typesDef-classification-with-map.json
new file mode 100644
index 0000000..cd67df4
--- /dev/null
+++ b/repository/src/test/resources/json/typesDef-classification-with-map.json
@@ -0,0 +1,30 @@
+{
+ "classificationDefs": [
+ {
+ "category": "CLASSIFICATION",
+ "guid": "d154cc2e-cd67-4620-9893-0cd6aec54244",
+ "createdBy": "admin",
+ "updatedBy": "admin",
+ "createTime": 1528460320484,
+ "updateTime": 1528460320484,
+ "version": 1,
+ "name": "tag_with_map_of_map",
+ "description": "tag_with_map_of_map",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "tag_with_map_of_map",
+ "typeName": "map<string,map<string,string>>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [],
+ "subTypes": []
+ }
+ ]
+}