You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/05/23 06:12:17 UTC
[6/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
index e863d9f..aee1b69 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
@@ -26,14 +26,16 @@ import org.apache.tinkerpop.gremlin.structure.io.graphson.TypeInfo;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
-import org.testng.ITestContext;
+import org.testng.SkipException;
import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.testng.AssertJUnit.assertTrue;
@@ -41,12 +43,16 @@ public class BaseUtils {
private static final String resourcesDirRelativePath = "/src/test/resources/";
private String resourceDir;
- protected final RelationshipTypeCache emptyRelationshipCache = new RelationshipTypeCache(new HashMap<>());
+ protected final ElementProcessors emptyRelationshipCache = new ElementProcessors(new HashMap<>(), new HashMap<>());
protected GraphSONUtility graphSONUtility;
- protected Object[][] getJsonNodeFromFile(String s) throws IOException {
+ protected JsonNode getJsonNodeFromFile(String s) {
File f = new File(getFilePath(s));
- return new Object[][]{{getEntityNode(FileUtils.readFileToString(f))}};
+ try {
+ return getEntityNode(FileUtils.readFileToString(f));
+ } catch (IOException e) {
+ throw new SkipException("getJsonNodeFromFile: " + s, e);
+ }
}
protected String getFilePath(String fileName) {
@@ -76,12 +82,18 @@ public class BaseUtils {
utility.vertexFromJson(tg, node);
}
- protected void addEdge(TinkerGraph tg, MappedElementCache cache) throws IOException {
+ protected void addEdge(TinkerGraph tg, MappedElementCache cache) {
GraphSONUtility gu = graphSONUtility;
- gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
- gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
- gu.edgeFromJson(tg, cache, (JsonNode) getEdge(null)[0][0]);
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, cache, getEdge(), getEdgeCol(), getEdgeCol2());
+ }
+
+ protected void addEdgesForMap(TinkerGraph tg, MappedElementCache cache) {
+ GraphSONUtility gu = graphSONUtility;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, cache, getEdgeCol3(), getEdgeCol4());
}
protected Vertex fetchTableVertex(TinkerGraph tg) {
@@ -91,29 +103,84 @@ public class BaseUtils {
return (Vertex) query.next();
}
- @DataProvider(name = "col1")
- public Object[][] getCol1(ITestContext context) throws IOException {
+ protected Map<String, Map<String, List<String>>> getTypePropertyMap(String type, String property, String category) {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(type, new HashMap<>());
+ map.get(type).put(category, new ArrayList<>());
+ map.get(type).get(category).add(property);
+ return map;
+ }
+
+ protected void addVertexToGraph(TinkerGraph tg, GraphSONUtility gu, JsonNode... nodes) {
+ for(JsonNode n : nodes) {
+ gu.vertexFromJson(tg, n);
+ }
+ }
+
+ protected void addEdgeToGraph(TinkerGraph tg, GraphSONUtility gu, MappedElementCache cache, JsonNode... nodes) {
+
+ for(JsonNode n : nodes) {
+ gu.edgeFromJson(tg, cache, n);
+ }
+ }
+
+ public JsonNode getCol1() {
return getJsonNodeFromFile("col-legacy.json");
}
- @DataProvider(name = "dbType")
- public Object[][] getDbType(ITestContext context) throws IOException {
+ public JsonNode getCol2() {
+ return getJsonNodeFromFile("col-2-legacy.json");
+ }
+
+ public JsonNode getCol3() {
+ return getJsonNodeFromFile("col-3-legacy.json");
+ }
+
+ public JsonNode getDbType() {
return getJsonNodeFromFile("db-type-legacy.json");
}
- @DataProvider(name = "edge")
- public Object[][] getEdge(ITestContext context) throws IOException {
+ public JsonNode getEdge() {
return getJsonNodeFromFile("edge-legacy.json");
}
- @DataProvider(name = "dbV")
- public Object[][] getDBV(ITestContext context) throws IOException {
- return getJsonNodeFromFile("db-v-65544.json");
+ public JsonNode getEdgeCol() {
+ return getJsonNodeFromFile("edge-legacy-col.json");
}
+ public JsonNode getEdgeCol2() {
+ return getJsonNodeFromFile("edge-legacy-col2.json");
+ }
+
+ public JsonNode getEdgeCol3() {
+ return getJsonNodeFromFile("edge-legacy-col3.json");
+ }
+
+ public JsonNode getEdgeCol4() {
+ return getJsonNodeFromFile("edge-legacy-col4.json");
+ }
+
+ public JsonNode getEdgeTag() {
+ return getJsonNodeFromFile("edge-legacy-tag.json");
+ }
+
+ public JsonNode getDBV() {
+ return getJsonNodeFromFile("db-v-65544.json");
+ }
- @DataProvider(name = "tableV")
- public Object[][] getTableV(ITestContext context) throws IOException {
+ public JsonNode getTableV() {
return getJsonNodeFromFile("table-v-147504.json");
}
+
+ public JsonNode getTagV() {
+ return getJsonNodeFromFile("tag-163856752.json");
+ }
+
+ public JsonNode getProcessV() {
+ return getJsonNodeFromFile("lineage-v-98312.json");
+ }
+
+ public JsonNode getEdgeProcess() {
+ return getJsonNodeFromFile("edge-legacy-process.json");
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
deleted file mode 100644
index 4d73c78..0000000
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graphdb.janus.migration;
-
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.eclipse.jetty.util.BlockingArrayQueue;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class GraphSONUtilityPostProcessTest extends BaseUtils {
- final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
- final String edgeId1 = "816u-35tc-ao0l-47so";
- final String edgeId2 = "82rq-35tc-ao0l-2glc";
-
- final String edgeId1x = "816u-35tc-ao0l-xxxx";
- final String edgeId2x = "82rq-35tc-ao0l-xxxx";
-
- private TinkerGraph tg;
- private MappedElementCache cache = new MappedElementCache();
- private Vertex tableV;
-
- @Test
- public void noRefNoUpdate() throws IOException {
- tg = TinkerGraph.open();
- graphSONUtility = new GraphSONUtility(emptyRelationshipCache);
-
- addEdge(tg, cache);
-
- tableV = fetchTableVertex(tg);
- assertNotNull(tableV);
-
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
-
- graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
- }
-
- @Test(dependsOnMethods = "noRefNoUpdate")
- public void refFoundVertexUpdated() throws IOException {
-
- cache.lruEdgeCache.put(edgeId1, edgeId1x);
- cache.lruEdgeCache.put(edgeId2, edgeId2x);
-
- graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1x, edgeId2x, tableV);
- }
-
- @Test(dependsOnMethods = "refFoundVertexUpdated")
- public void updateUsingPostProcessConsumer() throws IOException {
- MappedElementCache cache = new MappedElementCache();
- BlockingQueue<Object> bc = new BlockingArrayQueue<>();
- PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, tg, graphSONUtility,
- new String[] {HIVE_COLUMNS_PROPERTY}, cache, 5);
-
- cache.lruEdgeCache.put(edgeId1x, edgeId1);
- cache.lruEdgeCache.put(edgeId2x, edgeId2);
- consumer.processItem(tableV.id());
-
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
- }
-
- private void assertListProperty(String HIVE_COLUMNS_PROPERTY, String edgeId1, String edgeId2, Vertex tableV) {
- assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
- List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
-
- assertEquals(list.size(), 2);
- assertEquals(list.get(0), edgeId1);
- assertEquals(list.get(1), edgeId2);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
index 794b547..049fd64 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
@@ -18,36 +18,58 @@
package org.apache.atlas.repository.graphdb.janus.migration;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
import org.testng.Assert;
-import org.testng.ITestContext;
-import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY;
import static org.testng.Assert.*;
public class GraphSONUtilityTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void idFetch(JsonNode node) {
+ private final String HIVE_TABLE_NAME_PROPERTY = "Asset.name";
+ private final String HIVE_TABLE_NAME = "test_table_view";
+ private final String HIVE_TABLE_COLUMNS_RELATIONSHIP = "hive_table.columns";
+ private final String HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP = "hive_table.columnsMap";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP = "hive_table.parameters";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY = "transient_lastDdlTime";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY = String.format("%s.%s", HIVE_TABLE_COLUMNS_PARAMETERS_MAP, HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY);
+ private final String HIVE_TABLE_TYPE = "hive_table";
+
+ @Test
+ public void idFetch() {
+ JsonNode node = getCol1();
+ final int EXPECTED_ID = 98336;
Object o = GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokensTP2._ID));
assertNotNull(o);
- assertEquals((int) o, 98336);
+ assertEquals((int) o, EXPECTED_ID);
}
- @Test(dataProvider = "col1")
- public void verifyReadProperties(JsonNode node) {
+ @Test
+ public void verifyReadProperties() {
+ JsonNode node = getCol1();
Map<String, Object> props = GraphSONUtility.readProperties(node);
assertEquals(props.get("__superTypeNames").getClass(), ArrayList.class);
@@ -58,8 +80,9 @@ public class GraphSONUtilityTest extends BaseUtils {
assertNotNull(props);
}
- @Test(dataProvider = "col1")
- public void dataNodeReadAndVertexAddedToGraph(JsonNode entityNode) throws IOException {
+ @Test
+ public void dataNodeReadAndVertexAddedToGraph() throws IOException {
+ JsonNode entityNode = getCol1();
TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
Map<String, Object> map = gu.vertexFromJson(tg, entityNode);
@@ -71,8 +94,9 @@ public class GraphSONUtilityTest extends BaseUtils {
assertTrue(v.property(VERTEX_ID_IN_IMPORT_KEY).isPresent());
}
- @Test(dataProvider = "dbType")
- public void typeNodeReadAndVertexNotAddedToGraph(JsonNode entityNode) throws IOException {
+ @Test
+ public void typeNodeReadAndVertexNotAddedToGraph() throws IOException {
+ JsonNode entityNode = getDbType();
TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
gu.vertexFromJson(tg, entityNode);
@@ -81,19 +105,46 @@ public class GraphSONUtilityTest extends BaseUtils {
}
@Test
- public void edgeReadAndAddedToGraph() throws IOException {
+ public void updateNonPrimitiveArrayProperty() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getNonPrimitiveArray());
+
TinkerGraph tg = TinkerGraph.open();
- GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
- Map<String, Object> m = null;
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ addVertexToGraph(tg, gu, getTableV());
- m = gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
- assertNull(m);
+ Vertex v = tg.traversal().V().next();
+ assertNotNull(v);
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
- m = gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
- assertNull(m);
+ Map<String, String> list = (Map<String, String>) v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).value();
+ assertEquals(list.size(), 2);
+ }
- m = gu.edgeFromJson(tg, new MappedElementCache(), (JsonNode) getEdge(null)[0][0]);
- assertNull(m);
+
+ @Test
+ public void updatePrimitiveMapProperty() {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMapPrimitive());
+
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ addVertexToGraph(tg, gu, getTableV());
+
+ Vertex v = tg.traversal().V().next();
+ assertNotNull(v);
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).isPresent());
+ assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).size(), 1);
+ assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).get(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY), "1522693834");
+ assertFalse(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY).isPresent());
+ }
+
+ @Test
+ public void edgeReadAndAddedToGraph() {
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
+ Map<String, Object> m = null;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdge());
Assert.assertEquals((long) tg.traversal().V().count().next(), 2L);
Assert.assertEquals((long) tg.traversal().E().count().next(), 1L);
@@ -101,4 +152,179 @@ public class GraphSONUtilityTest extends BaseUtils {
Edge e = tg.edges().next();
assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
}
+
+ @Test
+ public void edgeReadAndArrayIndexAdded() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ Map<String, Object> m = null;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol(), getEdgeCol2());
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ String arraySpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+ if(arraySpecificLabel.equals(HIVE_TABLE_COLUMNS_RELATIONSHIP)) {
+ assertTrue(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+ }
+ assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
+ }
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.property(HIVE_TABLE_NAME_PROPERTY).isPresent()) {
+ if(v.property(HIVE_TABLE_NAME_PROPERTY).value().toString().equals(HIVE_TABLE_NAME)) {
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void nonPrimitiveMap_Removed() throws IOException {
+ Set<String> actualKeys = new HashSet<String>() {{
+ add("col3");
+ add("col4");
+ }};
+
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol3(), getEdgeCol4());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(!v.property(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP).isPresent()) continue;
+
+ fail("Non-primitive map should be removed during vertex creation.");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ String mapSpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+ assertEquals(mapSpecificLabel, HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+ assertTrue(e.property(ATTRIBUTE_KEY_PROPERTY_KEY).isPresent());
+
+ assertTrue(actualKeys.contains((String) e.property(ATTRIBUTE_KEY_PROPERTY_KEY).value()));
+ }
+ }
+
+ @Test
+ public void tagAssociated_NewAttributesAdded() throws IOException {
+
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.id().toString() != "16752") continue;
+
+ assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+ assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+ assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ @Test
+ public void processEdge_PropagateSetTo_NONE() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.id().toString() != "16752") continue;
+
+ assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+ assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+ assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ @Test
+ public void processEdge_PropagateSetTo_ONE_TO_TWO() throws IOException {
+ Map<String, RelationshipCacheGenerator.TypeInfo> typeCache = new HashMap<String, RelationshipCacheGenerator.TypeInfo>() {{
+ put("__Process.inputs", new RelationshipCacheGenerator.TypeInfo("dataset_process_inputs", AtlasRelationshipDef.PropagateTags.TWO_TO_ONE));
+ }};
+
+ ElementProcessors elementProcessors = new ElementProcessors(typeCache, getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getProcessV());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeProcess());
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).value().toString(), "TWO_TO_ONE");
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ private Map<String, Map<String, List<String>>> getPostProcessMap() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+
+ map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).put("MAP", new ArrayList<>());
+
+ map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+ map.get(HIVE_TABLE_TYPE).get("MAP").add(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+
+ return map;
+ }
+
+ private Map<String, Map<String, List<String>>> getPostProcessMapPrimitive() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+ map.get(HIVE_TABLE_TYPE).put("MAP_PRIMITIVE", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).get("MAP_PRIMITIVE").add(HIVE_TABLE_COLUMNS_PARAMETERS_MAP);
+
+ return map;
+ }
+
+ private Map<String, Map<String, List<String>>> getNonPrimitiveArray() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+ map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+
+ return map;
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
index 170b1af..83e0bbe 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
@@ -36,8 +36,9 @@ import static org.testng.AssertJUnit.assertNotNull;
public class JsonNodeParsersTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void parseVertex(JsonNode nd) {
+ @Test
+ public void parseVertex() {
+ JsonNode nd = getCol1();
final int COL1_ORIGINAL_ID = 98336;
Object nodeId = getId(nd);
@@ -68,8 +69,9 @@ public class JsonNodeParsersTest extends BaseUtils {
assertProperties(vUsingPe);
}
- @Test(dataProvider = "edge")
- public void parseEdge(JsonNode nd) throws IOException {
+ @Test
+ public void parseEdge() {
+ JsonNode nd = getEdge();
final String EDGE_ORIGINAL_ID = "8k5i-35tc-acyd-1eko";
Object nodeId = getId(nd);
@@ -78,12 +80,12 @@ public class JsonNodeParsersTest extends BaseUtils {
JsonNodeParsers.ParseElement peVertex = new JsonNodeParsers.ParseVertex();
peVertex.setContext(graphSONUtility);
- peVertex.parse(tg, cache, (JsonNode) (getDBV(null)[0][0]));
- peVertex.parse(tg, cache, (JsonNode) (getTableV(null)[0][0]));
+ peVertex.parse(tg, cache, getDBV());
+ peVertex.parse(tg, cache, getTableV());
JsonNodeParsers.ParseElement pe = new JsonNodeParsers.ParseEdge();
pe.setContext(graphSONUtility);
- pe.parse(tg, cache, (JsonNode) getEdge(null)[0][0]);
+ pe.parse(tg, cache, getEdge());
updateParseElement(tg, pe, nodeId);
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
index cac09d2..459efd2 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
@@ -31,8 +31,9 @@ import static org.testng.Assert.*;
public class MappedElementCacheTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void vertexFetch(JsonNode node) {
+ @Test
+ public void vertexFetch() {
+ JsonNode node = getCol1();
MappedElementCache cache = new MappedElementCache();
TinkerGraph tg = TinkerGraph.open();
@@ -41,32 +42,15 @@ public class MappedElementCacheTest extends BaseUtils {
Vertex vx = cache.getMappedVertex(tg, 98336);
assertNotNull(vx);
assertEquals(cache.lruVertexCache.size(), 1);
- assertEquals(cache.lruEdgeCache.size(), 0);
}
@Test
- public void edgeFetch() throws IOException {
+ public void edgeFetch() {
MappedElementCache cache = new MappedElementCache();
TinkerGraph tg = TinkerGraph.open();
addEdge(tg, cache);
- assertEquals(cache.lruVertexCache.size(), 2);
- assertEquals(cache.lruEdgeCache.size(), 0);
- }
-
-
- @Test
- public void nonExistentVertexReturnsNull() {
- TinkerGraph tg = TinkerGraph.open();
- MappedElementCache cache = new MappedElementCache();
-
- assertNull(cache.fetchVertex(tg, 1111));
- assertNull(cache.fetchEdge(tg, "abcd"));
- }
-
- @DataProvider(name = "col1")
- public Object[][] getCol1(ITestContext context) throws IOException {
- return getJsonNodeFromFile("col-legacy.json");
+ assertEquals(cache.lruVertexCache.size(), 4);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
new file mode 100644
index 0000000..6227f66
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.eclipse.jetty.util.BlockingArrayQueue;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class PostProcessListPropertyTest extends BaseUtils {
+ final String HIVE_TABLE_TYPE = "hive_table";
+ final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
+
+ final String col1EdgeId = "816u-35tc-ao0l-47so";
+ final String col2EdgeId = "82rq-35tc-ao0l-2glc";
+
+ @Test
+ public void noRefNoUpdate() throws IOException {
+ TestSetup ts = new TestSetup();
+
+ ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+ ts.assertIncomplete();
+ }
+
+ @Test
+ public void refFoundVertexUpdated() throws IOException {
+ TestSetup ts = new TestSetup();
+ assertNotNull(ts.getTable());
+
+ ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+ ts.assertComplete();
+ }
+
+ @Test
+ public void updateUsingPostProcessConsumer() throws IOException {
+ TestSetup ts = new TestSetup();
+
+ BlockingQueue<Object> bc = new BlockingArrayQueue<>();
+ PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, ts.getGraph(),
+ getTypePropertyMap("hive_table", HIVE_COLUMNS_PROPERTY, "ARRAY"), 5);
+
+ Vertex tableV = fetchTableVertex(ts.getGraph());
+ consumer.processItem(tableV.id());
+ ts.assertComplete();
+ }
+
+ private class TestSetup {
+ private PostProcessListProperty postProcessListProperty;
+ private TinkerGraph tg;
+ private MappedElementCache cache;
+ private Vertex tableV;
+
+ public PostProcessListProperty getPostProcessListProperty() {
+ return postProcessListProperty;
+ }
+
+ public TinkerGraph getGraph() {
+ return tg;
+ }
+
+ public MappedElementCache getCache() {
+ return cache;
+ }
+
+ public Vertex getTable() {
+ return tableV;
+ }
+
+ public TestSetup() throws IOException {
+ postProcessListProperty = new PostProcessListProperty();
+ tg = TinkerGraph.open();
+ cache = new MappedElementCache();
+ addEdge(tg, cache);
+ tableV = fetchTableVertex(tg);
+
+ assertSetup();
+ }
+
+ public void assertSetup() {
+ assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
+ List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
+
+ assertEquals(list.size(), 2);
+ assertEquals(list.get(0), col1EdgeId);
+ assertEquals(list.get(1), col2EdgeId);
+ }
+
+ private void assertIncomplete() {
+ assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+
+ Iterator<Edge> edges = tableV.edges(Direction.OUT, getEdgeLabel(HIVE_COLUMNS_PROPERTY));
+ while (edges.hasNext()) {
+ Edge e = edges.next();
+
+ assertFalse(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ private void assertComplete() {
+ assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+ }
+
+ private void assertPropertyRemoved(String property, Vertex tableV) {
+ assertFalse(tableV.property(property).isPresent());
+ }
+
+ public String getEdgeLabel(String property ) {
+ return Constants.INTERNAL_PROPERTY_KEY_PREFIX + property;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-2-legacy.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/col-2-legacy.json b/graphdb/janus/src/test/resources/col-2-legacy.json
new file mode 100644
index 0000000..f755462
--- /dev/null
+++ b/graphdb/janus/src/test/resources/col-2-legacy.json
@@ -0,0 +1,73 @@
+{
+ "Asset.name": {
+ "type": "string",
+ "value": "col2"
+ },
+ "hive_column.type": {
+ "type": "string",
+ "value": "string"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column owner anonymous qualifiedName stocks.test_table_view.col2@cl1 name col2 position 1 type string table "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.test_table_view.col2@cl1"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "9cef2494-766c-4671-96a8-828dce677e7e"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "Asset"
+ }, {
+ "type": "string",
+ "value": "DataSet"
+ }, {
+ "type": "string",
+ "value": "Referenceable"
+ }]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693838471
+ },
+ "Asset.owner": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "hive_column.position": {
+ "type": "integer",
+ "value": 1
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": 114816,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-3-legacy.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/col-3-legacy.json b/graphdb/janus/src/test/resources/col-3-legacy.json
new file mode 100644
index 0000000..b6e8c20
--- /dev/null
+++ b/graphdb/janus/src/test/resources/col-3-legacy.json
@@ -0,0 +1,86 @@
+{
+ "Asset.name": {
+ "type": "string",
+ "value": "open"
+ },
+ "hive_column.type": {
+ "type": "string",
+ "value": "string"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column owner anonymous qualifiedName stocks.stocks_daily.open@cl1 name open position 1 type string table Tag1 "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.stocks_daily.open@cl1"
+ },
+ "__traitNames": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "Tag1"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "8231a95f-d062-4685-81aa-0b62401bc796"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "Asset"
+ },
+ {
+ "type": "string",
+ "value": "DataSet"
+ },
+ {
+ "type": "string",
+ "value": "Referenceable"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693815850
+ },
+ "Asset.owner": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "hive_column.position": {
+ "type": "integer",
+ "value": 1
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693815850
+ },
+ "_id": 16752,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col.json b/graphdb/janus/src/test/resources/edge-legacy-col.json
new file mode 100644
index 0000000..1e82f04
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "816u-35tc-ao0l-47so",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 98336,
+ "_label": "__hive_table.columns"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col2.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col2.json b/graphdb/janus/src/test/resources/edge-legacy-col2.json
new file mode 100644
index 0000000..75e0ac4
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col2.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "82rq-35tc-ao0l-2glc",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 114816,
+ "_label": "__hive_table.columns"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col3.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col3.json b/graphdb/janus/src/test/resources/edge-legacy-col3.json
new file mode 100644
index 0000000..6145ce1
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col3.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "816u-35tc-ao0l-47aa",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 98336,
+ "_label": "__hive_table.columnsMap.col3"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col4.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col4.json b/graphdb/janus/src/test/resources/edge-legacy-col4.json
new file mode 100644
index 0000000..15cdda7
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col4.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "82rq-35tc-ao0l-2gaa",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 114816,
+ "_label": "__hive_table.columnsMap.col4"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-process.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-process.json b/graphdb/janus/src/test/resources/edge-legacy-process.json
new file mode 100644
index 0000000..9285958
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-process.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "_id": "6jgh-23uw-2uqd-1elc",
+ "_type": "edge",
+ "_outV": 98312,
+ "_inV": 147504,
+ "_label": "__Process.inputs"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-tag.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-tag.json b/graphdb/janus/src/test/resources/edge-legacy-tag.json
new file mode 100644
index 0000000..1bc200a
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-tag.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "_id": "2pk3la-cxc-m61h-2pk0og",
+ "_type": "edge",
+ "_outV": 16752,
+ "_inV": 163856752,
+ "_label": "Tag1"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/lineage-v-98312.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/lineage-v-98312.json b/graphdb/janus/src/test/resources/lineage-v-98312.json
new file mode 100644
index 0000000..abdf9cb
--- /dev/null
+++ b/graphdb/janus/src/test/resources/lineage-v-98312.json
@@ -0,0 +1,79 @@
+{
+ "hive_column_lineage.depenendencyType": {
+ "type": "string",
+ "value": "SIMPLE"
+ },
+ "Asset.name": {
+ "type": "string",
+ "value": "create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1"
+ },
+ "Process.inputs": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "6jgh-23uw-2uqd-1elc"
+ }]
+ },
+ "Process.outputs": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "6ept-23uw-2wb9-47so"
+ }]
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column_lineage outputs qualifiedName stocks.test_table_view@cl1:1522693834000:col1 inputs query name create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1 depenendencyType SIMPLE "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.test_table_view@cl1:1522693834000:col1"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "9336b7a4-9cc0-4ef0-8dc1-01fce2def6a5"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "Asset"
+ }, {
+ "type": "string",
+ "value": "Process"
+ }, {
+ "type": "string",
+ "value": "Referenceable"
+ }]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column_lineage"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "_id": 98312,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/table-v-147504.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/table-v-147504.json b/graphdb/janus/src/test/resources/table-v-147504.json
index 898dce5..dc5d788 100644
--- a/graphdb/janus/src/test/resources/table-v-147504.json
+++ b/graphdb/janus/src/test/resources/table-v-147504.json
@@ -33,10 +33,12 @@
},
"hive_table.parameters": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "transient_lastDdlTime"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "transient_lastDdlTime"
+ }
+ ]
},
"hive_table.retention": {
"type": "integer",
@@ -44,10 +46,12 @@
},
"hive_table.partitionKeys": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "8dty-35tc-amfp-23xs"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "8dty-35tc-amfp-23xs"
+ }
+ ]
},
"__guid": {
"type": "string",
@@ -63,16 +67,20 @@
},
"__superTypeNames": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "Asset"
- }, {
- "type": "string",
- "value": "DataSet"
- }, {
- "type": "string",
- "value": "Referenceable"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "Asset"
+ },
+ {
+ "type": "string",
+ "value": "DataSet"
+ },
+ {
+ "type": "string",
+ "value": "Referenceable"
+ }
+ ]
},
"hive_table.viewExpandedText": {
"type": "string",
@@ -104,13 +112,37 @@
},
"hive_table.columns": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "816u-35tc-ao0l-47so"
- }, {
- "type": "string",
- "value": "82rq-35tc-ao0l-2glc"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "816u-35tc-ao0l-47so"
+ },
+ {
+ "type": "string",
+ "value": "82rq-35tc-ao0l-2glc"
+ }
+ ]
+ },
+ "hive_table.columnsMap.col3": {
+ "type": "string",
+ "value": "816u-35tc-ao0l-47aa"
+ },
+ "hive_table.columnsMap.col4": {
+ "type": "string",
+ "value": "82rq-35tc-ao0l-2gaa"
+ },
+ "hive_table.columnsMap": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "col3"
+ },
+ {
+ "type": "string",
+ "value": "col4"
+ }
+ ]
},
"__timestamp": {
"type": "long",
@@ -119,3 +151,4 @@
"_id": 147504,
"_type": "vertex"
}
+
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/tag-163856752.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/tag-163856752.json b/graphdb/janus/src/test/resources/tag-163856752.json
new file mode 100644
index 0000000..7049819
--- /dev/null
+++ b/graphdb/janus/src/test/resources/tag-163856752.json
@@ -0,0 +1,35 @@
+{
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "[]"
+ }]
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "Tag1"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "_id": 163856752,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index b05754f..025f89a 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -21,16 +21,12 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
-import org.apache.atlas.model.typedef.AtlasClassificationDef.AtlasClassificationDefs;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
/**
* Interface to persistence store of TypeDef
@@ -107,6 +103,4 @@ public interface AtlasTypeDefStore {
AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
-
- void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 886ce77..4e9a651 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -18,7 +18,6 @@
package org.apache.atlas;
-import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
index 22cd552..081376f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
@@ -18,11 +18,11 @@
package org.apache.atlas.repository.migration;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.impexp.ImportTypeDefProcessor;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -56,11 +56,11 @@ public class DataMigrationService implements Service {
private final Thread thread;
@Inject
- public DataMigrationService(AtlasTypeDefStore typeDefStore, Configuration configuration,
+ public DataMigrationService(AtlasGraph graph, AtlasTypeDefStore typeDefStore, Configuration configuration,
GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer,
AtlasTypeRegistry typeRegistry) {
this.configuration = configuration;
- this.thread = new Thread(new FileImporter(typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
+ this.thread = new Thread(new FileImporter(graph, typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
}
@Override
@@ -83,15 +83,17 @@ public class DataMigrationService implements Service {
}
public static class FileImporter implements Runnable {
+ private final AtlasGraph graph;
private final AtlasTypeDefStore typeDefStore;
private final String importDirectory;
private final GraphBackedSearchIndexer indexer;
private final AtlasTypeRegistry typeRegistry;
private final AtlasTypeDefStoreInitializer storeInitializer;
- public FileImporter(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
+ public FileImporter(AtlasGraph graph, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
AtlasTypeDefStoreInitializer storeInitializer,
String directoryName, GraphBackedSearchIndexer indexer) {
+ this.graph = graph;
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.storeInitializer = storeInitializer;
@@ -99,7 +101,16 @@ public class DataMigrationService implements Service {
this.indexer = indexer;
}
- public void performImport() throws AtlasBaseException {
+ @Override
+ public void run() {
+ try {
+ performImport();
+ } catch (AtlasBaseException e) {
+ LOG.error("Data Migration:", e);
+ }
+ }
+
+ private void performImport() throws AtlasBaseException {
try {
if(!performAccessChecks(importDirectory)) {
return;
@@ -109,7 +120,7 @@ public class DataMigrationService implements Service {
FileInputStream fs = new FileInputStream(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_DATA_NAME));
- typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs);
+ graph.importLegacyGraphSON(typeRegistry, fs);
} catch (Exception ex) {
LOG.error("Import failed!", ex);
throw new AtlasBaseException(ex);
@@ -117,11 +128,13 @@ public class DataMigrationService implements Service {
}
private boolean performAccessChecks(String path) {
- boolean ret = false;
+ final boolean ret;
+
if(StringUtils.isEmpty(path)) {
ret = false;
} else {
File f = new File(path);
+
ret = f.exists() && f.isDirectory() && f.canRead();
}
@@ -137,17 +150,19 @@ public class DataMigrationService implements Service {
private void performInit() throws AtlasBaseException, AtlasException {
indexer.instanceIsActive();
storeInitializer.instanceIsActive();
+
processIncomingTypesDef(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_TYPESDEF_NAME));
}
- @VisibleForTesting
- void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
+ private void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
try {
- AtlasImportResult result = new AtlasImportResult();
- String jsonStr = FileUtils.readFileToString(typesDefFile);
- AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+ AtlasImportResult result = new AtlasImportResult();
+ String jsonStr = FileUtils.readFileToString(typesDefFile);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
ImportTypeDefProcessor processor = new ImportTypeDefProcessor(typeDefStore, typeRegistry);
+
processor.processTypes(typesDef, result);
+
LOG.info(" types migrated: {}", result.getMetrics());
} catch (IOException e) {
LOG.error("processIncomingTypesDef: Could not process file: {}! Imported data may not be usable.", typesDefFile.getName());
@@ -157,14 +172,5 @@ public class DataMigrationService implements Service {
private File getFileFromImportDirectory(String importDirectory, String fileName) {
return Paths.get(importDirectory, fileName).toFile();
}
-
- @Override
- public void run() {
- try {
- performImport();
- } catch (AtlasBaseException e) {
- LOG.error("Data Migration:", e);
- }
- }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java b/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
deleted file mode 100644
index 2550daf..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.migration;
-
-import org.apache.atlas.model.typedef.AtlasRelationshipDef;
-import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.commons.lang.StringUtils;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-public class RelationshipCacheGenerator {
-
- public static Map<String, String> get(AtlasTypeRegistry typeRegistry) {
- Map<String, String> ret = new HashMap<>();
- Collection<AtlasRelationshipType> relationshipTypes = typeRegistry.getAllRelationshipTypes();
-
- for (AtlasRelationshipType rt : relationshipTypes) {
- AtlasRelationshipDef rd = rt.getRelationshipDef();
- String relTypeName = rt.getTypeName();
-
- add(ret, getKey(rd.getEndDef1(), rt.getEnd1Type()), relTypeName);
- add(ret, getKey(rd.getEndDef2(), rt.getEnd2Type()), relTypeName);
- }
-
- return ret;
- }
-
- private static String getKey(AtlasRelationshipEndDef ed, AtlasEntityType rt) {
- return getKey(ed.getIsLegacyAttribute(), rt.getTypeName(), ed.getName());
- }
-
- private static String getKey(String lhs, String rhs) {
- return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs);
- }
-
- private static String getKey(boolean isLegacy, String typeName, String name) {
- if(!isLegacy) {
- return "";
- }
-
- return getKey(typeName, name);
- }
-
- private static void add(Map<String, String> map, String key, String value) {
- if(StringUtils.isEmpty(key) || map.containsKey(key)) {
- return;
- }
-
- map.put(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
index 3c84e3c..67631bf 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
@@ -24,7 +24,6 @@ import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.VERTEX_TYPE;
-import java.io.InputStream;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
@@ -49,7 +48,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -110,11 +108,6 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
}
- @Override
- public void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
- getAtlasGraph().loadLegacyGraphSON(relationshipCache, fs);
- }
-
AtlasGraph getAtlasGraph() { return atlasGraph; }
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
new file mode 100644
index 0000000..9451fa2
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ComplexAttributesTest extends MigrationBaseAsserts {
+
+ @Inject
+ public ComplexAttributesTest(AtlasGraph graph) {
+ super(graph);
+ }
+
+ @Test
+ public void verify() throws IOException, AtlasBaseException {
+ String STRUCT_TYPE = "struct_type";
+ String ENTITY_TYPE = "entity_type";
+ String ENTITY_WITH_COMPLEX_COLL_TYPE = "entity_with_complex_collection_attr";
+
+ final int EXPECTED_TOTAL_COUNT = 214;
+ final int EXPECTED_ENTITY_TYPE_COUNT = 16;
+ final int EXPECTED_STRUCT_TYPE_COUNT = 3;
+ final int EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT = 1;
+
+ runFileImporter("complex-attr_db");
+
+ assertTypeCountNameGuid(STRUCT_TYPE, EXPECTED_STRUCT_TYPE_COUNT,"", "");
+ assertTypeCountNameGuid(ENTITY_TYPE, EXPECTED_ENTITY_TYPE_COUNT, "", "");
+ assertTypeCountNameGuid(ENTITY_WITH_COMPLEX_COLL_TYPE, EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT, "", "");
+
+ assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),1, "__entity_with_complex_collection_attr.listOfEntities");
+ assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),9, "__entity_with_complex_collection_attr.mapOfStructs");
+
+ assertMigrationStatus(EXPECTED_TOTAL_COUNT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
index ac0b79d..7b23e59 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
@@ -21,17 +21,13 @@ package org.apache.atlas.repository.migration;
import com.google.inject.Inject;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.io.IOException;
-import java.util.List;
-import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class HiveParititionTest extends MigrationBaseAsserts {
@@ -41,7 +37,7 @@ public class HiveParititionTest extends MigrationBaseAsserts {
super(graph);
}
- @Test(enabled = false)
+ @Test
public void fileImporterTest() throws IOException, AtlasBaseException {
final int EXPECTED_TOTAL_COUNT = 141;
final int EXPECTED_DB_COUNT = 1;
@@ -50,8 +46,6 @@ public class HiveParititionTest extends MigrationBaseAsserts {
runFileImporter("parts_db");
- assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1);
- assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1);
assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95");
@@ -62,9 +56,4 @@ public class HiveParititionTest extends MigrationBaseAsserts {
assertMigrationStatus(EXPECTED_TOTAL_COUNT);
}
-
- private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) {
- List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys");
- assertEquals(keys.size(), expectedCount);
- }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
index ffbf320..721bb31 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
@@ -35,7 +35,7 @@ public class HiveStocksTest extends MigrationBaseAsserts {
super(graph);
}
- @Test(enabled = false)
+ @Test
public void migrateStocks() throws AtlasBaseException, IOException {
final int EXPECTED_TOTAL_COUNT = 188;
final int EXPECTED_DB_COUNT = 1;
@@ -47,7 +47,6 @@ public class HiveStocksTest extends MigrationBaseAsserts {
assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0");
assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774");
- assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns");
assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891");
assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec");
assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160");
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
index ec6e64a..4b02b7f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
@@ -34,7 +34,6 @@ import org.testng.annotations.AfterClass;
import java.io.IOException;
import java.util.Iterator;
-import java.util.List;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
@@ -48,10 +47,10 @@ public class MigrationBaseAsserts {
private final String R_GUID_PROPERTY_NAME = "_r__guid";
@Inject
- private AtlasTypeDefStore typeDefStore;
+ protected AtlasTypeDefStore typeDefStore;
@Inject
- private AtlasTypeRegistry typeRegistry;
+ protected AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStoreInitializer storeInitializer;
@@ -74,7 +73,7 @@ public class MigrationBaseAsserts {
}
}
- private void loadTypesFromJson() throws IOException, AtlasBaseException {
+ protected void loadTypesFromJson() throws IOException, AtlasBaseException {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
@@ -83,7 +82,7 @@ public class MigrationBaseAsserts {
protected void runFileImporter(String directoryToImport) throws IOException, AtlasBaseException {
loadTypesFromJson();
String directoryName = TestResourceFileUtils.getDirectory(directoryToImport);
- DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry,
+ DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(graph, typeDefStore, typeRegistry,
storeInitializer, directoryName, indexer);
fi.run();
@@ -152,14 +151,19 @@ public class MigrationBaseAsserts {
assertEquals(count, expectedItems, String.format("%s", edgeTypeName));
}
- protected void assertTypeAttribute(String typeName, int expectedSize, String name, String guid, String propertyName) {
- AtlasVertex v = getVertex(typeName, name);
- String guidActual = GraphHelper.getGuid(v);
- List list = (List) GraphHelper.getProperty(v, propertyName);
+ protected void assertEdgesWithLabel(Iterator<AtlasEdge> results, int startIdx, String edgeTypeName) {
+ int count = 0;
+ AtlasEdge e = null;
+ for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) {
+ e = it.next();
+ }
- assertEquals(guidActual, guid);
- assertNotNull(list);
- assertEquals(list.size(), expectedSize);
+ assertNotNull(GraphHelper.getProperty(e, R_GUID_PROPERTY_NAME));
+ assertNotNull(GraphHelper.getProperty(e, "tagPropagation"));
+
+ if(StringUtils.isNotEmpty(edgeTypeName)) {
+ assertEquals(e.getLabel(), edgeTypeName, edgeTypeName);
+ }
}
protected void assertTypeCountNameGuid(String typeName, int expectedItems, String name, String guid) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
new file mode 100644
index 0000000..362990f
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.janus.migration.RelationshipCacheGenerator;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang.StringUtils;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class RelationshipCacheGeneratorTest {
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+ }
+
+ @Test
+ public void createLookup() {
+ final String PROCESS_INPUT_KEY = "__Process.inputs";
+ final String PROCESS_OUTPUT_KEY = "__Process.outputs";
+ String ONE_TO_TWO_STR = ONE_TO_TWO.toString();
+ String TWO_TO_ONE_STR = TWO_TO_ONE.toString();
+
+ Map<String, RelationshipCacheGenerator.TypeInfo> cache = RelationshipCacheGenerator.get(typeRegistry);
+ assertEquals(cache.size(), getLegacyAttributeCount() - 1);
+ for (Map.Entry<String, RelationshipCacheGenerator.TypeInfo> entry : cache.entrySet()) {
+ assertTrue(StringUtils.isNotEmpty(entry.getKey()));
+ assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
+ }
+
+ assertEquals(cache.get(PROCESS_INPUT_KEY).getTypeName(), "dataset_process_inputs");
+ assertEquals(cache.get(PROCESS_INPUT_KEY).getPropagateTags(), ONE_TO_TWO_STR);
+
+ assertEquals(cache.get(PROCESS_OUTPUT_KEY).getTypeName(), "process_dataset_outputs");
+ assertEquals(cache.get(PROCESS_OUTPUT_KEY).getPropagateTags(), TWO_TO_ONE_STR);
+ }
+
+ private int getLegacyAttributeCount() {
+ int count = 0;
+ for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) {
+ AtlasRelationshipDef rd = rt.getRelationshipDef();
+ if(rd.getEndDef1().getIsLegacyAttribute()) {
+ count++;
+ }
+
+ if(rd.getEndDef2().getIsLegacyAttribute()) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
deleted file mode 100644
index 48e5391..0000000
--- a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.migration;
-
-import com.google.inject.Inject;
-import org.apache.atlas.TestModules;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.typedef.AtlasRelationshipDef;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
-import org.apache.atlas.repository.store.graph.AtlasEntityStore;
-import org.apache.atlas.store.AtlasTypeDefStore;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.commons.lang.StringUtils;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.jcodings.util.Hash;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class RelationshipMappingTest {
-
- @Inject
- private AtlasTypeDefStore typeDefStore;
-
- @Inject
- private AtlasTypeRegistry typeRegistry;
-
- @BeforeClass
- public void setup() throws IOException, AtlasBaseException {
- loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
- loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
- }
-
- @Test
- public void createLookup() {
- Map<String, String> cache = RelationshipCacheGenerator.get(typeRegistry);
- assertEquals(cache.size(), getLegacyAttributeCount() - 1);
- for (Map.Entry<String, String> entry : cache.entrySet()) {
- assertTrue(StringUtils.isNotEmpty(entry.getKey()));
- assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
- }
- }
-
- private int getLegacyAttributeCount() {
- int count = 0;
- for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) {
- AtlasRelationshipDef rd = rt.getRelationshipDef();
- if(rd.getEndDef1().getIsLegacyAttribute()) {
- count++;
- }
-
- if(rd.getEndDef2().getIsLegacyAttribute()) {
- count++;
- }
- }
-
- return count;
- }
-}