You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2018/05/23 06:12:12 UTC
[1/8] atlas git commit: ATLAS-2705 #2
Repository: atlas
Updated Branches:
refs/heads/branch-1.0 1e9c71152 -> bf037c92e
ATLAS-2705 #2
Change-Id: I37796a6596cb284ba885dc05ffacd1dd7bcb6fba
(cherry picked from commit a0269b9ce8c1ebbe056b786f4bc1e3a522ca0e50)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/e5e45589
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/e5e45589
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/e5e45589
Branch: refs/heads/branch-1.0
Commit: e5e45589cbb0f4f96ffe8ae64342a580f36ed9b4
Parents: 1e9c711
Author: apoorvnaik <ap...@apache.org>
Authored: Tue May 22 14:11:26 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue May 22 23:11:12 2018 -0700
----------------------------------------------------------------------
.../atlas/glossary/GlossaryCategoryUtils.java | 16 +++++-----
.../atlas/glossary/GlossaryTermUtils.java | 10 +++---
.../store/graph/AtlasRelationshipStore.java | 7 +++++
.../graph/v1/AtlasRelationshipStoreV1.java | 10 ++++--
.../store/graph/v1/DeleteHandlerV1.java | 33 ++++----------------
5 files changed, 35 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/e5e45589/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
index 42cdef8..38ce147 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryCategoryUtils.java
@@ -113,7 +113,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
updatedCategory.getAnchor().getGlossaryGuid(),
storeObject.getGuid());
}
- relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid());
+ relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid(), true);
// Derive the qualifiedName when anchor changes
String anchorGlossaryGuid = updatedCategory.getAnchor().getGlossaryGuid();
@@ -135,7 +135,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Deleting category anchor");
}
- relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid());
+ relationshipStore.deleteById(storeObject.getAnchor().getRelationGuid(), true);
}
break;
}
@@ -174,7 +174,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
relationshipStore.update(parentRelationship);
} else {
// Delete link to existing parent and link to new parent
- relationshipStore.deleteById(parentRelationship.getGuid());
+ relationshipStore.deleteById(parentRelationship.getGuid(), true);
createRelationship(defineCategoryHierarchyLink(newParent, storeObject.getGuid()));
}
}
@@ -209,7 +209,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Removing category parent, category = {}, parent = {}", storeObject.getGuid(), existingParent.getDisplayText());
}
- relationshipStore.deleteById(existingParent.getRelationGuid());
+ relationshipStore.deleteById(existingParent.getRelationGuid(), true);
// Parent deleted, qualifiedName needs recomputation
@@ -343,7 +343,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Creating term relation with category = {}, terms = {}", storeObject.getDisplayName(), term.getDisplayText());
}
- relationshipStore.deleteById(term.getRelationGuid());
+ relationshipStore.deleteById(term.getRelationGuid(), true);
}
}
}
@@ -485,7 +485,7 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Deleting child, category = {}, child = {}", storeObject.getDisplayName(), child.getDisplayText());
}
- relationshipStore.deleteById(child.getRelationGuid());
+ relationshipStore.deleteById(child.getRelationGuid(), true);
}
}
}
@@ -565,9 +565,11 @@ public class GlossaryCategoryUtils extends GlossaryUtils {
LOG.debug("Child anchor guid({}) doesn't match parent anchor guid({}). Updating child anchor", childAnchorGuid, parentAnchorGuid);
}
// Remove old glossary relation
- relationshipStore.deleteById(child.getAnchor().getRelationGuid());
+ relationshipStore.deleteById(child.getAnchor().getRelationGuid(), true);
+
// Link to new glossary
createRelationship(defineCategoryAnchorRelation(parentAnchorGuid, child.getGuid()));
+
// Update the child's anchor GUID
child.getAnchor().setGlossaryGuid(parentAnchorGuid);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e5e45589/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
index 2af4e79..8094019 100644
--- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
+++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryTermUtils.java
@@ -119,7 +119,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
}
AtlasRelatedObjectId existingTermRelation = assignedEntityMap.get(relatedObjectId.getGuid());
if (CollectionUtils.isNotEmpty(assignedEntities) && isRelationshipGuidSame(existingTermRelation, relatedObjectId)) {
- relationshipStore.deleteById(relatedObjectId.getRelationshipGuid());
+ relationshipStore.deleteById(relatedObjectId.getRelationshipGuid(), true);
} else {
throw new AtlasBaseException(AtlasErrorCode.INVALID_TERM_DISSOCIATION, relatedObjectId.getRelationshipGuid(), glossaryTerm.getGuid(), relatedObjectId.getGuid());
}
@@ -176,7 +176,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
updatedTermAnchor.getGlossaryGuid(),
storeObject.getDisplayName());
}
- relationshipStore.deleteById(existingAnchor.getRelationGuid());
+ relationshipStore.deleteById(existingAnchor.getRelationGuid(), true);
// Derive the qualifiedName when anchor changes
String anchorGlossaryGuid = updatedTermAnchor.getGlossaryGuid();
@@ -195,7 +195,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Deleting term anchor");
}
- relationshipStore.deleteById(existingAnchor.getRelationGuid());
+ relationshipStore.deleteById(existingAnchor.getRelationGuid(), true);
}
break;
}
@@ -406,7 +406,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Deleting relation guid = {}, text = {}", categorizationHeader.getRelationGuid(), categorizationHeader.getDisplayText());
}
- relationshipStore.deleteById(categorizationHeader.getRelationGuid());
+ relationshipStore.deleteById(categorizationHeader.getRelationGuid(), true);
}
}
}
@@ -466,7 +466,7 @@ public class GlossaryTermUtils extends GlossaryUtils {
if (DEBUG_ENABLED) {
LOG.debug("Deleting term relation = {}, terms = {}", relation, termHeader.getDisplayText());
}
- relationshipStore.deleteById(termHeader.getRelationGuid());
+ relationshipStore.deleteById(termHeader.getRelationGuid(), true);
}
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e5e45589/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
index 072ea94..4468951 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasRelationshipStore.java
@@ -71,4 +71,11 @@ public interface AtlasRelationshipStore {
* @param guid relationship instance guid
*/
void deleteById(String guid) throws AtlasBaseException;
+
+ /**
+ * Delete a relationship instance using guid.
+ * @param guid relationship instance guid
+ * @param forceDelete force delete the relationship edge
+ */
+ void deleteById(String guid, boolean forceDelete) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/e5e45589/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
index a600b20..b683c6b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasRelationshipStoreV1.java
@@ -234,8 +234,14 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
@Override
@GraphTransaction
public void deleteById(String guid) throws AtlasBaseException {
+ deleteById(guid, false);
+ }
+
+ @Override
+ @GraphTransaction
+ public void deleteById(String guid, boolean forceDelete) throws AtlasBaseException {
if (LOG.isDebugEnabled()) {
- LOG.debug("==> deleteById({})", guid);
+ LOG.debug("==> deleteById({}, {})", guid, forceDelete);
}
if (StringUtils.isEmpty(guid)) {
@@ -252,7 +258,7 @@ public class AtlasRelationshipStoreV1 implements AtlasRelationshipStore {
throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid);
}
- deleteHandler.deleteRelationships(Collections.singleton(edge));
+ deleteHandler.deleteRelationships(Collections.singleton(edge), forceDelete);
// notify entities for added/removed classification propagation
entityChangeNotifier.notifyPropagatedEntities();
http://git-wip-us.apache.org/repos/asf/atlas/blob/e5e45589/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
index b4d26d0..69a4758 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java
@@ -52,11 +52,7 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.apache.atlas.model.TypeCategory.ARRAY;
-import static org.apache.atlas.model.TypeCategory.CLASSIFICATION;
-import static org.apache.atlas.model.TypeCategory.MAP;
-import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
-import static org.apache.atlas.model.TypeCategory.STRUCT;
+import static org.apache.atlas.model.TypeCategory.*;
import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
@@ -64,25 +60,7 @@ import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_NAME_PRO
import static org.apache.atlas.repository.Constants.CLASSIFICATION_LABEL;
import static org.apache.atlas.repository.Constants.PROPAGATED_TRAIT_NAMES_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.TRAIT_NAMES_PROPERTY_KEY;
-import static org.apache.atlas.repository.graph.GraphHelper.addToPropagatedTraitNames;
-import static org.apache.atlas.repository.graph.GraphHelper.getAssociatedEntityVertex;
-import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdge;
-import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEdges;
-import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid;
-import static org.apache.atlas.repository.graph.GraphHelper.getClassificationName;
-import static org.apache.atlas.repository.graph.GraphHelper.getClassificationVertices;
-import static org.apache.atlas.repository.graph.GraphHelper.getCollectionElementsUsingRelationship;
-import static org.apache.atlas.repository.graph.GraphHelper.getGuid;
-import static org.apache.atlas.repository.graph.GraphHelper.getMapValuesUsingRelationship;
-import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedClassificationEdge;
-import static org.apache.atlas.repository.graph.GraphHelper.getPropagatedEdges;
-import static org.apache.atlas.repository.graph.GraphHelper.getPropagationEnabledClassificationVertices;
-import static org.apache.atlas.repository.graph.GraphHelper.getRelationshipGuid;
-import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames;
-import static org.apache.atlas.repository.graph.GraphHelper.getTypeName;
-import static org.apache.atlas.repository.graph.GraphHelper.string;
-import static org.apache.atlas.repository.graph.GraphHelper.updateModificationMetadata;
+import static org.apache.atlas.repository.graph.GraphHelper.*;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getIdFromEdge;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getQualifiedAttributePropertyKey;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.getState;
@@ -150,16 +128,17 @@ public abstract class DeleteHandlerV1 {
* @throws AtlasBaseException
*/
public void deleteRelationship(AtlasEdge edge) throws AtlasBaseException {
- deleteRelationships(Collections.singleton(edge));
+ deleteRelationships(Collections.singleton(edge), false);
}
/**
* Deletes the specified relationship edges.
*
* @param edges
+ * @param forceDelete
* @throws AtlasBaseException
*/
- public void deleteRelationships(Collection<AtlasEdge> edges) throws AtlasBaseException {
+ public void deleteRelationships(Collection<AtlasEdge> edges, final boolean forceDelete) throws AtlasBaseException {
for (AtlasEdge edge : edges) {
boolean isInternal = isInternalType(edge.getInVertex()) && isInternalType(edge.getOutVertex());
@@ -171,7 +150,7 @@ public abstract class DeleteHandlerV1 {
continue;
}
- deleteEdge(edge, isInternal);
+ deleteEdge(edge, isInternal || forceDelete);
}
}
[5/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
new file mode 100644
index 0000000..4b236c1
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/TypesWithCollectionsFinderTest.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.apache.atlas.repository.graphdb.janus.migration.TypesWithCollectionsFinder;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class TypesWithCollectionsFinderTest extends MigrationBaseAsserts {
+ @Inject
+ protected TypesWithCollectionsFinderTest(AtlasGraph graph) {
+ super(graph);
+ }
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadTypesFromJson();
+ }
+
+ @Test
+ public void fetchAll() {
+ Map<String, Map<String, List<String>>> typeAttrMap = TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry);
+
+ assertEquals(typeAttrMap.size(), 9);
+
+ assertProperties(typeAttrMap, "__AtlasUserProfile", "ARRAY", "__AtlasUserProfile.savedSearches");
+
+ assertProperties(typeAttrMap, "Process", "ARRAY", "Process.inputs");
+ assertProperties(typeAttrMap, "Process", "ARRAY", "Process.outputs");
+
+ assertProperties(typeAttrMap, "hdfs_path", "MAP_PRIMITIVE", "hdfs_path.extendedAttributes");
+
+ assertProperties(typeAttrMap, "hive_column_lineage", "ARRAY", "Process.inputs");
+ assertProperties(typeAttrMap, "hive_column_lineage", "ARRAY", "Process.outputs");
+
+ assertProperties(typeAttrMap, "hive_db", "MAP_PRIMITIVE", "hive_db.parameters");
+
+ assertProperties(typeAttrMap, "hive_process", "ARRAY", "Process.inputs");
+ assertProperties(typeAttrMap, "hive_process", "ARRAY", "Process.outputs");
+
+ assertProperties(typeAttrMap, "hive_storagedesc", "ARRAY", "hive_storagedesc.sortCols");
+
+ assertProperties(typeAttrMap, "hive_serde", "MAP_PRIMITIVE", "hive_serde.parameters");
+
+ assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.partitionKeys");
+ assertProperties(typeAttrMap, "hive_table", "ARRAY", "hive_table.columns");
+ assertProperties(typeAttrMap, "hive_table", "MAP_PRIMITIVE", "hive_table.parameters");
+ }
+
+ private void assertProperties(Map<String, Map<String, List<String>>> typeAttrMap, String typeName, String typeCategory, String propertyName) {
+ List<String> actualProperties = typeAttrMap.get(typeName).get(typeCategory);
+
+ assertTrue(actualProperties.contains(propertyName));
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
index d810a11..b18e5d3 100644
--- a/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityTestBase.java
@@ -35,7 +35,7 @@ import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
-import org.apache.atlas.repository.impexp.ExportService;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.repository.store.graph.AtlasEntityStore;
import org.apache.atlas.runner.LocalSolrRunner;
@@ -82,6 +82,9 @@ public class AtlasEntityTestBase {
@Inject
private EntityGraphMapper graphMapper;
+ @Inject
+ protected AtlasGraph graph;
+
AtlasEntityChangeNotifier mockChangeNotifier = mock(AtlasEntityChangeNotifier.class);
@BeforeClass
[6/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
index e863d9f..aee1b69 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/BaseUtils.java
@@ -26,14 +26,16 @@ import org.apache.tinkerpop.gremlin.structure.io.graphson.TypeInfo;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
import org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper;
-import org.testng.ITestContext;
+import org.testng.SkipException;
import org.testng.annotations.BeforeClass;
-import org.testng.annotations.DataProvider;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
import static org.testng.AssertJUnit.assertTrue;
@@ -41,12 +43,16 @@ public class BaseUtils {
private static final String resourcesDirRelativePath = "/src/test/resources/";
private String resourceDir;
- protected final RelationshipTypeCache emptyRelationshipCache = new RelationshipTypeCache(new HashMap<>());
+ protected final ElementProcessors emptyRelationshipCache = new ElementProcessors(new HashMap<>(), new HashMap<>());
protected GraphSONUtility graphSONUtility;
- protected Object[][] getJsonNodeFromFile(String s) throws IOException {
+ protected JsonNode getJsonNodeFromFile(String s) {
File f = new File(getFilePath(s));
- return new Object[][]{{getEntityNode(FileUtils.readFileToString(f))}};
+ try {
+ return getEntityNode(FileUtils.readFileToString(f));
+ } catch (IOException e) {
+ throw new SkipException("getJsonNodeFromFile: " + s, e);
+ }
}
protected String getFilePath(String fileName) {
@@ -76,12 +82,18 @@ public class BaseUtils {
utility.vertexFromJson(tg, node);
}
- protected void addEdge(TinkerGraph tg, MappedElementCache cache) throws IOException {
+ protected void addEdge(TinkerGraph tg, MappedElementCache cache) {
GraphSONUtility gu = graphSONUtility;
- gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
- gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
- gu.edgeFromJson(tg, cache, (JsonNode) getEdge(null)[0][0]);
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, cache, getEdge(), getEdgeCol(), getEdgeCol2());
+ }
+
+ protected void addEdgesForMap(TinkerGraph tg, MappedElementCache cache) {
+ GraphSONUtility gu = graphSONUtility;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, cache, getEdgeCol3(), getEdgeCol4());
}
protected Vertex fetchTableVertex(TinkerGraph tg) {
@@ -91,29 +103,84 @@ public class BaseUtils {
return (Vertex) query.next();
}
- @DataProvider(name = "col1")
- public Object[][] getCol1(ITestContext context) throws IOException {
+ protected Map<String, Map<String, List<String>>> getTypePropertyMap(String type, String property, String category) {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(type, new HashMap<>());
+ map.get(type).put(category, new ArrayList<>());
+ map.get(type).get(category).add(property);
+ return map;
+ }
+
+ protected void addVertexToGraph(TinkerGraph tg, GraphSONUtility gu, JsonNode... nodes) {
+ for(JsonNode n : nodes) {
+ gu.vertexFromJson(tg, n);
+ }
+ }
+
+ protected void addEdgeToGraph(TinkerGraph tg, GraphSONUtility gu, MappedElementCache cache, JsonNode... nodes) {
+
+ for(JsonNode n : nodes) {
+ gu.edgeFromJson(tg, cache, n);
+ }
+ }
+
+ public JsonNode getCol1() {
return getJsonNodeFromFile("col-legacy.json");
}
- @DataProvider(name = "dbType")
- public Object[][] getDbType(ITestContext context) throws IOException {
+ public JsonNode getCol2() {
+ return getJsonNodeFromFile("col-2-legacy.json");
+ }
+
+ public JsonNode getCol3() {
+ return getJsonNodeFromFile("col-3-legacy.json");
+ }
+
+ public JsonNode getDbType() {
return getJsonNodeFromFile("db-type-legacy.json");
}
- @DataProvider(name = "edge")
- public Object[][] getEdge(ITestContext context) throws IOException {
+ public JsonNode getEdge() {
return getJsonNodeFromFile("edge-legacy.json");
}
- @DataProvider(name = "dbV")
- public Object[][] getDBV(ITestContext context) throws IOException {
- return getJsonNodeFromFile("db-v-65544.json");
+ public JsonNode getEdgeCol() {
+ return getJsonNodeFromFile("edge-legacy-col.json");
}
+ public JsonNode getEdgeCol2() {
+ return getJsonNodeFromFile("edge-legacy-col2.json");
+ }
+
+ public JsonNode getEdgeCol3() {
+ return getJsonNodeFromFile("edge-legacy-col3.json");
+ }
+
+ public JsonNode getEdgeCol4() {
+ return getJsonNodeFromFile("edge-legacy-col4.json");
+ }
+
+ public JsonNode getEdgeTag() {
+ return getJsonNodeFromFile("edge-legacy-tag.json");
+ }
+
+ public JsonNode getDBV() {
+ return getJsonNodeFromFile("db-v-65544.json");
+ }
- @DataProvider(name = "tableV")
- public Object[][] getTableV(ITestContext context) throws IOException {
+ public JsonNode getTableV() {
return getJsonNodeFromFile("table-v-147504.json");
}
+
+ public JsonNode getTagV() {
+ return getJsonNodeFromFile("tag-163856752.json");
+ }
+
+ public JsonNode getProcessV() {
+ return getJsonNodeFromFile("lineage-v-98312.json");
+ }
+
+ public JsonNode getEdgeProcess() {
+ return getJsonNodeFromFile("edge-legacy-process.json");
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
deleted file mode 100644
index 4d73c78..0000000
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityPostProcessTest.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graphdb.janus.migration;
-
-import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.eclipse.jetty.util.BlockingArrayQueue;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.AssertJUnit.assertTrue;
-
-public class GraphSONUtilityPostProcessTest extends BaseUtils {
- final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
- final String edgeId1 = "816u-35tc-ao0l-47so";
- final String edgeId2 = "82rq-35tc-ao0l-2glc";
-
- final String edgeId1x = "816u-35tc-ao0l-xxxx";
- final String edgeId2x = "82rq-35tc-ao0l-xxxx";
-
- private TinkerGraph tg;
- private MappedElementCache cache = new MappedElementCache();
- private Vertex tableV;
-
- @Test
- public void noRefNoUpdate() throws IOException {
- tg = TinkerGraph.open();
- graphSONUtility = new GraphSONUtility(emptyRelationshipCache);
-
- addEdge(tg, cache);
-
- tableV = fetchTableVertex(tg);
- assertNotNull(tableV);
-
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
-
- graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
- }
-
- @Test(dependsOnMethods = "noRefNoUpdate")
- public void refFoundVertexUpdated() throws IOException {
-
- cache.lruEdgeCache.put(edgeId1, edgeId1x);
- cache.lruEdgeCache.put(edgeId2, edgeId2x);
-
- graphSONUtility.replaceReferencedEdgeIdForList(tg, cache, tableV, HIVE_COLUMNS_PROPERTY);
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1x, edgeId2x, tableV);
- }
-
- @Test(dependsOnMethods = "refFoundVertexUpdated")
- public void updateUsingPostProcessConsumer() throws IOException {
- MappedElementCache cache = new MappedElementCache();
- BlockingQueue<Object> bc = new BlockingArrayQueue<>();
- PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, tg, graphSONUtility,
- new String[] {HIVE_COLUMNS_PROPERTY}, cache, 5);
-
- cache.lruEdgeCache.put(edgeId1x, edgeId1);
- cache.lruEdgeCache.put(edgeId2x, edgeId2);
- consumer.processItem(tableV.id());
-
- assertListProperty(HIVE_COLUMNS_PROPERTY, edgeId1, edgeId2, tableV);
- }
-
- private void assertListProperty(String HIVE_COLUMNS_PROPERTY, String edgeId1, String edgeId2, Vertex tableV) {
- assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
- List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
-
- assertEquals(list.size(), 2);
- assertEquals(list.get(0), edgeId1);
- assertEquals(list.get(1), edgeId2);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
index 794b547..049fd64 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtilityTest.java
@@ -18,36 +18,58 @@
package org.apache.atlas.repository.graphdb.janus.migration;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.commons.lang.StringUtils;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
import org.apache.tinkerpop.shaded.jackson.databind.JsonNode;
import org.testng.Assert;
-import org.testng.ITestContext;
-import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY;
import static org.testng.Assert.*;
public class GraphSONUtilityTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void idFetch(JsonNode node) {
+ private final String HIVE_TABLE_NAME_PROPERTY = "Asset.name";
+ private final String HIVE_TABLE_NAME = "test_table_view";
+ private final String HIVE_TABLE_COLUMNS_RELATIONSHIP = "hive_table.columns";
+ private final String HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP = "hive_table.columnsMap";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP = "hive_table.parameters";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY = "transient_lastDdlTime";
+ private final String HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY = String.format("%s.%s", HIVE_TABLE_COLUMNS_PARAMETERS_MAP, HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY);
+ private final String HIVE_TABLE_TYPE = "hive_table";
+
+ @Test
+ public void idFetch() {
+ JsonNode node = getCol1();
+ final int EXPECTED_ID = 98336;
Object o = GraphSONUtility.getTypedValueFromJsonNode(node.get(GraphSONTokensTP2._ID));
assertNotNull(o);
- assertEquals((int) o, 98336);
+ assertEquals((int) o, EXPECTED_ID);
}
- @Test(dataProvider = "col1")
- public void verifyReadProperties(JsonNode node) {
+ @Test
+ public void verifyReadProperties() {
+ JsonNode node = getCol1();
Map<String, Object> props = GraphSONUtility.readProperties(node);
assertEquals(props.get("__superTypeNames").getClass(), ArrayList.class);
@@ -58,8 +80,9 @@ public class GraphSONUtilityTest extends BaseUtils {
assertNotNull(props);
}
- @Test(dataProvider = "col1")
- public void dataNodeReadAndVertexAddedToGraph(JsonNode entityNode) throws IOException {
+ @Test
+ public void dataNodeReadAndVertexAddedToGraph() throws IOException {
+ JsonNode entityNode = getCol1();
TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
Map<String, Object> map = gu.vertexFromJson(tg, entityNode);
@@ -71,8 +94,9 @@ public class GraphSONUtilityTest extends BaseUtils {
assertTrue(v.property(VERTEX_ID_IN_IMPORT_KEY).isPresent());
}
- @Test(dataProvider = "dbType")
- public void typeNodeReadAndVertexNotAddedToGraph(JsonNode entityNode) throws IOException {
+ @Test
+ public void typeNodeReadAndVertexNotAddedToGraph() throws IOException {
+ JsonNode entityNode = getDbType();
TinkerGraph tg = TinkerGraph.open();
GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
gu.vertexFromJson(tg, entityNode);
@@ -81,19 +105,46 @@ public class GraphSONUtilityTest extends BaseUtils {
}
@Test
- public void edgeReadAndAddedToGraph() throws IOException {
+ public void updateNonPrimitiveArrayProperty() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getNonPrimitiveArray());
+
TinkerGraph tg = TinkerGraph.open();
- GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
- Map<String, Object> m = null;
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ addVertexToGraph(tg, gu, getTableV());
- m = gu.vertexFromJson(tg, (JsonNode) (getDBV(null)[0][0]));
- assertNull(m);
+ Vertex v = tg.traversal().V().next();
+ assertNotNull(v);
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
- m = gu.vertexFromJson(tg, (JsonNode) (getTableV(null))[0][0]);
- assertNull(m);
+ Map<String, String> list = (Map<String, String>) v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).value();
+ assertEquals(list.size(), 2);
+ }
- m = gu.edgeFromJson(tg, new MappedElementCache(), (JsonNode) getEdge(null)[0][0]);
- assertNull(m);
+
+ @Test
+ public void updatePrimitiveMapProperty() {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMapPrimitive());
+
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ addVertexToGraph(tg, gu, getTableV());
+
+ Vertex v = tg.traversal().V().next();
+ assertNotNull(v);
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).isPresent());
+ assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).size(), 1);
+ assertEquals(((Map) v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP).value()).get(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_KEY), "1522693834");
+ assertFalse(v.property(HIVE_TABLE_COLUMNS_PARAMETERS_MAP_VALUE_KEY).isPresent());
+ }
+
+ @Test
+ public void edgeReadAndAddedToGraph() {
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(emptyRelationshipCache);
+ Map<String, Object> m = null;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdge());
Assert.assertEquals((long) tg.traversal().V().count().next(), 2L);
Assert.assertEquals((long) tg.traversal().E().count().next(), 1L);
@@ -101,4 +152,179 @@ public class GraphSONUtilityTest extends BaseUtils {
Edge e = tg.edges().next();
assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
}
+
+ @Test
+ public void edgeReadAndArrayIndexAdded() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+ Map<String, Object> m = null;
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol(), getEdgeCol2());
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ String arraySpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+ if(arraySpecificLabel.equals(HIVE_TABLE_COLUMNS_RELATIONSHIP)) {
+ assertTrue(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+ }
+ assertTrue(e.property(EDGE_ID_IN_IMPORT_KEY).isPresent());
+ }
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.property(HIVE_TABLE_NAME_PROPERTY).isPresent()) {
+ if(v.property(HIVE_TABLE_NAME_PROPERTY).value().toString().equals(HIVE_TABLE_NAME)) {
+ assertTrue(v.property(HIVE_TABLE_COLUMNS_RELATIONSHIP).isPresent());
+ }
+ }
+ }
+ }
+
+ @Test
+ public void nonPrimitiveMap_Removed() throws IOException {
+ Set<String> actualKeys = new HashSet<String>() {{
+ add("col3");
+ add("col4");
+ }};
+
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getCol1(), getCol2());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeCol3(), getEdgeCol4());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(!v.property(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP).isPresent()) continue;
+
+ fail("Non-primitive map should be removed during vertex creation.");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ String mapSpecificLabel = StringUtils.remove(e.label(), Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+ assertEquals(mapSpecificLabel, HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+ assertTrue(e.property(ATTRIBUTE_KEY_PROPERTY_KEY).isPresent());
+
+ assertTrue(actualKeys.contains((String) e.property(ATTRIBUTE_KEY_PROPERTY_KEY).value()));
+ }
+ }
+
+ @Test
+ public void tagAssociated_NewAttributesAdded() throws IOException {
+
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.id().toString() != "16752") continue;
+
+ assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+ assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+ assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ @Test
+ public void processEdge_PropagateSetTo_NONE() throws IOException {
+ ElementProcessors elementProcessors = new ElementProcessors(new HashMap<>(), getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getTagV(), getDBV(), getTableV(), getCol3());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeTag());
+
+ Iterator<Vertex> vertices = tg.vertices();
+ while(vertices.hasNext()) {
+ Vertex v = vertices.next();
+ if(v.id().toString() != "16752") continue;
+
+ assertTrue(v.property(CLASSIFICATION_ENTITY_GUID).isPresent());
+ assertTrue(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).isPresent());
+ assertEquals(v.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY).values(), "NONE");
+ }
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ @Test
+ public void processEdge_PropagateSetTo_ONE_TO_TWO() throws IOException {
+ Map<String, RelationshipCacheGenerator.TypeInfo> typeCache = new HashMap<String, RelationshipCacheGenerator.TypeInfo>() {{
+ put("__Process.inputs", new RelationshipCacheGenerator.TypeInfo("dataset_process_inputs", AtlasRelationshipDef.PropagateTags.TWO_TO_ONE));
+ }};
+
+ ElementProcessors elementProcessors = new ElementProcessors(typeCache, getPostProcessMap());
+ TinkerGraph tg = TinkerGraph.open();
+ GraphSONUtility gu = new GraphSONUtility(elementProcessors);
+
+ addVertexToGraph(tg, gu, getDBV(), getTableV(), getProcessV());
+ addEdgeToGraph(tg, gu, new MappedElementCache(), getEdgeProcess());
+
+ Iterator<Edge> edges = tg.edges();
+ while(edges.hasNext()) {
+ Edge e = edges.next();
+ assertTrue(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).isPresent());
+ assertEquals(e.property(Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY).value().toString(), "TWO_TO_ONE");
+ assertEquals(e.property(STATE_PROPERTY_KEY).value(), "ACTIVE");
+ assertTrue(e.property(Constants.RELATIONSHIP_GUID_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ private Map<String, Map<String, List<String>>> getPostProcessMap() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+
+ map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).put("MAP", new ArrayList<>());
+
+ map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+ map.get(HIVE_TABLE_TYPE).get("MAP").add(HIVE_TABLE_COLUMNS_MAP_RELATIONSHIP);
+
+ return map;
+ }
+
+ private Map<String, Map<String, List<String>>> getPostProcessMapPrimitive() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+ map.get(HIVE_TABLE_TYPE).put("MAP_PRIMITIVE", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).get("MAP_PRIMITIVE").add(HIVE_TABLE_COLUMNS_PARAMETERS_MAP);
+
+ return map;
+ }
+
+ private Map<String, Map<String, List<String>>> getNonPrimitiveArray() {
+ Map<String, Map<String, List<String>>> map = new HashMap<>();
+ map.put(HIVE_TABLE_TYPE, new HashMap<>());
+ map.get(HIVE_TABLE_TYPE).put("ARRAY", new ArrayList<>());
+ map.get(HIVE_TABLE_TYPE).get("ARRAY").add(HIVE_TABLE_COLUMNS_RELATIONSHIP);
+
+ return map;
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
index 170b1af..83e0bbe 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeParsersTest.java
@@ -36,8 +36,9 @@ import static org.testng.AssertJUnit.assertNotNull;
public class JsonNodeParsersTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void parseVertex(JsonNode nd) {
+ @Test
+ public void parseVertex() {
+ JsonNode nd = getCol1();
final int COL1_ORIGINAL_ID = 98336;
Object nodeId = getId(nd);
@@ -68,8 +69,9 @@ public class JsonNodeParsersTest extends BaseUtils {
assertProperties(vUsingPe);
}
- @Test(dataProvider = "edge")
- public void parseEdge(JsonNode nd) throws IOException {
+ @Test
+ public void parseEdge() {
+ JsonNode nd = getEdge();
final String EDGE_ORIGINAL_ID = "8k5i-35tc-acyd-1eko";
Object nodeId = getId(nd);
@@ -78,12 +80,12 @@ public class JsonNodeParsersTest extends BaseUtils {
JsonNodeParsers.ParseElement peVertex = new JsonNodeParsers.ParseVertex();
peVertex.setContext(graphSONUtility);
- peVertex.parse(tg, cache, (JsonNode) (getDBV(null)[0][0]));
- peVertex.parse(tg, cache, (JsonNode) (getTableV(null)[0][0]));
+ peVertex.parse(tg, cache, getDBV());
+ peVertex.parse(tg, cache, getTableV());
JsonNodeParsers.ParseElement pe = new JsonNodeParsers.ParseEdge();
pe.setContext(graphSONUtility);
- pe.parse(tg, cache, (JsonNode) getEdge(null)[0][0]);
+ pe.parse(tg, cache, getEdge());
updateParseElement(tg, pe, nodeId);
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
index cac09d2..459efd2 100644
--- a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCacheTest.java
@@ -31,8 +31,9 @@ import static org.testng.Assert.*;
public class MappedElementCacheTest extends BaseUtils {
- @Test(dataProvider = "col1")
- public void vertexFetch(JsonNode node) {
+ @Test
+ public void vertexFetch() {
+ JsonNode node = getCol1();
MappedElementCache cache = new MappedElementCache();
TinkerGraph tg = TinkerGraph.open();
@@ -41,32 +42,15 @@ public class MappedElementCacheTest extends BaseUtils {
Vertex vx = cache.getMappedVertex(tg, 98336);
assertNotNull(vx);
assertEquals(cache.lruVertexCache.size(), 1);
- assertEquals(cache.lruEdgeCache.size(), 0);
}
@Test
- public void edgeFetch() throws IOException {
+ public void edgeFetch() {
MappedElementCache cache = new MappedElementCache();
TinkerGraph tg = TinkerGraph.open();
addEdge(tg, cache);
- assertEquals(cache.lruVertexCache.size(), 2);
- assertEquals(cache.lruEdgeCache.size(), 0);
- }
-
-
- @Test
- public void nonExistentVertexReturnsNull() {
- TinkerGraph tg = TinkerGraph.open();
- MappedElementCache cache = new MappedElementCache();
-
- assertNull(cache.fetchVertex(tg, 1111));
- assertNull(cache.fetchEdge(tg, "abcd"));
- }
-
- @DataProvider(name = "col1")
- public Object[][] getCol1(ITestContext context) throws IOException {
- return getJsonNodeFromFile("col-legacy.json");
+ assertEquals(cache.lruVertexCache.size(), 4);
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
new file mode 100644
index 0000000..6227f66
--- /dev/null
+++ b/graphdb/janus/src/test/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessListPropertyTest.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
+import org.eclipse.jetty.util.BlockingArrayQueue;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertTrue;
+
+public class PostProcessListPropertyTest extends BaseUtils {
+ final String HIVE_TABLE_TYPE = "hive_table";
+ final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
+
+ final String col1EdgeId = "816u-35tc-ao0l-47so";
+ final String col2EdgeId = "82rq-35tc-ao0l-2glc";
+
+ @Test
+ public void noRefNoUpdate() throws IOException {
+ TestSetup ts = new TestSetup();
+
+ ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+ ts.assertIncomplete();
+ }
+
+ @Test
+ public void refFoundVertexUpdated() throws IOException {
+ TestSetup ts = new TestSetup();
+ assertNotNull(ts.getTable());
+
+ ts.getPostProcessListProperty().process(ts.getTable(), HIVE_TABLE_TYPE, HIVE_COLUMNS_PROPERTY);
+ ts.assertComplete();
+ }
+
+ @Test
+ public void updateUsingPostProcessConsumer() throws IOException {
+ TestSetup ts = new TestSetup();
+
+ BlockingQueue<Object> bc = new BlockingArrayQueue<>();
+ PostProcessManager.Consumer consumer = new PostProcessManager.Consumer(bc, ts.getGraph(),
+ getTypePropertyMap("hive_table", HIVE_COLUMNS_PROPERTY, "ARRAY"), 5);
+
+ Vertex tableV = fetchTableVertex(ts.getGraph());
+ consumer.processItem(tableV.id());
+ ts.assertComplete();
+ }
+
+ private class TestSetup {
+ private PostProcessListProperty postProcessListProperty;
+ private TinkerGraph tg;
+ private MappedElementCache cache;
+ private Vertex tableV;
+
+ public PostProcessListProperty getPostProcessListProperty() {
+ return postProcessListProperty;
+ }
+
+ public TinkerGraph getGraph() {
+ return tg;
+ }
+
+ public MappedElementCache getCache() {
+ return cache;
+ }
+
+ public Vertex getTable() {
+ return tableV;
+ }
+
+ public TestSetup() throws IOException {
+ postProcessListProperty = new PostProcessListProperty();
+ tg = TinkerGraph.open();
+ cache = new MappedElementCache();
+ addEdge(tg, cache);
+ tableV = fetchTableVertex(tg);
+
+ assertSetup();
+ }
+
+ public void assertSetup() {
+ assertTrue(tableV.property(HIVE_COLUMNS_PROPERTY).isPresent());
+ List list = (List) tableV.property(HIVE_COLUMNS_PROPERTY).value();
+
+ assertEquals(list.size(), 2);
+ assertEquals(list.get(0), col1EdgeId);
+ assertEquals(list.get(1), col2EdgeId);
+ }
+
+ private void assertIncomplete() {
+ assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+
+ Iterator<Edge> edges = tableV.edges(Direction.OUT, getEdgeLabel(HIVE_COLUMNS_PROPERTY));
+ while (edges.hasNext()) {
+ Edge e = edges.next();
+
+ assertFalse(e.property(ATTRIBUTE_INDEX_PROPERTY_KEY).isPresent());
+ }
+ }
+
+ private void assertComplete() {
+ assertPropertyRemoved(HIVE_COLUMNS_PROPERTY, tableV);
+ }
+
+ private void assertPropertyRemoved(String property, Vertex tableV) {
+ assertFalse(tableV.property(property).isPresent());
+ }
+
+ public String getEdgeLabel(String property ) {
+ return Constants.INTERNAL_PROPERTY_KEY_PREFIX + property;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-2-legacy.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/col-2-legacy.json b/graphdb/janus/src/test/resources/col-2-legacy.json
new file mode 100644
index 0000000..f755462
--- /dev/null
+++ b/graphdb/janus/src/test/resources/col-2-legacy.json
@@ -0,0 +1,73 @@
+{
+ "Asset.name": {
+ "type": "string",
+ "value": "col2"
+ },
+ "hive_column.type": {
+ "type": "string",
+ "value": "string"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column owner anonymous qualifiedName stocks.test_table_view.col2@cl1 name col2 position 1 type string table "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.test_table_view.col2@cl1"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "9cef2494-766c-4671-96a8-828dce677e7e"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "Asset"
+ }, {
+ "type": "string",
+ "value": "DataSet"
+ }, {
+ "type": "string",
+ "value": "Referenceable"
+ }]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693838471
+ },
+ "Asset.owner": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "hive_column.position": {
+ "type": "integer",
+ "value": 1
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": 114816,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/col-3-legacy.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/col-3-legacy.json b/graphdb/janus/src/test/resources/col-3-legacy.json
new file mode 100644
index 0000000..b6e8c20
--- /dev/null
+++ b/graphdb/janus/src/test/resources/col-3-legacy.json
@@ -0,0 +1,86 @@
+{
+ "Asset.name": {
+ "type": "string",
+ "value": "open"
+ },
+ "hive_column.type": {
+ "type": "string",
+ "value": "string"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column owner anonymous qualifiedName stocks.stocks_daily.open@cl1 name open position 1 type string table Tag1 "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.stocks_daily.open@cl1"
+ },
+ "__traitNames": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "Tag1"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "8231a95f-d062-4685-81aa-0b62401bc796"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "Asset"
+ },
+ {
+ "type": "string",
+ "value": "DataSet"
+ },
+ {
+ "type": "string",
+ "value": "Referenceable"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693815850
+ },
+ "Asset.owner": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "hive_column.position": {
+ "type": "integer",
+ "value": 1
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693815850
+ },
+ "_id": 16752,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col.json b/graphdb/janus/src/test/resources/edge-legacy-col.json
new file mode 100644
index 0000000..1e82f04
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "816u-35tc-ao0l-47so",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 98336,
+ "_label": "__hive_table.columns"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col2.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col2.json b/graphdb/janus/src/test/resources/edge-legacy-col2.json
new file mode 100644
index 0000000..75e0ac4
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col2.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "82rq-35tc-ao0l-2glc",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 114816,
+ "_label": "__hive_table.columns"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col3.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col3.json b/graphdb/janus/src/test/resources/edge-legacy-col3.json
new file mode 100644
index 0000000..6145ce1
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col3.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "816u-35tc-ao0l-47aa",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 98336,
+ "_label": "__hive_table.columnsMap.col3"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-col4.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-col4.json b/graphdb/janus/src/test/resources/edge-legacy-col4.json
new file mode 100644
index 0000000..15cdda7
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-col4.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693835017
+ },
+ "_id": "82rq-35tc-ao0l-2gaa",
+ "_type": "edge",
+ "_outV": 147504,
+ "_inV": 114816,
+ "_label": "__hive_table.columnsMap.col4"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-process.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-process.json b/graphdb/janus/src/test/resources/edge-legacy-process.json
new file mode 100644
index 0000000..9285958
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-process.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "_id": "6jgh-23uw-2uqd-1elc",
+ "_type": "edge",
+ "_outV": 98312,
+ "_inV": 147504,
+ "_label": "__Process.inputs"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/edge-legacy-tag.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/edge-legacy-tag.json b/graphdb/janus/src/test/resources/edge-legacy-tag.json
new file mode 100644
index 0000000..1bc200a
--- /dev/null
+++ b/graphdb/janus/src/test/resources/edge-legacy-tag.json
@@ -0,0 +1,27 @@
+{
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "_id": "2pk3la-cxc-m61h-2pk0og",
+ "_type": "edge",
+ "_outV": 16752,
+ "_inV": 163856752,
+ "_label": "Tag1"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/lineage-v-98312.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/lineage-v-98312.json b/graphdb/janus/src/test/resources/lineage-v-98312.json
new file mode 100644
index 0000000..abdf9cb
--- /dev/null
+++ b/graphdb/janus/src/test/resources/lineage-v-98312.json
@@ -0,0 +1,79 @@
+{
+ "hive_column_lineage.depenendencyType": {
+ "type": "string",
+ "value": "SIMPLE"
+ },
+ "Asset.name": {
+ "type": "string",
+ "value": "create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1"
+ },
+ "Process.inputs": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "6jgh-23uw-2uqd-1elc"
+ }]
+ },
+ "Process.outputs": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "6ept-23uw-2wb9-47so"
+ }]
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column_lineage outputs qualifiedName stocks.test_table_view@cl1:1522693834000:col1 inputs query name create view test_table_view partitioned on (col4) as select col1, col2, col4 from test_table:col1 depenendencyType SIMPLE "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "stocks.test_table_view@cl1:1522693834000:col1"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "9336b7a4-9cc0-4ef0-8dc1-01fce2def6a5"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "Asset"
+ }, {
+ "type": "string",
+ "value": "Process"
+ }, {
+ "type": "string",
+ "value": "Referenceable"
+ }]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column_lineage"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522693837285
+ },
+ "_id": 98312,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/table-v-147504.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/table-v-147504.json b/graphdb/janus/src/test/resources/table-v-147504.json
index 898dce5..dc5d788 100644
--- a/graphdb/janus/src/test/resources/table-v-147504.json
+++ b/graphdb/janus/src/test/resources/table-v-147504.json
@@ -33,10 +33,12 @@
},
"hive_table.parameters": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "transient_lastDdlTime"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "transient_lastDdlTime"
+ }
+ ]
},
"hive_table.retention": {
"type": "integer",
@@ -44,10 +46,12 @@
},
"hive_table.partitionKeys": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "8dty-35tc-amfp-23xs"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "8dty-35tc-amfp-23xs"
+ }
+ ]
},
"__guid": {
"type": "string",
@@ -63,16 +67,20 @@
},
"__superTypeNames": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "Asset"
- }, {
- "type": "string",
- "value": "DataSet"
- }, {
- "type": "string",
- "value": "Referenceable"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "Asset"
+ },
+ {
+ "type": "string",
+ "value": "DataSet"
+ },
+ {
+ "type": "string",
+ "value": "Referenceable"
+ }
+ ]
},
"hive_table.viewExpandedText": {
"type": "string",
@@ -104,13 +112,37 @@
},
"hive_table.columns": {
"type": "list",
- "value": [{
- "type": "string",
- "value": "816u-35tc-ao0l-47so"
- }, {
- "type": "string",
- "value": "82rq-35tc-ao0l-2glc"
- }]
+ "value": [
+ {
+ "type": "string",
+ "value": "816u-35tc-ao0l-47so"
+ },
+ {
+ "type": "string",
+ "value": "82rq-35tc-ao0l-2glc"
+ }
+ ]
+ },
+ "hive_table.columnsMap.col3": {
+ "type": "string",
+ "value": "816u-35tc-ao0l-47aa"
+ },
+ "hive_table.columnsMap.col4": {
+ "type": "string",
+ "value": "82rq-35tc-ao0l-2gaa"
+ },
+ "hive_table.columnsMap": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "col3"
+ },
+ {
+ "type": "string",
+ "value": "col4"
+ }
+ ]
},
"__timestamp": {
"type": "long",
@@ -119,3 +151,4 @@
"_id": 147504,
"_type": "vertex"
}
+
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/test/resources/tag-163856752.json
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/test/resources/tag-163856752.json b/graphdb/janus/src/test/resources/tag-163856752.json
new file mode 100644
index 0000000..7049819
--- /dev/null
+++ b/graphdb/janus/src/test/resources/tag-163856752.json
@@ -0,0 +1,35 @@
+{
+ "__superTypeNames": {
+ "type": "list",
+ "value": [{
+ "type": "string",
+ "value": "[]"
+ }]
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "Tag1"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522694252176
+ },
+ "_id": 163856752,
+ "_type": "vertex"
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
index b05754f..025f89a 100644
--- a/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
+++ b/intg/src/main/java/org/apache/atlas/store/AtlasTypeDefStore.java
@@ -21,16 +21,12 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.SearchFilter;
import org.apache.atlas.model.typedef.AtlasBaseTypeDef;
import org.apache.atlas.model.typedef.AtlasClassificationDef;
-import org.apache.atlas.model.typedef.AtlasClassificationDef.AtlasClassificationDefs;
import org.apache.atlas.model.typedef.AtlasEntityDef;
import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasRelationshipDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasTypesDef;
-import java.io.InputStream;
-import java.util.List;
-import java.util.Map;
/**
* Interface to persistence store of TypeDef
@@ -107,6 +103,4 @@ public interface AtlasTypeDefStore {
AtlasBaseTypeDef getByName(String name) throws AtlasBaseException;
AtlasBaseTypeDef getByGuid(String guid) throws AtlasBaseException;
-
- void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
----------------------------------------------------------------------
diff --git a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
index 886ce77..4e9a651 100755
--- a/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
+++ b/intg/src/test/java/org/apache/atlas/TestUtilsV2.java
@@ -18,7 +18,6 @@
package org.apache.atlas;
-import com.sun.org.apache.xpath.internal.operations.Bool;
import org.apache.atlas.model.instance.AtlasEntity;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo;
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
index 22cd552..081376f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationService.java
@@ -18,11 +18,11 @@
package org.apache.atlas.repository.migration;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.impexp.AtlasImportResult;
import org.apache.atlas.model.typedef.AtlasTypesDef;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.impexp.ImportTypeDefProcessor;
import org.apache.atlas.repository.store.bootstrap.AtlasTypeDefStoreInitializer;
import org.apache.atlas.store.AtlasTypeDefStore;
@@ -56,11 +56,11 @@ public class DataMigrationService implements Service {
private final Thread thread;
@Inject
- public DataMigrationService(AtlasTypeDefStore typeDefStore, Configuration configuration,
+ public DataMigrationService(AtlasGraph graph, AtlasTypeDefStore typeDefStore, Configuration configuration,
GraphBackedSearchIndexer indexer, AtlasTypeDefStoreInitializer storeInitializer,
AtlasTypeRegistry typeRegistry) {
this.configuration = configuration;
- this.thread = new Thread(new FileImporter(typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
+ this.thread = new Thread(new FileImporter(graph, typeDefStore, typeRegistry, storeInitializer, getFileName(), indexer));
}
@Override
@@ -83,15 +83,17 @@ public class DataMigrationService implements Service {
}
public static class FileImporter implements Runnable {
+ private final AtlasGraph graph;
private final AtlasTypeDefStore typeDefStore;
private final String importDirectory;
private final GraphBackedSearchIndexer indexer;
private final AtlasTypeRegistry typeRegistry;
private final AtlasTypeDefStoreInitializer storeInitializer;
- public FileImporter(AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
+ public FileImporter(AtlasGraph graph, AtlasTypeDefStore typeDefStore, AtlasTypeRegistry typeRegistry,
AtlasTypeDefStoreInitializer storeInitializer,
String directoryName, GraphBackedSearchIndexer indexer) {
+ this.graph = graph;
this.typeDefStore = typeDefStore;
this.typeRegistry = typeRegistry;
this.storeInitializer = storeInitializer;
@@ -99,7 +101,16 @@ public class DataMigrationService implements Service {
this.indexer = indexer;
}
- public void performImport() throws AtlasBaseException {
+ @Override
+ public void run() {
+ try {
+ performImport();
+ } catch (AtlasBaseException e) {
+ LOG.error("Data Migration:", e);
+ }
+ }
+
+ private void performImport() throws AtlasBaseException {
try {
if(!performAccessChecks(importDirectory)) {
return;
@@ -109,7 +120,7 @@ public class DataMigrationService implements Service {
FileInputStream fs = new FileInputStream(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_DATA_NAME));
- typeDefStore.loadLegacyData(RelationshipCacheGenerator.get(typeRegistry), fs);
+ graph.importLegacyGraphSON(typeRegistry, fs);
} catch (Exception ex) {
LOG.error("Import failed!", ex);
throw new AtlasBaseException(ex);
@@ -117,11 +128,13 @@ public class DataMigrationService implements Service {
}
private boolean performAccessChecks(String path) {
- boolean ret = false;
+ final boolean ret;
+
if(StringUtils.isEmpty(path)) {
ret = false;
} else {
File f = new File(path);
+
ret = f.exists() && f.isDirectory() && f.canRead();
}
@@ -137,17 +150,19 @@ public class DataMigrationService implements Service {
private void performInit() throws AtlasBaseException, AtlasException {
indexer.instanceIsActive();
storeInitializer.instanceIsActive();
+
processIncomingTypesDef(getFileFromImportDirectory(importDirectory, ATLAS_MIGRATION_TYPESDEF_NAME));
}
- @VisibleForTesting
- void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
+ private void processIncomingTypesDef(File typesDefFile) throws AtlasBaseException {
try {
- AtlasImportResult result = new AtlasImportResult();
- String jsonStr = FileUtils.readFileToString(typesDefFile);
- AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
+ AtlasImportResult result = new AtlasImportResult();
+ String jsonStr = FileUtils.readFileToString(typesDefFile);
+ AtlasTypesDef typesDef = AtlasType.fromJson(jsonStr, AtlasTypesDef.class);
ImportTypeDefProcessor processor = new ImportTypeDefProcessor(typeDefStore, typeRegistry);
+
processor.processTypes(typesDef, result);
+
LOG.info(" types migrated: {}", result.getMetrics());
} catch (IOException e) {
LOG.error("processIncomingTypesDef: Could not process file: {}! Imported data may not be usable.", typesDefFile.getName());
@@ -157,14 +172,5 @@ public class DataMigrationService implements Service {
private File getFileFromImportDirectory(String importDirectory, String fileName) {
return Paths.get(importDirectory, fileName).toFile();
}
-
- @Override
- public void run() {
- try {
- performImport();
- } catch (AtlasBaseException e) {
- LOG.error("Data Migration:", e);
- }
- }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java b/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
deleted file mode 100644
index 2550daf..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/migration/RelationshipCacheGenerator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.migration;
-
-import org.apache.atlas.model.typedef.AtlasRelationshipDef;
-import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasEntityType;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.commons.lang.StringUtils;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-public class RelationshipCacheGenerator {
-
- public static Map<String, String> get(AtlasTypeRegistry typeRegistry) {
- Map<String, String> ret = new HashMap<>();
- Collection<AtlasRelationshipType> relationshipTypes = typeRegistry.getAllRelationshipTypes();
-
- for (AtlasRelationshipType rt : relationshipTypes) {
- AtlasRelationshipDef rd = rt.getRelationshipDef();
- String relTypeName = rt.getTypeName();
-
- add(ret, getKey(rd.getEndDef1(), rt.getEnd1Type()), relTypeName);
- add(ret, getKey(rd.getEndDef2(), rt.getEnd2Type()), relTypeName);
- }
-
- return ret;
- }
-
- private static String getKey(AtlasRelationshipEndDef ed, AtlasEntityType rt) {
- return getKey(ed.getIsLegacyAttribute(), rt.getTypeName(), ed.getName());
- }
-
- private static String getKey(String lhs, String rhs) {
- return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs);
- }
-
- private static String getKey(boolean isLegacy, String typeName, String name) {
- if(!isLegacy) {
- return "";
- }
-
- return getKey(typeName, name);
- }
-
- private static void add(Map<String, String> map, String key, String value) {
- if(StringUtils.isEmpty(key) || map.containsKey(key)) {
- return;
- }
-
- map.put(key, value);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
index 3c84e3c..67631bf 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasTypeDefGraphStoreV1.java
@@ -24,7 +24,6 @@ import static org.apache.atlas.repository.Constants.TYPE_CATEGORY_PROPERTY_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1.VERTEX_TYPE;
-import java.io.InputStream;
import java.util.Date;
import java.util.HashSet;
import java.util.Iterator;
@@ -49,7 +48,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
-import javax.annotation.PostConstruct;
import javax.inject.Inject;
import javax.inject.Singleton;
@@ -110,11 +108,6 @@ public class AtlasTypeDefGraphStoreV1 extends AtlasTypeDefGraphStore {
LOG.info("<== AtlasTypeDefGraphStoreV1.init()");
}
- @Override
- public void loadLegacyData(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
- getAtlasGraph().loadLegacyGraphSON(relationshipCache, fs);
- }
-
AtlasGraph getAtlasGraph() { return atlasGraph; }
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
new file mode 100644
index 0000000..9451fa2
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/ComplexAttributesTest.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
+import org.apache.atlas.repository.graphdb.AtlasGraph;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class ComplexAttributesTest extends MigrationBaseAsserts {
+
+ @Inject
+ public ComplexAttributesTest(AtlasGraph graph) {
+ super(graph);
+ }
+
+ @Test
+ public void verify() throws IOException, AtlasBaseException {
+ String STRUCT_TYPE = "struct_type";
+ String ENTITY_TYPE = "entity_type";
+ String ENTITY_WITH_COMPLEX_COLL_TYPE = "entity_with_complex_collection_attr";
+
+ final int EXPECTED_TOTAL_COUNT = 214;
+ final int EXPECTED_ENTITY_TYPE_COUNT = 16;
+ final int EXPECTED_STRUCT_TYPE_COUNT = 3;
+ final int EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT = 1;
+
+ runFileImporter("complex-attr_db");
+
+ assertTypeCountNameGuid(STRUCT_TYPE, EXPECTED_STRUCT_TYPE_COUNT,"", "");
+ assertTypeCountNameGuid(ENTITY_TYPE, EXPECTED_ENTITY_TYPE_COUNT, "", "");
+ assertTypeCountNameGuid(ENTITY_WITH_COMPLEX_COLL_TYPE, EXPECTED_ENTITY_WITH_COMPLEX_COLL_TYPE_COUNT, "", "");
+
+ assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),1, "__entity_with_complex_collection_attr.listOfEntities");
+ assertEdgesWithLabel(getVertex(ENTITY_WITH_COMPLEX_COLL_TYPE, "").getEdges(AtlasEdgeDirection.OUT).iterator(),9, "__entity_with_complex_collection_attr.mapOfStructs");
+
+ assertMigrationStatus(EXPECTED_TOTAL_COUNT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
index ac0b79d..7b23e59 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveParititionTest.java
@@ -21,17 +21,13 @@ package org.apache.atlas.repository.migration;
import com.google.inject.Inject;
import org.apache.atlas.TestModules;
import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import java.io.IOException;
-import java.util.List;
-import static org.testng.Assert.assertEquals;
@Guice(modules = TestModules.TestOnlyModule.class)
public class HiveParititionTest extends MigrationBaseAsserts {
@@ -41,7 +37,7 @@ public class HiveParititionTest extends MigrationBaseAsserts {
super(graph);
}
- @Test(enabled = false)
+ @Test
public void fileImporterTest() throws IOException, AtlasBaseException {
final int EXPECTED_TOTAL_COUNT = 141;
final int EXPECTED_DB_COUNT = 1;
@@ -50,8 +46,6 @@ public class HiveParititionTest extends MigrationBaseAsserts {
runFileImporter("parts_db");
- assertPartitionKeyProperty(getVertex("hive_table", "t1"), 1);
- assertPartitionKeyProperty(getVertex("hive_table", "tv1"), 1);
assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
assertTypeCountNameGuid("hive_db", 1, "parts_db", "ae30d78b-51b4-42ab-9436-8d60c8f68b95");
@@ -62,9 +56,4 @@ public class HiveParititionTest extends MigrationBaseAsserts {
assertMigrationStatus(EXPECTED_TOTAL_COUNT);
}
-
- private void assertPartitionKeyProperty(AtlasVertex vertex, int expectedCount) {
- List<String> keys = GraphHelper.getListProperty(vertex, "hive_table.partitionKeys");
- assertEquals(keys.size(), expectedCount);
- }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
index ffbf320..721bb31 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/HiveStocksTest.java
@@ -35,7 +35,7 @@ public class HiveStocksTest extends MigrationBaseAsserts {
super(graph);
}
- @Test(enabled = false)
+ @Test
public void migrateStocks() throws AtlasBaseException, IOException {
final int EXPECTED_TOTAL_COUNT = 188;
final int EXPECTED_DB_COUNT = 1;
@@ -47,7 +47,6 @@ public class HiveStocksTest extends MigrationBaseAsserts {
assertHiveVertices(EXPECTED_DB_COUNT, EXPECTED_TABLE_COUNT, EXPECTED_COLUMN_COUNT);
assertTypeCountNameGuid("hive_db", 1, "stocks", "4e13b36b-9c54-4616-9001-1058221165d0");
assertTypeCountNameGuid("hive_table", 1, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774");
- assertTypeAttribute("hive_table", 7, "stocks_daily", "5cfc2540-9947-40e0-8905-367e07481774", "hive_table.columns");
assertTypeCountNameGuid("hive_column", 1, "high", "d72ce4fb-6f17-4e68-aa85-967366c9e891");
assertTypeCountNameGuid("hive_column", 1, "open", "788ba8fe-b7d8-41ba-84ef-c929732924ec");
assertTypeCountNameGuid("hive_column", 1, "dt", "643a0a71-0d97-477d-a43b-7ca433f85160");
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
index ec6e64a..4b02b7f 100644
--- a/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/MigrationBaseAsserts.java
@@ -34,7 +34,6 @@ import org.testng.annotations.AfterClass;
import java.io.IOException;
import java.util.Iterator;
-import java.util.List;
import static org.apache.atlas.graph.GraphSandboxUtil.useLocalSolr;
import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
@@ -48,10 +47,10 @@ public class MigrationBaseAsserts {
private final String R_GUID_PROPERTY_NAME = "_r__guid";
@Inject
- private AtlasTypeDefStore typeDefStore;
+ protected AtlasTypeDefStore typeDefStore;
@Inject
- private AtlasTypeRegistry typeRegistry;
+ protected AtlasTypeRegistry typeRegistry;
@Inject
private AtlasTypeDefStoreInitializer storeInitializer;
@@ -74,7 +73,7 @@ public class MigrationBaseAsserts {
}
}
- private void loadTypesFromJson() throws IOException, AtlasBaseException {
+ protected void loadTypesFromJson() throws IOException, AtlasBaseException {
loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1020-fs_model.json", typeDefStore, typeRegistry);
loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
@@ -83,7 +82,7 @@ public class MigrationBaseAsserts {
protected void runFileImporter(String directoryToImport) throws IOException, AtlasBaseException {
loadTypesFromJson();
String directoryName = TestResourceFileUtils.getDirectory(directoryToImport);
- DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(typeDefStore, typeRegistry,
+ DataMigrationService.FileImporter fi = new DataMigrationService.FileImporter(graph, typeDefStore, typeRegistry,
storeInitializer, directoryName, indexer);
fi.run();
@@ -152,14 +151,19 @@ public class MigrationBaseAsserts {
assertEquals(count, expectedItems, String.format("%s", edgeTypeName));
}
- protected void assertTypeAttribute(String typeName, int expectedSize, String name, String guid, String propertyName) {
- AtlasVertex v = getVertex(typeName, name);
- String guidActual = GraphHelper.getGuid(v);
- List list = (List) GraphHelper.getProperty(v, propertyName);
+ protected void assertEdgesWithLabel(Iterator<AtlasEdge> results, int startIdx, String edgeTypeName) {
+ int count = 0;
+ AtlasEdge e = null;
+ for (Iterator<AtlasEdge> it = results; it.hasNext() && count < startIdx; count++) {
+ e = it.next();
+ }
- assertEquals(guidActual, guid);
- assertNotNull(list);
- assertEquals(list.size(), expectedSize);
+ assertNotNull(GraphHelper.getProperty(e, R_GUID_PROPERTY_NAME));
+ assertNotNull(GraphHelper.getProperty(e, "tagPropagation"));
+
+ if(StringUtils.isNotEmpty(edgeTypeName)) {
+ assertEquals(e.getLabel(), edgeTypeName, edgeTypeName);
+ }
}
protected void assertTypeCountNameGuid(String typeName, int expectedItems, String name, String guid) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
new file mode 100644
index 0000000..362990f
--- /dev/null
+++ b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipCacheGeneratorTest.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.migration;
+
+import com.google.inject.Inject;
+import org.apache.atlas.TestModules;
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.graphdb.janus.migration.RelationshipCacheGenerator;
+import org.apache.atlas.store.AtlasTypeDefStore;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang.StringUtils;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.Guice;
+import org.testng.annotations.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO;
+import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE;
+import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+@Guice(modules = TestModules.TestOnlyModule.class)
+public class RelationshipCacheGeneratorTest {
+
+ @Inject
+ private AtlasTypeDefStore typeDefStore;
+
+ @Inject
+ private AtlasTypeRegistry typeRegistry;
+
+ @BeforeClass
+ public void setup() throws IOException, AtlasBaseException {
+ loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
+ loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
+ }
+
+ @Test
+ public void createLookup() {
+ final String PROCESS_INPUT_KEY = "__Process.inputs";
+ final String PROCESS_OUTPUT_KEY = "__Process.outputs";
+ String ONE_TO_TWO_STR = ONE_TO_TWO.toString();
+ String TWO_TO_ONE_STR = TWO_TO_ONE.toString();
+
+ Map<String, RelationshipCacheGenerator.TypeInfo> cache = RelationshipCacheGenerator.get(typeRegistry);
+ assertEquals(cache.size(), getLegacyAttributeCount() - 1);
+ for (Map.Entry<String, RelationshipCacheGenerator.TypeInfo> entry : cache.entrySet()) {
+ assertTrue(StringUtils.isNotEmpty(entry.getKey()));
+ assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
+ }
+
+ assertEquals(cache.get(PROCESS_INPUT_KEY).getTypeName(), "dataset_process_inputs");
+ assertEquals(cache.get(PROCESS_INPUT_KEY).getPropagateTags(), ONE_TO_TWO_STR);
+
+ assertEquals(cache.get(PROCESS_OUTPUT_KEY).getTypeName(), "process_dataset_outputs");
+ assertEquals(cache.get(PROCESS_OUTPUT_KEY).getPropagateTags(), TWO_TO_ONE_STR);
+ }
+
+ private int getLegacyAttributeCount() {
+ int count = 0;
+ for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) {
+ AtlasRelationshipDef rd = rt.getRelationshipDef();
+ if(rd.getEndDef1().getIsLegacyAttribute()) {
+ count++;
+ }
+
+ if(rd.getEndDef2().getIsLegacyAttribute()) {
+ count++;
+ }
+ }
+
+ return count;
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java b/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
deleted file mode 100644
index 48e5391..0000000
--- a/repository/src/test/java/org/apache/atlas/repository/migration/RelationshipMappingTest.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.migration;
-
-import com.google.inject.Inject;
-import org.apache.atlas.TestModules;
-import org.apache.atlas.exception.AtlasBaseException;
-import org.apache.atlas.model.typedef.AtlasRelationshipDef;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.impexp.ZipFileResourceTestUtils;
-import org.apache.atlas.repository.store.graph.AtlasEntityStore;
-import org.apache.atlas.store.AtlasTypeDefStore;
-import org.apache.atlas.type.AtlasRelationshipType;
-import org.apache.atlas.type.AtlasTypeRegistry;
-import org.apache.commons.lang.StringUtils;
-import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerGraph;
-import org.jcodings.util.Hash;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Guice;
-import org.testng.annotations.Test;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Map;
-
-import static org.apache.atlas.repository.impexp.ZipFileResourceTestUtils.loadModelFromJson;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-
-@Guice(modules = TestModules.TestOnlyModule.class)
-public class RelationshipMappingTest {
-
- @Inject
- private AtlasTypeDefStore typeDefStore;
-
- @Inject
- private AtlasTypeRegistry typeRegistry;
-
- @BeforeClass
- public void setup() throws IOException, AtlasBaseException {
- loadModelFromJson("0000-Area0/0010-base_model.json", typeDefStore, typeRegistry);
- loadModelFromJson("1000-Hadoop/1030-hive_model.json", typeDefStore, typeRegistry);
- }
-
- @Test
- public void createLookup() {
- Map<String, String> cache = RelationshipCacheGenerator.get(typeRegistry);
- assertEquals(cache.size(), getLegacyAttributeCount() - 1);
- for (Map.Entry<String, String> entry : cache.entrySet()) {
- assertTrue(StringUtils.isNotEmpty(entry.getKey()));
- assertTrue(entry.getKey().startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX), entry.getKey());
- }
- }
-
- private int getLegacyAttributeCount() {
- int count = 0;
- for (AtlasRelationshipType rt : typeRegistry.getAllRelationshipTypes()) {
- AtlasRelationshipDef rd = rt.getRelationshipDef();
- if(rd.getEndDef1().getIsLegacyAttribute()) {
- count++;
- }
-
- if(rd.getEndDef2().getIsLegacyAttribute()) {
- count++;
- }
- }
-
- return count;
- }
-}
[3/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/resources/complex-attr_db/atlas-migration-typesdef.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/complex-attr_db/atlas-migration-typesdef.json b/repository/src/test/resources/complex-attr_db/atlas-migration-typesdef.json
new file mode 100644
index 0000000..42d02cd
--- /dev/null
+++ b/repository/src/test/resources/complex-attr_db/atlas-migration-typesdef.json
@@ -0,0 +1,2303 @@
+{
+ "enumDefs": [
+ {
+ "category": "ENUM",
+ "guid": "460e6d99-b06c-48d3-8bb2-9c40ca1235d0",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656721280,
+ "updateTime": 1523656721280,
+ "version": 1,
+ "name": "hive_principal_type",
+ "description": "hive_principal_type",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "value": "USER",
+ "ordinal": 1
+ },
+ {
+ "value": "ROLE",
+ "ordinal": 2
+ },
+ {
+ "value": "GROUP",
+ "ordinal": 3
+ }
+ ]
+ },
+ {
+ "category": "ENUM",
+ "guid": "df00ab81-0791-4dbb-bd56-e4183e039111",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656718218,
+ "updateTime": 1523656718218,
+ "version": 1,
+ "name": "file_action",
+ "description": "file_action",
+ "typeVersion": "1.0",
+ "elementDefs": [
+ {
+ "value": "NONE",
+ "ordinal": 0
+ },
+ {
+ "value": "EXECUTE",
+ "ordinal": 1
+ },
+ {
+ "value": "WRITE",
+ "ordinal": 2
+ },
+ {
+ "value": "WRITE_EXECUTE",
+ "ordinal": 3
+ },
+ {
+ "value": "READ",
+ "ordinal": 4
+ },
+ {
+ "value": "READ_EXECUTE",
+ "ordinal": 5
+ },
+ {
+ "value": "READ_WRITE",
+ "ordinal": 6
+ },
+ {
+ "value": "ALL",
+ "ordinal": 7
+ }
+ ]
+ }
+ ],
+ "structDefs": [
+ {
+ "category": "STRUCT",
+ "guid": "bd0ba5b3-525e-4d35-9527-7ffa10b4e5bc",
+ "createdBy": "admin",
+ "updatedBy": "admin",
+ "createTime": 1526494657148,
+ "updateTime": 1526494657148,
+ "version": 1,
+ "name": "struct_type",
+ "description": "struct_type_description",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ]
+ },
+ {
+ "category": "STRUCT",
+ "guid": "30993e57-4d69-46ab-b531-02371eace3d9",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656721304,
+ "updateTime": 1523656721304,
+ "version": 1,
+ "name": "hive_order",
+ "description": "hive_order",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "order",
+ "typeName": "int",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "col",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ]
+ },
+ {
+ "category": "STRUCT",
+ "guid": "db96593c-2453-4795-966f-edaa05e69cde",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722372,
+ "updateTime": 1523656722372,
+ "version": 1,
+ "name": "hive_serde",
+ "description": "hive_serde",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "serializationLib",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ]
+ },
+ {
+ "category": "STRUCT",
+ "guid": "9d847c57-5940-4ff9-a9d1-ee36365b9cb2",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656718322,
+ "updateTime": 1523656718322,
+ "version": 1,
+ "name": "fs_permissions",
+ "description": "fs_permissions",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "group",
+ "typeName": "file_action",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "user",
+ "typeName": "file_action",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "sticky",
+ "typeName": "boolean",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "others",
+ "typeName": "file_action",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ]
+ }
+ ],
+ "classificationDefs": [
+ {
+ "category": "CLASSIFICATION",
+ "guid": "5af3e7e8-68a6-4cf6-b88e-4c619731f3e2",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656711303,
+ "updateTime": 1523656711303,
+ "version": 1,
+ "name": "TaxonomyTerm",
+ "description": "TaxonomyTerm",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "atlas.taxonomy",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [],
+ "subTypes": []
+ }
+ ],
+ "entityDefs": [
+ {
+ "category": "ENTITY",
+ "guid": "861f6201-6bbd-4a94-ac04-6b4982c4f94e",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656725295,
+ "updateTime": 1523656725295,
+ "version": 1,
+ "name": "falcon_feed_replication",
+ "description": "falcon_feed_replication",
+ "typeVersion": "1.0",
+ "attributeDefs": [],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "0a5f7d23-c5bd-4ff0-84df-0c0df226e47e",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656725387,
+ "updateTime": 1523656725387,
+ "version": 1,
+ "name": "falcon_process",
+ "description": "falcon_process",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "runs-on",
+ "typeName": "falcon_cluster",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "pipelines",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "workflow-properties",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "38c97017-cb6c-4f64-9610-6930d04c7645",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656712591,
+ "updateTime": 1523656712591,
+ "version": 1,
+ "name": "DataSet",
+ "description": "DataSet",
+ "typeVersion": "1.0",
+ "attributeDefs": [],
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "subTypes": [
+ "hbase_column",
+ "jms_topic",
+ "falcon_feed",
+ "hbase_table",
+ "hbase_column_family",
+ "kafka_topic",
+ "hive_table",
+ "hive_column",
+ "sqoop_dbdatastore",
+ "hbase_namespace",
+ "fs_path"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "c289abd1-5065-41f3-92dd-265f4297d933",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656725408,
+ "updateTime": 1523656725408,
+ "version": 1,
+ "name": "falcon_feed_creation",
+ "description": "falcon_feed_creation",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "3da46128-6e98-4bce-b3d1-cbc4bf9b9055",
+ "createdBy": "admin",
+ "updatedBy": "admin",
+ "createTime": 1526494658830,
+ "updateTime": 1526494658830,
+ "version": 1,
+ "name": "entity_with_complex_collection_attr",
+ "description": "entity_with_complex_collection_attr_description",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "listOfStructs",
+ "typeName": "array<struct_type>",
+ "isOptional": true,
+ "cardinality": "LIST",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "listOfEntities",
+ "typeName": "array<entity_type>",
+ "isOptional": true,
+ "cardinality": "LIST",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "mapOfStructs",
+ "typeName": "map<string,struct_type>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "mapOfEntities",
+ "typeName": "map<string,entity_type>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ }
+ ],
+ "superTypes": [],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "e273b345-4c6b-4272-8b2c-f2936eb1941b",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656713377,
+ "updateTime": 1523656713377,
+ "version": 1,
+ "name": "Process",
+ "description": "Process",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<DataSet>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<DataSet>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "subTypes": [
+ "falcon_feed_replication",
+ "falcon_process",
+ "falcon_feed_creation",
+ "sqoop_process",
+ "storm_topology",
+ "hive_column_lineage",
+ "hive_process"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "dfab14ca-1978-4157-8a9d-3c4e1d608830",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722469,
+ "updateTime": 1523656731237,
+ "version": 2,
+ "name": "hive_table",
+ "description": "hive_table",
+ "typeVersion": "1.1",
+ "options": {
+ "schemaElementsAttribute": "columns"
+ },
+ "attributeDefs": [
+ {
+ "name": "db",
+ "typeName": "hive_db",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "lastAccessTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "retention",
+ "typeName": "int",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "sd",
+ "typeName": "hive_storagedesc",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "partitionKeys",
+ "typeName": "array<hive_column>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "aliases",
+ "typeName": "array<string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "columns",
+ "typeName": "array<hive_column>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "viewOriginalText",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "viewExpandedText",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "tableType",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "temporary",
+ "typeName": "boolean",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "69331a04-420b-4610-9892-398e18e1fa5f",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656714170,
+ "updateTime": 1523656714170,
+ "version": 1,
+ "name": "__AtlasUserProfile",
+ "description": "__AtlasUserProfile",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "fullName",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "savedSearches",
+ "typeName": "array<__AtlasUserSavedSearch>",
+ "isOptional": true,
+ "cardinality": "LIST",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ }
+ ],
+ "superTypes": [
+ "__internal"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "5e1898f6-4a62-40b2-a9c8-3401d5936b61",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722615,
+ "updateTime": 1523656722615,
+ "version": 1,
+ "name": "hive_db",
+ "description": "hive_db",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "ownerType",
+ "typeName": "hive_principal_type",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "1a219261-1258-492b-a145-a1ed96ac6024",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656724387,
+ "updateTime": 1523656724387,
+ "version": 1,
+ "name": "sqoop_dbdatastore",
+ "description": "sqoop_dbdatastore",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "dbStoreType",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "storeUse",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "storeUri",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "source",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "ff0890ec-c80d-4a35-bd7c-b0145cb714a9",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656726191,
+ "updateTime": 1523656726191,
+ "version": 1,
+ "name": "hbase_namespace",
+ "description": "hbase_namespace",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "a382861a-9dd2-4310-897e-20f8666198b4",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722390,
+ "updateTime": 1523656722390,
+ "version": 1,
+ "name": "hive_process",
+ "description": "hive_process",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "operationType",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "queryText",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "queryPlan",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "queryId",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "recentQueries",
+ "typeName": "array<string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "queryGraph",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "a2456728-4a88-43f6-8d64-d875802db437",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656727979,
+ "updateTime": 1523656727979,
+ "version": 1,
+ "name": "storm_node",
+ "description": "storm_node",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "driverClass",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [],
+ "subTypes": [
+ "storm_bolt",
+ "storm_spout"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "b44eaa9e-3727-4b0c-aad1-c0e59eea4d73",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656726233,
+ "updateTime": 1523656726233,
+ "version": 1,
+ "name": "hbase_column",
+ "description": "hbase_column",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "column_family",
+ "typeName": "hbase_column_family",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "columns"
+ }
+ }
+ ]
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "f4b80d9b-e72c-419f-ab40-9a3256a541b9",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656711776,
+ "updateTime": 1523656711776,
+ "version": 1,
+ "name": "Referenceable",
+ "description": "Referenceable",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "qualifiedName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [],
+ "subTypes": [
+ "hive_storagedesc",
+ "DataSet",
+ "Infrastructure",
+ "Process",
+ "hive_db"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "6303fe7d-466d-4750-aef5-fb9cacbdef55",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656727081,
+ "updateTime": 1523656727081,
+ "version": 1,
+ "name": "jms_topic",
+ "description": "jms_topic",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "e82bc222-1bd2-429b-a73d-6a4b45d1e104",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656726210,
+ "updateTime": 1523656728900,
+ "version": 2,
+ "name": "hbase_table",
+ "description": "hbase_table",
+ "typeVersion": "1.2",
+ "attributeDefs": [
+ {
+ "name": "uri",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "column_families",
+ "typeName": "array<hbase_column_family>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "namespace",
+ "typeName": "hbase_namespace",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "fce68d4b-42d3-4579-bc0d-0465f18788fd",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656725316,
+ "updateTime": 1523656725316,
+ "version": 1,
+ "name": "falcon_feed",
+ "description": "falcon_feed",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "frequency",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "stored-in",
+ "typeName": "falcon_cluster",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "groups",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "86313734-80ae-469f-b4c2-f505e1d68846",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656727932,
+ "updateTime": 1523656727932,
+ "version": 1,
+ "name": "storm_topology",
+ "description": "storm_topology",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "id",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "conf",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "nodes",
+ "typeName": "array<storm_node>",
+ "isOptional": false,
+ "cardinality": "LIST",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "887f7f1f-c16a-44af-a399-b81a30502f52",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656713347,
+ "updateTime": 1523656713347,
+ "version": 1,
+ "name": "Infrastructure",
+ "description": "Infrastructure",
+ "typeVersion": "1.0",
+ "attributeDefs": [],
+ "superTypes": [
+ "Referenceable",
+ "Asset"
+ ],
+ "subTypes": [
+ "falcon_cluster"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "1ac62577-4ec0-4932-8913-67a4f51cfaa7",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656727994,
+ "updateTime": 1523656727994,
+ "version": 1,
+ "name": "storm_spout",
+ "description": "storm_spout",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "isOptional": false,
+ "cardinality": "LIST",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "storm_node"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "55673891-cb91-4169-8e77-65d402943ce2",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656726223,
+ "updateTime": 1523656729584,
+ "version": 2,
+ "name": "hbase_column_family",
+ "description": "hbase_column_family",
+ "typeVersion": "1.1",
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hbase_table",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "column_families"
+ }
+ }
+ ]
+ },
+ {
+ "name": "columns",
+ "typeName": "array<hbase_column>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "ownedRef"
+ }
+ ]
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "4dffd142-fd58-4103-80ef-6a773b25fcb2",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656712492,
+ "updateTime": 1523656712492,
+ "version": 1,
+ "name": "Asset",
+ "description": "Asset",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "description",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "owner",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [],
+ "subTypes": [
+ "DataSet",
+ "Infrastructure",
+ "Process",
+ "hive_db"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "210330a4-4777-4fa4-a3a0-a455e8d2a959",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722634,
+ "updateTime": 1523656731493,
+ "version": 4,
+ "name": "hive_column",
+ "description": "hive_column",
+ "typeVersion": "1.3",
+ "options": {
+ "schemaAttributes": "[\"name\", \"description\", \"owner\", \"type\", \"comment\", \"position\"]"
+ },
+ "attributeDefs": [
+ {
+ "name": "type",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "comment",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "position",
+ "typeName": "int",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "679ffbf8-db6d-4fad-95fb-f0e86561e1a6",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656714878,
+ "updateTime": 1523656714878,
+ "version": 1,
+ "name": "__AtlasUserSavedSearch",
+ "description": "__AtlasUserSavedSearch",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "ownerName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "uniqueName",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "searchType",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "searchParameters",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "__internal"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "35619cb1-5e95-43a0-b51d-b8f228290f1c",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656727066,
+ "updateTime": 1523656727066,
+ "version": 1,
+ "name": "kafka_topic",
+ "description": "kafka_topic",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "topic",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "uri",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "e974af16-6278-4fee-83f5-90603a58ae83",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722572,
+ "updateTime": 1523656722572,
+ "version": 1,
+ "name": "hive_storagedesc",
+ "description": "hive_storagedesc",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "table",
+ "typeName": "hive_table",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false,
+ "constraints": [
+ {
+ "type": "inverseRef",
+ "params": {
+ "attribute": "sd"
+ }
+ }
+ ]
+ },
+ {
+ "name": "location",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "inputFormat",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "outputFormat",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "compressed",
+ "typeName": "boolean",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "numBuckets",
+ "typeName": "int",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "serdeInfo",
+ "typeName": "hive_serde",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "bucketCols",
+ "typeName": "array<string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "sortCols",
+ "typeName": "array<hive_order>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "parameters",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "storedAsSubDirectories",
+ "typeName": "boolean",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Referenceable"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "cc19bd6e-fe24-4499-ac74-ae40ca934eef",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656718524,
+ "updateTime": 1523656718524,
+ "version": 1,
+ "name": "hdfs_path",
+ "description": "hdfs_path",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "clusterName",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "numberOfReplicas",
+ "typeName": "int",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "extendedAttributes",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "fs_path"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "e37409c4-6e2d-4ee7-b87a-f1dc32bb8aee",
+ "createdBy": "admin",
+ "updatedBy": "admin",
+ "createTime": 1526494658119,
+ "updateTime": 1526494658119,
+ "version": 1,
+ "name": "entity_type",
+ "description": "entity_type_description",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "name",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": true,
+ "isIndexable": true
+ },
+ {
+ "name": "isReplicated",
+ "typeName": "boolean",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "3eb69f70-1569-41a9-b827-c16a77946b41",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656724359,
+ "updateTime": 1523656724359,
+ "version": 1,
+ "name": "sqoop_process",
+ "description": "sqoop_process",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "operation",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "commandlineOpts",
+ "typeName": "map<string,string>",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "startTime",
+ "typeName": "date",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "endTime",
+ "typeName": "date",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "userName",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "8d0bf90b-0a3c-4e66-ba32-5349732c0967",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656722705,
+ "updateTime": 1523656722705,
+ "version": 1,
+ "name": "hive_column_lineage",
+ "description": "hive_column_lineage",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "query",
+ "typeName": "hive_process",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "depenendencyType",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "expression",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Process"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "25e22509-b6fd-42ed-b5bc-8e1726f37103",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656711806,
+ "updateTime": 1523656711806,
+ "version": 1,
+ "name": "__internal",
+ "description": "__internal",
+ "typeVersion": "1.0",
+ "attributeDefs": [],
+ "superTypes": [],
+ "subTypes": [
+ "__AtlasUserSavedSearch",
+ "__AtlasUserProfile"
+ ]
+ },
+ {
+ "category": "ENTITY",
+ "guid": "d1e4da7c-83ea-48f1-b098-1f3201c4d58e",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656728002,
+ "updateTime": 1523656728002,
+ "version": 1,
+ "name": "storm_bolt",
+ "description": "storm_bolt",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "inputs",
+ "typeName": "array<string>",
+ "isOptional": false,
+ "cardinality": "LIST",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 2147483647,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "outputs",
+ "typeName": "array<string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "storm_node"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "f252485f-992a-4588-9b54-f96cf98ef3d2",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656725304,
+ "updateTime": 1523656725304,
+ "version": 1,
+ "name": "falcon_cluster",
+ "description": "falcon_cluster",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "colo",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "tags",
+ "typeName": "map<string,string>",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ }
+ ],
+ "superTypes": [
+ "Infrastructure"
+ ],
+ "subTypes": []
+ },
+ {
+ "category": "ENTITY",
+ "guid": "c24bf9da-6c38-4f69-bf72-9017a7a32707",
+ "createdBy": "root",
+ "updatedBy": "root",
+ "createTime": 1523656718424,
+ "updateTime": 1523656718424,
+ "version": 1,
+ "name": "fs_path",
+ "description": "fs_path",
+ "typeVersion": "1.0",
+ "attributeDefs": [
+ {
+ "name": "path",
+ "typeName": "string",
+ "isOptional": false,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 1,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "createTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "modifiedTime",
+ "typeName": "date",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "isFile",
+ "typeName": "boolean",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "isSymlink",
+ "typeName": "boolean",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "fileSize",
+ "typeName": "long",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": false
+ },
+ {
+ "name": "group",
+ "typeName": "string",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ },
+ {
+ "name": "posixPermissions",
+ "typeName": "fs_permissions",
+ "isOptional": true,
+ "cardinality": "SINGLE",
+ "valuesMinCount": 0,
+ "valuesMaxCount": 1,
+ "isUnique": false,
+ "isIndexable": true
+ }
+ ],
+ "superTypes": [
+ "DataSet"
+ ],
+ "subTypes": [
+ "hdfs_path"
+ ]
+ }
+ ]
+}
\ No newline at end of file
[2/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/resources/parts_db/atlas-migration-data.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/parts_db/atlas-migration-data.json b/repository/src/test/resources/parts_db/atlas-migration-data.json
index 1414ea1..1fa277e 100644
--- a/repository/src/test/resources/parts_db/atlas-migration-data.json
+++ b/repository/src/test/resources/parts_db/atlas-migration-data.json
@@ -1 +1,3989 @@
-{"mode":"EXTENDED","vertices":[{"__type.name":{"type":"string","value":"hive_order"},"__modifiedBy":{"type":"string","value":"root"},"__type.hive_order.order":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"order\",\"isUnique\":false}"},"__type.options":{"type":"string","value":"null"},"__guid":{"type":"string","value":"2acd11e7-9cbd-4d47-9ded-831dabf80206"},"__version":{"type":"long","value":1},"__type.hive_order":{"type":"list","value":[{"type":"string","value":"order"},{"type":"string","value":"col"}]},"__type.hive_order.col":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"col\",\"isUnique\":false}"},"__type.category":{"type":"string","value":"STRUCT"},"__t
ype.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695099005},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"hive_order"},"__timestamp":{"type":"long","value":1522695099005},"_id":16448,"_type":"vertex"},{"__type.hive_storagedesc.outputFormat":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"outputFormat\",\"isUnique\":false}"},"__type.hive_storagedesc.bucketCols":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"bucketCols\",\"isUnique\":false}"},"__type.name":{"type":"string","value":"hive_storagedesc"},"__type.
hive_storagedesc.parameters":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"null"},"__type.hive_storagedesc.numBuckets":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"numBuckets\",\"isUnique\":false}"},"__type.hive_storagedesc.sortCols":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<hive_order>\",\"name\":\"sortCols\",\"isUnique\":false}"},"__guid":{"type":"string","value"
:"8b3f3741-3ba9-49f1-b545-43f12fa6fe29"},"__type.hive_storagedesc":{"type":"list","value":[{"type":"string","value":"table"},{"type":"string","value":"location"},{"type":"string","value":"inputFormat"},{"type":"string","value":"outputFormat"},{"type":"string","value":"compressed"},{"type":"string","value":"numBuckets"},{"type":"string","value":"serdeInfo"},{"type":"string","value":"bucketCols"},{"type":"string","value":"sortCols"},{"type":"string","value":"parameters"},{"type":"string","value":"storedAsSubDirectories"}]},"__version":{"type":"long","value":1},"__type.hive_storagedesc.inputFormat":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"inputFormat\",\"isUnique\":false}"},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"roo
t"},"__type.hive_storagedesc.serdeInfo":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_serde\",\"name\":\"serdeInfo\",\"isUnique\":false}"},"__type.hive_storagedesc.storedAsSubDirectories":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"storedAsSubDirectories\",\"isUnique\":false}"},"__modificationTimestamp":{"type":"long","value":1522695099132},"__type":{"type":"string","value":"typeSystem"},"__type.hive_storagedesc.location":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"location\",\"isUnique\":false}"},
"__type.hive_storagedesc.compressed":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"compressed\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"hive_storagedesc"},"__type.hive_storagedesc.table":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":\"sd\",\"dataType\":\"hive_table\",\"name\":\"table\",\"isUnique\":false}"},"__timestamp":{"type":"long","value":1522695099132},"_id":32832,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_db"},"__type.hive_db.parameters":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<str
ing,string>\",\"name\":\"parameters\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.hive_db":{"type":"list","value":[{"type":"string","value":"clusterName"},{"type":"string","value":"location"},{"type":"string","value":"parameters"},{"type":"string","value":"ownerType"}]},"__type.options":{"type":"string","value":"null"},"__guid":{"type":"string","value":"14d28b0c-30b0-42e4-addd-52501ea461ed"},"__version":{"type":"long","value":1},"__type.hive_db.clusterName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__type.hive_db.location":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":fals
e,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"location\",\"isUnique\":false}"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695099163},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"hive_db"},"__timestamp":{"type":"long","value":1522695099163},"__type.hive_db.ownerType":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_principal_type\",\"name\":\"ownerType\",\"isUnique\":false}"},"_id":49216,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col4"},"hive_column.type":{"type":"string","value":"string"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.
t1.col4@cl1 name col4 position 0 type string table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1.col4@cl1"},"__guid":{"type":"string","value":"aae38dd6-5632-48a4-9e41-cea65323e47b"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":0},"__timestamp":{"type":"long","value":1522695179454},"_id":65600,"_type":"vertex"},{"__type.name":{"type":"string","value":"Asset"},"__modifiedBy":{"type":"string","value":"root"},"__type.Asset.name":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\"
:true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":false}"},"__type.Asset.owner":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"owner\",\"isUnique\":false}"},"__type.options":{"type":"string","value":"null"},"__guid":{"type":"string","value":"80dfb85b-bbde-4e1b-a194-cc69519123bc"},"__type.Asset":{"type":"list","value":[{"type":"string","value":"name"},{"type":"string","value":"description"},{"type":"string","value":"owner"}]},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695094647},"__type":{"type":"string","value":"typeSystem"},"__type.Asset.description":{"type":"strin
g","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"description\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"Asset"},"__timestamp":{"type":"long","value":1522695094647},"_id":16520,"_type":"vertex"},{"__type.hive_process.operationType":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"operationType\",\"isUnique\":false}"},"__type.name":{"type":"string","value":"hive_process"},"__type.hive_process.userName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"userName\",\"isUnique\":false}"},"__m
odifiedBy":{"type":"string","value":"root"},"__type.hive_process.queryId":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryId\",\"isUnique\":false}"},"__type.hive_process.queryPlan":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryPlan\",\"isUnique\":false}"},"__type.options":{"type":"string","value":"null"},"__type.hive_process.recentQueries":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"recentQueries\",\"isUnique\":false}"},"__guid":{"type":"string","value":"fb115e8e-818c-411
b-9b93-4caf289f5413"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__type.hive_process":{"type":"list","value":[{"type":"string","value":"startTime"},{"type":"string","value":"endTime"},{"type":"string","value":"userName"},{"type":"string","value":"operationType"},{"type":"string","value":"queryText"},{"type":"string","value":"queryPlan"},{"type":"string","value":"queryId"},{"type":"string","value":"recentQueries"},{"type":"string","value":"clusterName"},{"type":"string","value":"queryGraph"}]},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__type.hive_process.clusterName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"},"__modificationTimestamp":{"type":"long","value":1522695099034},"__type
":{"type":"string","value":"typeSystem"},"__type.hive_process.endTime":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"endTime\",\"isUnique\":false}"},"__type.hive_process.queryText":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryText\",\"isUnique\":false}"},"__type.hive_process.queryGraph":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryGraph\",\"isUnique\":false}"},"__type.hive_process.startTime":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUni
que\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"startTime\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"hive_process"},"__timestamp":{"type":"long","value":1522695099034},"_id":32904,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_column"},"__type.hive_column.table":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_table\",\"name\":\"table\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"{\"schemaAttributes\":\"[\\\"name\\\", \\\"description\\\", \\\"owner\\\", \\\"type\\\", \\\"comment\\\", \\\"position\\\"]\"}"},"__type.hive_column":{"type":"list","value":[{"type":"string","value":"type"},{"type":"string","value":"comment"},{"type":"string","value":"posit
ion"},{"type":"string","value":"table"}]},"__type.hive_column.type":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"type\",\"isUnique\":false}"},"__type.hive_column.comment":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"comment\",\"isUnique\":false}"},"__guid":{"type":"string","value":"2e7ffda9-b958-4353-9183-22741089ea6d"},"__version":{"type":"long","value":4},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.3"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695100619},"__type":{"type":"string","value":"typeSystem"},"__type.hive_column.position":{"type":"s
tring","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"position\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"hive_column"},"__timestamp":{"type":"long","value":1522695099177},"_id":49288,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col1"},"hive_column.type":{"type":"string","value":"int"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.tv1.col1@cl1 name col1 position 0 type int table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1.col1@cl1"},"__guid":{"type":"string","value":"fd4ef7cb-84ed-4068-b56e-efabcc83ae7c"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"s
tring","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":0},"__timestamp":{"type":"long","value":1522695183992},"_id":65672,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col2"},"hive_column.type":{"type":"string","value":"string"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.tv1.col2@cl1 name col2 position 1 type string table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1.col2@cl1"},"__guid":{"type":"string","value":"33fc7cdf-fcee-461d-aa8f-c6b8f3ff0bb4"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list"
,"value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":1},"__timestamp":{"type":"long","value":1522695183992},"_id":82056,"_type":"vertex"},{"hive_storagedesc.inputFormat":{"type":"string","value":"org.apache.hadoop.mapred.SequenceFileInputFormat"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_storagedesc qualifiedName parts_db.tv1@cl1_storage storedAsSubDirectories false compressed false inputFormat org.apache.hadoop.mapred.SequenceFileInputFormat outputFormat org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat table serdeInfo serializationLib null
name null parameters null numBuckets -1 "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1@cl1_storage"},"__guid":{"type":"string","value":"02b6baa9-02c0-4ef6-a82c-a47b28bab91a"},"hive_storagedesc.storedAsSubDirectories":{"type":"boolean","value":false},"__version":{"type":"integer","value":0},"hive_storagedesc.outputFormat":{"type":"string","value":"org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat"},"hive_storagedesc.parameters":{"type":"list","value":[]},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_storagedesc"},"__modificationTimestamp":{"type":"long","value":1522695186970},"hive_storagedesc.compressed":{"type":"boolean","value":false},"hive_storagedesc.numBuckets":{"type":"integer","value":-1},"__timestamp":{"type":"long","value":1522695183992},"_id":98440,"_type":"vertex"},{"hive_serde.parameters":{"type":"list",
"value":[]},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_serde"},"__modificationTimestamp":{"type":"long","value":1522695186970},"__timestamp":{"type":"long","value":1522695183992},"_id":114824,"_type":"vertex"},{"__type.name":{"type":"string","value":"DataSet"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__modifiedBy":{"type":"string","value":"root"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695095354},"__type":{"type":"string","value":"typeSystem"},"__type.options":{"type":"string","value":"null"},"__type.description":{"type":"string","value":"DataSet"},"__guid":{"type":"string","value":"8eb418e7-6f18-44eb-a0d0-d762e753b071"},"__timestamp":{"type":"long","value":1522695095354},"_id":
16568,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col2"},"hive_column.type":{"type":"string","value":"string"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.t1.col2@cl1 name col2 position 1 type string table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1.col2@cl1"},"__guid":{"type":"string","value":"4a56254e-eb4b-4d8c-bbdf-bdb7a81a4b18"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":1},"__tim
estamp":{"type":"long","value":1522695179454},"_id":32952,"_type":"vertex"},{"hive_storagedesc.location":{"type":"string","value":"hdfs://localhost.localdomain:8020/apps/hive/warehouse/parts_db.db/t1"},"hive_storagedesc.inputFormat":{"type":"string","value":"org.apache.hadoop.mapred.TextInputFormat"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_storagedesc qualifiedName parts_db.t1@cl1_storage storedAsSubDirectories false location hdfs://localhost.localdomain:8020/apps/hive/warehouse/parts_db.db/t1 compressed false inputFormat org.apache.hadoop.mapred.TextInputFormat outputFormat org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat table serdeInfo serializationLib org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe name null parameters serialization.format 1 numBuckets -1 "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1@cl1_storage"},"__guid":{"type":"string","
value":"793900ed-16eb-436e-ad3f-dae391b3c09e"},"hive_storagedesc.storedAsSubDirectories":{"type":"boolean","value":false},"__version":{"type":"integer","value":0},"hive_storagedesc.outputFormat":{"type":"string","value":"org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat"},"hive_storagedesc.parameters":{"type":"list","value":[]},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_storagedesc"},"__modificationTimestamp":{"type":"long","value":1522695186970},"hive_storagedesc.compressed":{"type":"boolean","value":false},"hive_storagedesc.numBuckets":{"type":"integer","value":-1},"__timestamp":{"type":"long","value":1522695179454},"_id":49336,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col4"},"hive_column.type":{"type":"string","value":"string"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"AC
TIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.tv1.col4@cl1 name col4 position 0 type string table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1.col4@cl1"},"__guid":{"type":"string","value":"615a5604-ac47-45b0-9ec3-73b705b5c619"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":0},"__timestamp":{"type":"long","value":1522695183992},"_id":65720,"_type":"vertex"},{"hive_column_lineage.depenendencyType":{"type":"string","value":"SIMPLE"},"Asset.name":{"type":"string","value":"create view tv1 partitio
ned on (col4) as select col1, col2, col4 from t1:col2"},"Process.inputs":{"type":"list","value":[{"type":"string","value":"4fmf-1rco-2uqd-pfc"}]},"Process.outputs":{"type":"list","value":[{"type":"string","value":"4avr-1rco-2wb9-1rbc"}]},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column_lineage outputs qualifiedName parts_db.tv1@cl1:1522695183000:col2 inputs query name create view tv1 partitioned on (col4) as select col1, col2, col4 from t1:col2 depenendencyType SIMPLE "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1@cl1:1522695183000:col2"},"__guid":{"type":"string","value":"872073fc-fd7a-4587-9f9c-9de1be8444d5"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"Process"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__
typeName":{"type":"string","value":"hive_column_lineage"},"__modificationTimestamp":{"type":"long","value":1522695186459},"__timestamp":{"type":"long","value":1522695186459},"_id":82104,"_type":"vertex"},{"__type.name":{"type":"string","value":"Referenceable"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"null"},"__type.Referenceable":{"type":"list","value":[{"type":"string","value":"qualifiedName"}]},"__guid":{"type":"string","value":"2b9a6b23-c108-4a8d-ac12-483ce314a846"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695093946},"__type":{"type":"string","value":"typeSystem"},"__type.Referenceable.qualifiedName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposit
e\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"qualifiedName\",\"isUnique\":true}"},"__type.description":{"type":"string","value":"Referenceable"},"__timestamp":{"type":"long","value":1522695093946},"_id":16592,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col1"},"hive_column.type":{"type":"string","value":"int"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.t1.col1@cl1 name col1 position 0 type int table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1.col1@cl1"},"__guid":{"type":"string","value":"d1a16e97-7171-4119-bfc1-8a66e36ddfb1"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},
"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":0},"__timestamp":{"type":"long","value":1522695179454},"_id":32976,"_type":"vertex"},{"__type.name":{"type":"string","value":"__AtlasUserSavedSearch"},"__type.__AtlasUserSavedSearch.searchType":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"searchType\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.__AtlasUserSavedSearch.ownerName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"ownerName\",\"isUnique\":false}"}
,"__type.__AtlasUserSavedSearch":{"type":"list","value":[{"type":"string","value":"name"},{"type":"string","value":"ownerName"},{"type":"string","value":"uniqueName"},{"type":"string","value":"searchType"},{"type":"string","value":"searchParameters"}]},"__type.options":{"type":"string","value":"null"},"__type.__AtlasUserSavedSearch.name":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":false}"},"__type.__AtlasUserSavedSearch.searchParameters":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"searchParameters\",\"isUnique\":false}"},"__guid":{"type":"string","value":"7ad1392b-05bb-4919-be2b-861b6a165f76"},"__version":{"type":"long","value":1
},"__type.__AtlasUserSavedSearch.uniqueName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"uniqueName\",\"isUnique\":true}"},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695096763},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"__AtlasUserSavedSearch"},"__timestamp":{"type":"long","value":1522695096763},"_id":16616,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_principal_type"},"__type.hive_principal_type.ROLE":{"type":"integer","value":2},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"null"},"__type.hive_principal_type.USER":{"type":"integer","value":1},"__g
uid":{"type":"string","value":"252ac491-054a-4074-b636-cf7024553ead"},"__type.hive_principal_type.GROUP":{"type":"integer","value":3},"__version":{"type":"long","value":1},"__type.hive_principal_type":{"type":"list","value":[{"type":"string","value":"USER"},{"type":"string","value":"ROLE"},{"type":"string","value":"GROUP"}]},"__type.category":{"type":"string","value":"ENUM"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695098299},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"hive_principal_type"},"__timestamp":{"type":"long","value":1522695098299},"_id":16664,"_type":"vertex"},{"hive_table.createTime":{"type":"long","value":1522695183000},"hive_table.tableType":{"type":"string","value":"VIRTUAL_VIEW"},"Asset.name":{"type":"string","value":"tv1"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":
"ACTIVE"},"entityText":{"type":"string","value":"hive_table owner anonymous temporary false lastAccessTime Mon Apr 02 11:53:03 PDT 2018 qualifiedName parts_db.tv1@cl1 columns viewExpandedText select `t1`.`col1`, `t1`.`col2`, `t1`.`col4` from `parts_db`.`t1` sd tableType VIRTUAL_VIEW createTime Mon Apr 02 11:53:03 PDT 2018 name tv1 partitionKeys parameters transient_lastDdlTime 1522695183 db retention 0 viewOriginalText select col1, col2, col4 from t1 "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1@cl1"},"hive_table.parameters.transient_lastDdlTime":{"type":"string","value":"1522695183"},"hive_table.parameters":{"type":"list","value":[{"type":"string","value":"transient_lastDdlTime"}]},"hive_table.retention":{"type":"integer","value":0},"hive_table.partitionKeys":{"type":"list","value":[{"type":"string","value":"1ueb-pi0-7q51-1epk"}]},"__guid":{"type":"string","value":"3f248307-5307-4c01-b12e-776c3f410448"},"hive_table.temporary":{"type":"boolean","value":fals
e},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"hive_table.viewExpandedText":{"type":"string","value":"select `t1`.`col1`, `t1`.`col2`, `t1`.`col4` from `parts_db`.`t1`"},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_table"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_table.lastAccessTime":{"type":"long","value":1522695183000},"hive_table.viewOriginalText":{"type":"string","value":"select col1, col2, col4 from t1"},"hive_table.columns":{"type":"list","value":[{"type":"string","value":"1hr7-pi0-7rpx-1eo8"},{"type":"string","value":"1jc3-pi0-7rpx-1rbc"}]},"__timestamp":{"type":"long","value":1522695183992},"_id":33048,"_type":"vertex"},{"hive_db.parameters":{"type":"list","value":[]},"Asset.name":{"type":"st
ring","value":"parts_db"},"hive_db.ownerType":{"type":"string","value":"USER"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_db owner anonymous ownerType USER qualifiedName parts_db@cl1 clusterName cl1 name parts_db location hdfs://localhost.localdomain:8020/apps/hive/warehouse/parts_db.db "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db@cl1"},"__guid":{"type":"string","value":"ae30d78b-51b4-42ab-9436-8d60c8f68b95"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_db"},"__modificationTimestamp":{"type":"long","value":1522695186970},"hive_db.clusterName":{"type":"string","value":"cl1"},"Asset.owner":{"type":"string","value":"anonymous"},"hive_db.location":{"type":"s
tring","value":"hdfs://localhost.localdomain:8020/apps/hive/warehouse/parts_db.db"},"__timestamp":{"type":"long","value":1522695166550},"_id":16672,"_type":"vertex"},{"Asset.name":{"type":"string","value":"col3"},"hive_column.type":{"type":"string","value":"string"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column owner anonymous qualifiedName parts_db.t1.col3@cl1 name col3 position 2 type string table "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1.col3@cl1"},"__guid":{"type":"string","value":"fbc6012f-ddd5-4626-a132-8da1c8fbca54"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column"},"__modificationTimestamp":{"type
":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_column.position":{"type":"integer","value":2},"__timestamp":{"type":"long","value":1522695179454},"_id":33056,"_type":"vertex"},{"hive_column_lineage.depenendencyType":{"type":"string","value":"SIMPLE"},"Asset.name":{"type":"string","value":"create view tv1 partitioned on (col4) as select col1, col2, col4 from t1:col1"},"Process.inputs":{"type":"list","value":[{"type":"string","value":"3fac-125c-2uqd-pg0"}]},"Process.outputs":{"type":"list","value":[{"type":"string","value":"3ajo-125c-2wb9-1eo8"}]},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_column_lineage outputs qualifiedName parts_db.tv1@cl1:1522695183000:col1 inputs query name create view tv1 partitioned on (col4) as select col1, col2, col4 from t1:col1 depenendencyType SIMPLE "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.
tv1@cl1:1522695183000:col1"},"__guid":{"type":"string","value":"7d689c57-7b36-45c1-8052-e19c673736ec"},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"Process"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_column_lineage"},"__modificationTimestamp":{"type":"long","value":1522695186970},"__timestamp":{"type":"long","value":1522695186970},"_id":49440,"_type":"vertex"},{"__type.name":{"type":"string","value":"TaxonomyTerm"},"__type.TaxonomyTerm.atlas.taxonomy":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"atlas.taxonomy\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"
null"},"__guid":{"type":"string","value":"c98d432c-b263-4ff0-a0ec-0ed5bec2d5c4"},"__version":{"type":"long","value":1},"__type.TaxonomyTerm":{"type":"list","value":[{"type":"string","value":"atlas.taxonomy"}]},"__type.category":{"type":"string","value":"TRAIT"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695093163},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"TaxonomyTerm"},"__timestamp":{"type":"long","value":1522695093163},"_id":16784,"_type":"vertex"},{"__type.name":{"type":"string","value":"__internal"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__modifiedBy":{"type":"string","value":"root"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695093960},"__type":{"type":"string","valu
e":"typeSystem"},"__type.options":{"type":"string","value":"null"},"__type.description":{"type":"string","value":"__internal"},"__guid":{"type":"string","value":"1d67b415-d737-44d6-a017-41b5fdf032aa"},"__timestamp":{"type":"long","value":1522695093960},"_id":33168,"_type":"vertex"},{"__type.name":{"type":"string","value":"Infrastructure"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__modifiedBy":{"type":"string","value":"root"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695095359},"__type":{"type":"string","value":"typeSystem"},"__type.options":{"type":"string","value":"null"},"__type.description":{"type":"string","value":"Infrastructure"},"__guid":{"type":"string","value":"7debb9e2-47a3-45ce-9d96-db6d91aad5d9"},"__timestamp":{"type":"long","value":1522695095359},"_id":49552,"_type":"vertex"},{"__type.name":{"type":"string","value":
"Process"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"null"},"__type.Process.inputs":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<DataSet>\",\"name\":\"inputs\",\"isUnique\":false}"},"__guid":{"type":"string","value":"e31c5978-6c25-48a7-8aa5-093c6b5ba5ef"},"__version":{"type":"long","value":1},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695095364},"__type":{"type":"string","value":"typeSystem"},"__type.Process":{"type":"list","value":[{"type":"string","value":"inputs"},{"type":"string","value":"outputs"}]},"__type.Process.outputs":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":fals
e}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<DataSet>\",\"name\":\"outputs\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"Process"},"__timestamp":{"type":"long","value":1522695095364},"_id":65936,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_serde"},"__type.hive_serde.parameters":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","value":"null"},"__type.hive_serde.serializationLib":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"serializa
tionLib\",\"isUnique\":false}"},"__guid":{"type":"string","value":"088a8b06-c0f9-4b49-9dc4-fe02affcb609"},"__version":{"type":"long","value":1},"__type.hive_serde":{"type":"list","value":[{"type":"string","value":"name"},{"type":"string","value":"serializationLib"},{"type":"string","value":"parameters"}]},"__type.category":{"type":"string","value":"STRUCT"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695099017},"__type":{"type":"string","value":"typeSystem"},"__type.hive_serde.name":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"hive_serde"},"__timestamp":{"type":"long","value":1522695099017},"_id":82320,"_type":"vertex"},{"hive_table.create
Time":{"type":"long","value":1522695179000},"hive_table.tableType":{"type":"string","value":"MANAGED_TABLE"},"Asset.name":{"type":"string","value":"t1"},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"entityText":{"type":"string","value":"hive_table owner anonymous temporary false lastAccessTime Mon Apr 02 11:52:59 PDT 2018 qualifiedName parts_db.t1@cl1 columns sd tableType MANAGED_TABLE createTime Mon Apr 02 11:52:59 PDT 2018 name t1 partitionKeys parameters transient_lastDdlTime 1522695179 db retention 0 "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.t1@cl1"},"hive_table.parameters.transient_lastDdlTime":{"type":"string","value":"1522695179"},"hive_table.parameters":{"type":"list","value":[{"type":"string","value":"transient_lastDdlTime"}]},"hive_table.retention":{"type":"integer","value":0},"hive_table.partitionKeys":{"type":"list","value":[{"type":"string","value":"4lyq-245s-7q51-1em8"}]},"__guid":{"type":"s
tring","value":"808b29b8-9ce0-4f86-aadd-639d3401bccb"},"hive_table.temporary":{"type":"boolean","value":false},"__version":{"type":"integer","value":0},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"DataSet"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_table"},"__modificationTimestamp":{"type":"long","value":1522695186970},"Asset.owner":{"type":"string","value":"anonymous"},"hive_table.lastAccessTime":{"type":"long","value":1522695179000},"hive_table.columns":{"type":"list","value":[{"type":"string","value":"49bm-245s-7rpx-pg0"},{"type":"string","value":"4awi-245s-7rpx-pfc"},{"type":"string","value":"4che-245s-7rpx-pi8"}]},"__timestamp":{"type":"long","value":1522695179454},"_id":98704,"_type":"vertex"},{"__type.name":{"type":"string","value":"__AtlasUserProfile"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"typ
e":"string","value":"null"},"__type.__AtlasUserProfile.name":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":true}"},"__guid":{"type":"string","value":"881822d1-efd1-4dc6-aea4-03d88b13f989"},"__type.__AtlasUserProfile.fullName":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"fullName\",\"isUnique\":false}"},"__version":{"type":"long","value":1},"__type.__AtlasUserProfile.savedSearches":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<__AtlasUserSavedSearch>\",\"name\":\"savedSearche
s\",\"isUnique\":false}"},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__type.__AtlasUserProfile":{"type":"list","value":[{"type":"string","value":"name"},{"type":"string","value":"fullName"},{"type":"string","value":"savedSearches"}]},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695096058},"__type":{"type":"string","value":"typeSystem"},"__type.description":{"type":"string","value":"__AtlasUserProfile"},"__timestamp":{"type":"long","value":1522695096058},"_id":16808,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_table"},"__modifiedBy":{"type":"string","value":"root"},"__type.hive_table.tableType":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"tableType\",\"isUnique\":false}"},"__type.hive_ta
ble.retention":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"retention\",\"isUnique\":false}"},"__version":{"type":"long","value":2},"__type.hive_table.db":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_db\",\"name\":\"db\",\"isUnique\":false}"},"__type.hive_table.viewOriginalText":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"viewOriginalText\",\"isUnique\":false}"},"__type.version":{"type":"string","value":"1.1"},"__type":{"type":"string","value":"typeSystem"},"__type.hive_table.partitionKeys":{"type":"st
ring","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hive_column>\",\"name\":\"partitionKeys\",\"isUnique\":false}"},"__timestamp":{"type":"long","value":1522695099075},"__type.hive_table.aliases":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"aliases\",\"isUnique\":false}"},"__type.hive_table.sd":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"hive_storagedesc\",\"name\":\"sd\",\"isUnique\":false}"},"__type.hive_table.columns":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIn
dexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hive_column>\",\"name\":\"columns\",\"isUnique\":false}"},"__type.hive_table.temporary":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"temporary\",\"isUnique\":false}"},"__type.options":{"type":"string","value":"{\"schemaElementsAttribute\":\"columns\"}"},"__type.hive_table.comment":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"comment\",\"isUnique\":false}"},"__type.hive_table.viewExpandedText":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":n
ull,\"dataType\":\"string\",\"name\":\"viewExpandedText\",\"isUnique\":false}"},"__guid":{"type":"string","value":"abe70487-b34e-4b54-9e08-2426e958d376"},"__type.hive_table.createTime":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"},"__type.hive_table.lastAccessTime":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"lastAccessTime\",\"isUnique\":false}"},"__type.hive_table":{"type":"list","value":[{"type":"string","value":"db"},{"type":"string","value":"createTime"},{"type":"string","value":"lastAccessTime"},{"type":"string","value":"comment"},{"type":"string","value":"retention"},{"type":"string","value":"sd"},{"type":"string",
"value":"partitionKeys"},{"type":"string","value":"aliases"},{"type":"string","value":"columns"},{"type":"string","value":"parameters"},{"type":"string","value":"viewOriginalText"},{"type":"string","value":"viewExpandedText"},{"type":"string","value":"tableType"},{"type":"string","value":"temporary"}]},"__type.hive_table.parameters":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"},"__type.category":{"type":"string","value":"CLASS"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695100458},"__type.description":{"type":"string","value":"hive_table"},"_id":33192,"_type":"vertex"},{"__type.name":{"type":"string","value":"hive_column_lineage"},"__modifiedBy":{"type":"string","value":"root"},"__type.options":{"type":"string","va
lue":"null"},"__type.hive_column_lineage.expression":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"expression\",\"isUnique\":false}"},"__guid":{"type":"string","value":"78572253-4763-405b-bbc9-0b77626535fe"},"__version":{"type":"long","value":1},"__type.hive_column_lineage":{"type":"list","value":[{"type":"string","value":"query"},{"type":"string","value":"depenendencyType"},{"type":"string","value":"expression"}]},"__type.category":{"type":"string","value":"CLASS"},"__type.version":{"type":"string","value":"1.0"},"__createdBy":{"type":"string","value":"root"},"__modificationTimestamp":{"type":"long","value":1522695099192},"__type":{"type":"string","value":"typeSystem"},"__type.hive_column_lineage.depenendencyType":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\"
,\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"depenendencyType\",\"isUnique\":false}"},"__type.hive_column_lineage.query":{"type":"string","value":"{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_process\",\"name\":\"query\",\"isUnique\":false}"},"__type.description":{"type":"string","value":"hive_column_lineage"},"__timestamp":{"type":"long","value":1522695099192},"_id":49576,"_type":"vertex"},{"hive_serde.parameters.serialization.format":{"type":"string","value":"1"},"hive_serde.parameters":{"type":"list","value":[{"type":"string","value":"serialization.format"}]},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_serde"},"__modificationTimestamp":{"type"
:"long","value":1522695186970},"hive_serde.serializationLib":{"type":"string","value":"org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe"},"__timestamp":{"type":"long","value":1522695179454},"_id":65960,"_type":"vertex"},{"Asset.name":{"type":"string","value":"create view tv1 partitioned on (col4) as select col1, col2, col4 from t1"},"Process.inputs":{"type":"list","value":[{"type":"string","value":"4sad-1rjc-2uqd-245s"}]},"Process.outputs":{"type":"list","value":[{"type":"string","value":"4lyt-1rjc-2wb9-pi0"}]},"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"hive_process.queryPlan":{"type":"string","value":"Not Supported"},"hive_process.queryText":{"type":"string","value":"create view tv1 partitioned on (col4) as select col1, col2, col4 from t1"},"entityText":{"type":"string","value":"hive_process outputs recentQueries create view tv1 partitioned on (col4) as select col1, col2, col4 from t1 qualifiedName parts_db.tv1@cl1:152269518
3000 inputs userName anonymous queryId hive_20180402115303_03c23526-e1ee-4cf9-80af-6c8652bcefa3 clusterName cl1 name create view tv1 partitioned on (col4) as select col1, col2, col4 from t1 queryText create view tv1 partitioned on (col4) as select col1, col2, col4 from t1 startTime Mon Apr 02 11:53:03 PDT 2018 operationType CREATEVIEW queryPlan Not Supported endTime Mon Apr 02 11:53:03 PDT 2018 "},"Referenceable.qualifiedName":{"type":"string","value":"parts_db.tv1@cl1:1522695183000"},"__guid":{"type":"string","value":"330938ef-2f16-43c5-b251-a72455775440"},"hive_process.operationType":{"type":"string","value":"CREATEVIEW"},"hive_process.endTime":{"type":"long","value":1522695183616},"__version":{"type":"integer","value":0},"hive_process.recentQueries":{"type":"list","value":[{"type":"string","value":"create view tv1 partitioned on (col4) as select col1, col2, col4 from t1"}]},"hive_process.startTime":{"type":"long","value":1522695183339},"hive_process.clusterName":{"type":"string",
"value":"cl1"},"__superTypeNames":{"type":"list","value":[{"type":"string","value":"Asset"},{"type":"string","value":"Process"},{"type":"string","value":"Referenceable"}]},"__createdBy":{"type":"string","value":"anonymous"},"__typeName":{"type":"string","value":"hive_process"},"hive_process.queryId":{"type":"string","value":"hive_20180402115303_03c23526-e1ee-4cf9-80af-6c8652bcefa3"},"__modificationTimestamp":{"type":"long","value":1522695186970},"hive_process.userName":{"type":"string","value":"anonymous"},"__timestamp":{"type":"long","value":1522695185696},"_id":82344,"_type":"vertex"}],"edges":[{"_id":"2rk8-pc0-2711-csw","_type":"edge","_outV":32832,"_inV":16592,"_label":"__type..supertype"},{"_id":"2mtk-pc0-6gb9-pm0","_type":"edge","_outV":32832,"_inV":33192,"_label":"__type.edge.hive_storagedesc.table"},{"_id":"2oeg-pc0-6hw5-1rio","_type":"edge","_outV":32832,"_inV":82320,"_label":"__type.edge.hive_storagedesc.serdeInfo"},{"_id":"2pzc-pc0-6jh1-cow","_type":"edge","_outV":32832,"
_inV":16448,"_label":"__type.edge.hive_storagedesc.sortCols"},{"_id":"2waw-11z4-2711-cqw","_type":"edge","_outV":49216,"_inV":16520,"_label":"__type..supertype"},{"_id":"2uq0-11z4-2711-csw","_type":"edge","_outV":49216,"_inV":16592,"_label":"__type..supertype"},{"_id":"2t54-11z4-6l1x-cuw","_type":"edge","_outV":49216,"_inV":16664,"_label":"__type.edge.hive_db.ownerType"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"3oqw-1em8-8tn9-245s","_type":"edge","_outV":65600,"_inV":98704,"_label":"__hive_column.table"},{"_id":"2l8x-pe0-2711-1evk","_type":"edge","_outV":32904,"_inV":65936,"_label":"__type..supertype"},{"_id":"2oep-1214-2711-cs8","_type":"edge","_outV":49288,"_inV":16568,"_label":"__type..supertype"},{"_id":"2mtt-1214-6mmt-pm0","_type":"edge","_outV":492
88,"_inV":33192,"_label":"__type.edge.hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"56hd-1eo8-8tn9-pi0","_type":"edge","_outV":65672,"_inV":33048,"_label":"__hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"5fyp-1rbc-8tn9-pi0","_type":"edge","_outV":82056,"_inV":33048,"_label":"__hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":152
2695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"5r0x-23yg-82s5-pi0","_type":"edge","_outV":98440,"_inV":33048,"_label":"__hive_storagedesc.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"658x-23yg-9frp-2glk","_type":"edge","_outV":98440,"_inV":114824,"_label":"__hive_storagedesc.serdeInfo"},{"_id":"npz-cs8-2711-cqw","_type":"edge","_outV":16568,"_inV":16520,"_label":"__type..supertype"},{"_id":"m53-cs8-2711-csw","_type":"edge","_outV":16568,"_inV":16592,"_label":"__type..supertype"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":
1522695179454},"_id":"1vyv-pfc-8tn9-245s","_type":"edge","_outV":32952,"_inV":98704,"_label":"__hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"28lz-122g-82s5-245s","_type":"edge","_outV":49336,"_inV":98704,"_label":"__hive_storagedesc.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"2a6v-122g-9frp-1ew8","_type":"edge","_outV":49336,"_inV":65960,"_label":"__hive_storagedesc.serdeInfo"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string"
,"value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"3c47-1epk-8tn9-pi0","_type":"edge","_outV":65720,"_inV":33048,"_label":"__hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186459},"__timestamp":{"type":"long","value":1522695186459},"_id":"4fmf-1rco-2uqd-pfc","_type":"edge","_outV":82104,"_inV":32952,"_label":"__Process.inputs"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186459},"__timestamp":{"type":"long","value":1522695186459},"_id":"4avr-1rco-2wb9-1rbc","_type":"edge","_outV":82104,"_inV":82056,"_label":"__Process.outputs"},{"__modifiedBy":{"type":"s
tring","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186459},"__timestamp":{"type":"long","value":1522695186459},"_id":"4is7-1rco-8v85-1rjc","_type":"edge","_outV":82104,"_inV":82344,"_label":"__hive_column_lineage.query"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"1g62-pg0-8tn9-245s","_type":"edge","_outV":32976,"_inV":98704,"_label":"__hive_column.table"},{"_id":"vml-ctk-2711-plc","_type":"edge","_outV":16616,"_inV":33168,"_label":"__type..supertype"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"t
ype":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"20pv-pi0-7gnp-cv4","_type":"edge","_outV":33048,"_inV":16672,"_label":"__hive_table.db"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"1o2r-pi0-7ok5-23yg","_type":"edge","_outV":33048,"_inV":98440,"_label":"__hive_table.sd"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"1ueb-pi0-7q51-1epk","_type":"edge","_outV":33048,"_inV":65720,"_label":"__hive_table.partitionKeys"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"st
ring","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"1hr7-pi0-7rpx-1eo8","_type":"edge","_outV":33048,"_inV":65672,"_label":"__hive_table.columns"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695183992},"__timestamp":{"type":"long","value":1522695183992},"_id":"1jc3-pi0-7rpx-1rbc","_type":"edge","_outV":33048,"_inV":82056,"_label":"__hive_table.columns"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"1mhw-pi8-8tn9-245s","_type":"edge","_outV":33056,"_inV":9870
4,"_label":"__hive_column.table"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186970},"__timestamp":{"type":"long","value":1522695186970},"_id":"3fac-125c-2uqd-pg0","_type":"edge","_outV":49440,"_inV":32976,"_label":"__Process.inputs"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186970},"__timestamp":{"type":"long","value":1522695186970},"_id":"3ajo-125c-2wb9-1eo8","_type":"edge","_outV":49440,"_inV":65672,"_label":"__Process.outputs"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695186970},"__timestamp":{"
type":"long","value":1522695186970},"_id":"3ig4-125c-8v85-1rjc","_type":"edge","_outV":49440,"_inV":82344,"_label":"__hive_column_lineage.query"},{"_id":"2l9u-128g-2711-cqw","_type":"edge","_outV":49552,"_inV":16520,"_label":"__type..supertype"},{"_id":"2joy-128g-2711-csw","_type":"edge","_outV":49552,"_inV":16592,"_label":"__type..supertype"},{"_id":"2rle-1evk-2711-cqw","_type":"edge","_outV":65936,"_inV":16520,"_label":"__type..supertype"},{"_id":"2q0i-1evk-2711-csw","_type":"edge","_outV":65936,"_inV":16592,"_label":"__type..supertype"},{"_id":"2muq-1evk-28lx-cs8","_type":"edge","_outV":65936,"_inV":16568,"_label":"__type.edge.Process.inputs"},{"_id":"2ofm-1evk-2a6t-cs8","_type":"edge","_outV":65936,"_inV":16568,"_label":"__type.edge.Process.outputs"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"t
ype":"long","value":1522695179454},"_id":"4saa-245s-7gnp-cv4","_type":"edge","_outV":98704,"_inV":16672,"_label":"__hive_table.db"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"4fn6-245s-7ok5-122g","_type":"edge","_outV":98704,"_inV":49336,"_label":"__hive_table.sd"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"4lyq-245s-7q51-1em8","_type":"edge","_outV":98704,"_inV":65600,"_label":"__hive_table.partitionKeys"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"st
ring","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"4awi-245s-7rpx-pfc","_type":"edge","_outV":98704,"_inV":32952,"_label":"__hive_table.columns"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"49bm-245s-7rpx-pg0","_type":"edge","_outV":98704,"_inV":32976,"_label":"__hive_table.columns"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695179454},"__timestamp":{"type":"long","value":1522695179454},"_id":"4che-245s-7rpx-pi8","_type":"edge","_outV":98704,"_inV":33056,"_label":"__hive_table.columns"},{"_id":"u2d-cy
w-2711-plc","_type":"edge","_outV":16808,"_inV":33168,"_label":"__type..supertype"},{"_id":"shh-cyw-2brp-ctk","_type":"edge","_outV":16808,"_inV":16616,"_label":"__type.edge.__AtlasUserProfile.savedSearches"},{"_id":"312t-pm0-2711-cs8","_type":"edge","_outV":33192,"_inV":16568,"_label":"__type..supertype"},{"_id":"2ur9-pm0-69zp-11z4","_type":"edge","_outV":33192,"_inV":49216,"_label":"__type.edge.hive_table.db"},{"_id":"2wc5-pm0-6bkl-pc0","_type":"edge","_outV":33192,"_inV":32832,"_label":"__type.edge.hive_table.sd"},{"_id":"2xx1-pm0-6d5h-1214","_type":"edge","_outV":33192,"_inV":49288,"_label":"__type.edge.hive_table.partitionKeys"},{"_id":"2zhx-pm0-6eqd-1214","_type":"edge","_outV":33192,"_inV":49288,"_label":"__type.edge.hive_table.columns"},{"_id":"348l-1294-2711-1evk","_type":"edge","_outV":49576,"_inV":65936,"_label":"__type..supertype"},{"_id":"32np-1294-6o7p-pe0","_type":"edge","_outV":49576,"_inV":32904,"_label":"__type.edge.hive_column_lineage.query"},{"__modifiedBy":{"typ
e":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695185696},"__timestamp":{"type":"long","value":1522695185696},"_id":"4sad-1rjc-2uqd-245s","_type":"edge","_outV":82344,"_inV":98704,"_label":"__Process.inputs"},{"__modifiedBy":{"type":"string","value":"anonymous"},"__state":{"type":"string","value":"ACTIVE"},"__createdBy":{"type":"string","value":"anonymous"},"__modificationTimestamp":{"type":"long","value":1522695185696},"__timestamp":{"type":"long","value":1522695185696},"_id":"4lyt-1rjc-2wb9-pi0","_type":"edge","_outV":82344,"_inV":33048,"_label":"__Process.outputs"}]}
\ No newline at end of file
+{
+ "mode": "EXTENDED",
+ "vertices": [
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_order"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_order.order": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"order\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "2acd11e7-9cbd-4d47-9ded-831dabf80206"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hive_order": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "order"
+ },
+ {
+ "type": "string",
+ "value": "col"
+ }
+ ]
+ },
+ "__type.hive_order.col": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"col\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "STRUCT"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522695099005
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_order"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522695099005
+ },
+ "_id": 16448,
+ "_type": "vertex"
+ },
+ {
+ "__type.hive_storagedesc.outputFormat": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"outputFormat\",\"isUnique\":false}"
+ },
+ "__type.hive_storagedesc.bucketCols": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"bucketCols\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "hive_storagedesc"
+ },
+ "__type.hive_storagedesc.parameters": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hive_storagedesc.numBuckets": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"numBuckets\",\"isUnique\":false}"
+ },
+ "__type.hive_storagedesc.sortCols": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<hive_order>\",\"name\":\"sortCols\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "8b3f3741-3ba9-49f1-b545-43f12fa6fe29"
+ },
+ "__type.hive_storagedesc": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "table"
+ },
+ {
+ "type": "string",
+ "value": "location"
+ },
+ {
+ "type": "string",
+ "value": "inputFormat"
+ },
+ {
+ "type": "string",
+ "value": "outputFormat"
+ },
+ {
+ "type": "string",
+ "value": "compressed"
+ },
+ {
+ "type": "string",
+ "value": "numBuckets"
+ },
+ {
+ "type": "string",
+ "value": "serdeInfo"
+ },
+ {
+ "type": "string",
+ "value": "bucketCols"
+ },
+ {
+ "type": "string",
+ "value": "sortCols"
+ },
+ {
+ "type": "string",
+ "value": "parameters"
+ },
+ {
+ "type": "string",
+ "value": "storedAsSubDirectories"
+ }
+ ]
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hive_storagedesc.inputFormat": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"inputFormat\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_storagedesc.serdeInfo": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_serde\",\"name\":\"serdeInfo\",\"isUnique\":false}"
+ },
+ "__type.hive_storagedesc.storedAsSubDirectories": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"storedAsSubDirectories\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522695099132
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hive_storagedesc.location": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"location\",\"isUnique\":false}"
+ },
+ "__type.hive_storagedesc.compressed": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"compressed\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_storagedesc"
+ },
+ "__type.hive_storagedesc.table": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":\"sd\",\"dataType\":\"hive_table\",\"name\":\"table\",\"isUnique\":false}"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522695099132
+ },
+ "_id": 32832,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_db"
+ },
+ "__type.hive_db.parameters": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_db": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "location"
+ },
+ {
+ "type": "string",
+ "value": "parameters"
+ },
+ {
+ "type": "string",
+ "value": "ownerType"
+ }
+ ]
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "14d28b0c-30b0-42e4-addd-52501ea461ed"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hive_db.clusterName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__type.hive_db.location": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"location\",\"isUnique\":false}"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522695099163
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_db"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522695099163
+ },
+ "__type.hive_db.ownerType": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_principal_type\",\"name\":\"ownerType\",\"isUnique\":false}"
+ },
+ "_id": 49216,
+ "_type": "vertex"
+ },
+ {
+ "Asset.name": {
+ "type": "string",
+ "value": "col4"
+ },
+ "hive_column.type": {
+ "type": "string",
+ "value": "string"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "hive_column owner anonymous qualifiedName parts_db.t1.col4@cl1 name col4 position 0 type string table "
+ },
+ "Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "parts_db.t1.col4@cl1"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "aae38dd6-5632-48a4-9e41-cea65323e47b"
+ },
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "__superTypeNames": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "Asset"
+ },
+ {
+ "type": "string",
+ "value": "DataSet"
+ },
+ {
+ "type": "string",
+ "value": "Referenceable"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522695186970
+ },
+ "Asset.owner": {
+ "type": "string",
+ "value": "anonymous"
+ },
+ "hive_column.position": {
+ "type": "integer",
+ "value": 0
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522695179454
+ },
+ "_id": 65600,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "Asset"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.Asset.name": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":false}"
+ },
+ "__type.Asset.owner": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"owner\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "80dfb85b-bbde-4e1b-a194-cc69519123bc"
+ },
+ "__type.Asset": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "name"
+ },
+ {
+ "type": "string",
+ "value": "description"
+ },
+ {
+ "type": "string",
+ "value": "owner"
+ }
+ ]
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1522695094647
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.Asset.description": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"description\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "Asset"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1522695094647
+ },
+ "_id": 16520,
+ "_type": "vertex"
+ },
+ {
+ "__type.hive_process.operationType": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"operationType\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "hive_process"
+ },
+ "__type.hive_process.userName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"userName\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_process.queryId": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryId\",\"isUnique\":false}"
+ },
+ "__type.hive_process.queryPlan": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryPlan\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hive_process.recentQueries": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"recentQueries\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "fb115e8e-818c-411b-9b93-4caf289f5413"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.hive_process": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "startTime"
+ },
+ {
+ "type": "string",
+ "value": "endTime"
+ },
+ {
+ "type": "string",
+ "value": "userName"
+ },
+ {
+ "type": "string",
+ "value": "operationType"
+ },
+ {
+ "type": "string",
+ "value": "queryText"
+ },
+ {
+ "type": "string",
+ "value": "queryPlan"
+ },
+ {
+ "type": "string",
+ "value": "queryId"
+ },
+ {
+ "type": "string",
+ "value": "recentQueries"
+ },
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "queryGraph"
+ }
+ ]
+ },
+ "__type.version": {
+ "type": "s
<TRUNCATED>
[8/8] atlas git commit: ATLAS-2712: Update v2 Audit API to handle v1
to v2 conversion
Posted by ma...@apache.org.
ATLAS-2712: Update v2 Audit API to handle v1 to v2 conversion
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
(cherry picked from commit d343a4860df49e767eeefd46ab5fa2b147cc5cd3)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/bf037c92
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/bf037c92
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/bf037c92
Branch: refs/heads/branch-1.0
Commit: bf037c92e643659b8ad0580afd3b52a7e1051785
Parents: 5cbfca2
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Tue May 22 18:38:33 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue May 22 23:11:56 2018 -0700
----------------------------------------------------------------------
.../atlas/model/audit/EntityAuditEventV2.java | 51 +++++--
.../AbstractStorageBasedAuditRepository.java | 19 ++-
.../audit/CassandraBasedAuditRepository.java | 4 +-
.../repository/audit/EntityAuditListener.java | 42 +++++-
.../repository/audit/EntityAuditListenerV2.java | 76 +++++++---
.../audit/HBaseBasedAuditRepository.java | 151 ++++++++++++++++---
.../converters/AtlasInstanceConverter.java | 28 ++--
.../graph/v1/AtlasEntityChangeNotifier.java | 8 +-
.../audit/AuditRepositoryTestBase.java | 8 +-
9 files changed, 300 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
----------------------------------------------------------------------
diff --git a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
index d14f6ae..787f5a9 100644
--- a/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
+++ b/intg/src/main/java/org/apache/atlas/model/audit/EntityAuditEventV2.java
@@ -32,6 +32,7 @@ import java.util.Objects;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE;
import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType.ENTITY_AUDIT_V2;
/**
* Structure of v2 entity audit event
@@ -42,13 +43,15 @@ import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_
@XmlRootElement
@XmlAccessorType(XmlAccessType.PROPERTY)
public class EntityAuditEventV2 implements Serializable {
- public enum EntityAuditAction {
+ public enum EntityAuditType { ENTITY_AUDIT_V1, ENTITY_AUDIT_V2 }
+
+ public enum EntityAuditActionV2 {
ENTITY_CREATE, ENTITY_UPDATE, ENTITY_DELETE,
ENTITY_IMPORT_CREATE, ENTITY_IMPORT_UPDATE, ENTITY_IMPORT_DELETE,
CLASSIFICATION_ADD, CLASSIFICATION_DELETE, CLASSIFICATION_UPDATE,
PROPAGATED_CLASSIFICATION_ADD, PROPAGATED_CLASSIFICATION_DELETE, PROPAGATED_CLASSIFICATION_UPDATE;
- public static EntityAuditAction fromString(String strValue) {
+ public static EntityAuditActionV2 fromString(String strValue) {
switch (strValue) {
case "ENTITY_CREATE":
return ENTITY_CREATE;
@@ -79,28 +82,35 @@ public class EntityAuditEventV2 implements Serializable {
return PROPAGATED_CLASSIFICATION_UPDATE;
}
- throw new IllegalArgumentException("No enum constant " + EntityAuditAction.class.getCanonicalName() + "." + strValue);
+ throw new IllegalArgumentException("No enum constant " + EntityAuditActionV2.class.getCanonicalName() + "." + strValue);
}
}
- private String entityId;
- private long timestamp;
- private String user;
- private EntityAuditAction action;
- private String details;
- private String eventKey;
- private AtlasEntity entity;
+ private String entityId;
+ private long timestamp;
+ private String user;
+ private EntityAuditActionV2 action;
+ private String details;
+ private String eventKey;
+ private AtlasEntity entity;
+ private EntityAuditType type;
public EntityAuditEventV2() { }
- public EntityAuditEventV2(String entityId, long timestamp, String user, EntityAuditAction action, String details,
+ public EntityAuditEventV2(String entityId, long timestamp, String user, EntityAuditActionV2 action, String details,
AtlasEntity entity) {
+ this(entityId, timestamp, user, action, details, entity, ENTITY_AUDIT_V2);
+ }
+
+ public EntityAuditEventV2(String entityId, long timestamp, String user, EntityAuditActionV2 action, String details,
+ AtlasEntity entity, EntityAuditType auditType) {
setEntityId(entityId);
setTimestamp(timestamp);
setUser(user);
setAction(action);
setDetails(details);
setEntity(entity);
+ setType(auditType);
}
public String getEntityId() {
@@ -127,11 +137,11 @@ public class EntityAuditEventV2 implements Serializable {
this.user = user;
}
- public EntityAuditAction getAction() {
+ public EntityAuditActionV2 getAction() {
return action;
}
- public void setAction(EntityAuditAction action) {
+ public void setAction(EntityAuditActionV2 action) {
this.action = action;
}
@@ -159,6 +169,14 @@ public class EntityAuditEventV2 implements Serializable {
this.entity = entity;
}
+ public EntityAuditType getType() {
+ return type;
+ }
+
+ public void setType(EntityAuditType type) {
+ this.type = type;
+ }
+
@JsonIgnore
public String getEntityDefinitionString() {
if (entity != null) {
@@ -185,17 +203,19 @@ public class EntityAuditEventV2 implements Serializable {
action == that.action &&
Objects.equals(details, that.details) &&
Objects.equals(eventKey, that.eventKey) &&
- Objects.equals(entity, that.entity);
+ Objects.equals(entity, that.entity) &&
+ Objects.equals(type, that.type);
}
@Override
public int hashCode() {
- return Objects.hash(entityId, timestamp, user, action, details, eventKey, entity);
+ return Objects.hash(entityId, timestamp, user, action, details, eventKey, entity, type);
}
@Override
public String toString() {
final StringBuilder sb = new StringBuilder("EntityAuditEventV2{");
+
sb.append("entityId='").append(entityId).append('\'');
sb.append(", timestamp=").append(timestamp);
sb.append(", user='").append(user).append('\'');
@@ -203,6 +223,7 @@ public class EntityAuditEventV2 implements Serializable {
sb.append(", details='").append(details).append('\'');
sb.append(", eventKey='").append(eventKey).append('\'');
sb.append(", entity=").append(entity);
+ sb.append(", type=").append(type);
sb.append('}');
return sb.toString();
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java
index 39b1ef2..f6b741b 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/AbstractStorageBasedAuditRepository.java
@@ -42,17 +42,16 @@ import java.util.Map;
public abstract class AbstractStorageBasedAuditRepository implements Service, EntityAuditRepository, ActiveStateChangeHandler {
private static final Logger LOG = LoggerFactory.getLogger(HBaseBasedAuditRepository.class);
- private static final String AUDIT_REPOSITORY_MAX_SIZE_PROPERTY = "atlas.hbase.client.keyvalue.maxsize";
- private static final String AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY = "atlas.audit.hbase.entity";
- protected static final String FIELD_SEPARATOR = ":";
- private static final long ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE = 1024 * 1024;
- protected static Configuration APPLICATION_PROPERTIES = null;
- public static final String CONFIG_PREFIX = "atlas.audit";
- public static final String CONFIG_PERSIST_ENTITY_DEFINITION = CONFIG_PREFIX + ".persistEntityDefinition";
-
+ private static final String AUDIT_REPOSITORY_MAX_SIZE_PROPERTY = "atlas.hbase.client.keyvalue.maxsize";
+ private static final String AUDIT_EXCLUDE_ATTRIBUTE_PROPERTY = "atlas.audit.hbase.entity";
+ private static final long ATLAS_HBASE_KEYVALUE_DEFAULT_SIZE = 1024 * 1024;
+ public static final String CONFIG_PREFIX = "atlas.audit";
+ public static final String CONFIG_PERSIST_ENTITY_DEFINITION = CONFIG_PREFIX + ".persistEntityDefinition";
+ protected static final String FIELD_SEPARATOR = ":";
+
+ protected static Configuration APPLICATION_PROPERTIES = null;
protected Map<String, List<String>> auditExcludedAttributesCache = new HashMap<>();
-
- protected static boolean persistEntityDefinition;
+ protected static boolean persistEntityDefinition;
static {
try {
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java
index 9d6aaae..8b185a3 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/CassandraBasedAuditRepository.java
@@ -25,13 +25,11 @@ import com.datastax.driver.core.ResultSet;
import com.datastax.driver.core.Row;
import com.datastax.driver.core.Session;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.audit.EntityAuditEventV2;
-import org.apache.commons.configuration.Configuration;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -176,7 +174,7 @@ public class CassandraBasedAuditRepository extends AbstractStorageBasedAuditRepo
}
EntityAuditEventV2 event = new EntityAuditEventV2();
event.setEntityId(rowEntityId);
- event.setAction(EntityAuditEventV2.EntityAuditAction.fromString(row.getString(ACTION)));
+ event.setAction(EntityAuditEventV2.EntityAuditActionV2.fromString(row.getString(ACTION)));
event.setDetails(row.getString(DETAIL));
event.setUser(row.getString(USER));
event.setTimestamp(row.getLong(CREATED));
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
index 6e868e6..a085e8e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListener.java
@@ -146,7 +146,7 @@ public class EntityAuditListener implements EntityChangeListener {
private String getAuditEventDetail(Referenceable entity, EntityAuditAction action) throws AtlasException {
Map<String, Object> prunedAttributes = pruneEntityAttributesForAudit(entity);
- String auditPrefix = getAuditPrefix(action);
+ String auditPrefix = getV1AuditPrefix(action);
String auditString = auditPrefix + AtlasType.toV1Json(entity);
byte[] auditBytes = auditString.getBytes(StandardCharsets.UTF_8);
long auditSize = auditBytes != null ? auditBytes.length : 0;
@@ -259,7 +259,7 @@ public class EntityAuditListener implements EntityChangeListener {
}
}
- private String getAuditPrefix(EntityAuditAction action) {
+ public static String getV1AuditPrefix(EntityAuditAction action) {
final String ret;
switch (action) {
@@ -296,4 +296,42 @@ public class EntityAuditListener implements EntityChangeListener {
return ret;
}
+
+ public static String getV2AuditPrefix(EntityAuditAction action) {
+ final String ret;
+
+ switch (action) {
+ case ENTITY_CREATE:
+ ret = "Created: ";
+ break;
+ case ENTITY_UPDATE:
+ ret = "Updated: ";
+ break;
+ case ENTITY_DELETE:
+ ret = "Deleted: ";
+ break;
+ case TAG_ADD:
+ ret = "Added classification: ";
+ break;
+ case TAG_DELETE:
+ ret = "Deleted classification: ";
+ break;
+ case TAG_UPDATE:
+ ret = "Updated classification: ";
+ break;
+ case ENTITY_IMPORT_CREATE:
+ ret = "Created by import: ";
+ break;
+ case ENTITY_IMPORT_UPDATE:
+ ret = "Updated by import: ";
+ break;
+ case ENTITY_IMPORT_DELETE:
+ ret = "Deleted by import: ";
+ break;
+ default:
+ ret = "Unknown: ";
+ }
+
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
index 4fd2fd9..970b14a 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/EntityAuditListenerV2.java
@@ -17,9 +17,11 @@
*/
package org.apache.atlas.repository.audit;
+import org.apache.atlas.EntityAuditEvent;
+import org.apache.atlas.EntityAuditEvent.EntityAuditAction;
import org.apache.atlas.RequestContextV1;
import org.apache.atlas.model.audit.EntityAuditEventV2;
-import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction;
+import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.EntityChangeListenerV2;
import org.apache.atlas.model.instance.AtlasClassification;
@@ -42,18 +44,18 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_ADD;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_DELETE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_UPDATE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_CREATE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_DELETE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_CREATE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_DELETE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_UPDATE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.ENTITY_UPDATE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_ADD;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_DELETE;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_ADD;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_DELETE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_CREATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_DELETE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_CREATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_DELETE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_DELETE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_UPDATE;
@Component
public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
@@ -165,21 +167,21 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
}
}
- private EntityAuditEventV2 createEvent(AtlasEntity entity, EntityAuditAction action, String details) {
+ private EntityAuditEventV2 createEvent(AtlasEntity entity, EntityAuditActionV2 action, String details) {
return new EntityAuditEventV2(entity.getGuid(), RequestContextV1.get().getRequestTime(),
RequestContextV1.get().getUser(), action, details, entity);
}
- private EntityAuditEventV2 createEvent(AtlasEntity entity, EntityAuditAction action) {
+ private EntityAuditEventV2 createEvent(AtlasEntity entity, EntityAuditActionV2 action) {
String detail = getAuditEventDetail(entity, action);
return createEvent(entity, action, detail);
}
- private String getAuditEventDetail(AtlasEntity entity, EntityAuditAction action) {
+ private String getAuditEventDetail(AtlasEntity entity, EntityAuditActionV2 action) {
Map<String, Object> prunedAttributes = pruneEntityAttributesForAudit(entity);
- String auditPrefix = getAuditPrefix(action);
+ String auditPrefix = getV2AuditPrefix(action);
String auditString = auditPrefix + AtlasType.toJson(entity);
byte[] auditBytes = auditString.getBytes(StandardCharsets.UTF_8);
long auditSize = auditBytes != null ? auditBytes.length : 0;
@@ -277,7 +279,45 @@ public class EntityAuditListenerV2 implements EntityChangeListenerV2 {
}
}
- private String getAuditPrefix(EntityAuditAction action) {
+ private String getV1AuditPrefix(EntityAuditAction action) {
+ final String ret;
+
+ switch (action) {
+ case ENTITY_CREATE:
+ ret = "Created: ";
+ break;
+ case ENTITY_UPDATE:
+ ret = "Updated: ";
+ break;
+ case ENTITY_DELETE:
+ ret = "Deleted: ";
+ break;
+ case TAG_ADD:
+ ret = "Added classification: ";
+ break;
+ case TAG_DELETE:
+ ret = "Deleted classification: ";
+ break;
+ case TAG_UPDATE:
+ ret = "Updated classification: ";
+ break;
+ case ENTITY_IMPORT_CREATE:
+ ret = "Created by import: ";
+ break;
+ case ENTITY_IMPORT_UPDATE:
+ ret = "Updated by import: ";
+ break;
+ case ENTITY_IMPORT_DELETE:
+ ret = "Deleted by import: ";
+ break;
+ default:
+ ret = "Unknown: ";
+ }
+
+ return ret;
+ }
+
+ private String getV2AuditPrefix(EntityAuditActionV2 action) {
final String ret;
switch (action) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
index e55864b..3656b85 100644
--- a/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
+++ b/repository/src/main/java/org/apache/atlas/repository/audit/HBaseBasedAuditRepository.java
@@ -22,12 +22,21 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.EntityAuditEvent;
+import org.apache.atlas.EntityAuditEvent.EntityAuditAction;
import org.apache.atlas.annotation.ConditionalOnAtlasProperty;
import org.apache.atlas.model.audit.EntityAuditEventV2;
-import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction;
+import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.ha.HAConfiguration;
+import org.apache.atlas.model.instance.AtlasClassification;
+import org.apache.atlas.model.instance.AtlasEntity;
+import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo;
+import org.apache.atlas.repository.converters.AtlasInstanceConverter;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.v1.model.instance.Referenceable;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -50,6 +59,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Component;
+import javax.inject.Inject;
import javax.inject.Singleton;
import java.io.Closeable;
import java.io.IOException;
@@ -57,6 +67,14 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_ADD;
+import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_DELETE;
+import static org.apache.atlas.EntityAuditEvent.EntityAuditAction.TAG_UPDATE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType.ENTITY_AUDIT_V1;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditType.ENTITY_AUDIT_V2;
+import static org.apache.atlas.repository.audit.EntityAuditListener.getV2AuditPrefix;
+
/**
* HBase based repository for entity audit events
* <p>
@@ -77,17 +95,23 @@ import java.util.List;
public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditRepository {
private static final Logger LOG = LoggerFactory.getLogger(HBaseBasedAuditRepository.class);
- public static final String CONFIG_TABLE_NAME = CONFIG_PREFIX + ".hbase.tablename";
+ public static final String CONFIG_TABLE_NAME = CONFIG_PREFIX + ".hbase.tablename";
public static final String DEFAULT_TABLE_NAME = "ATLAS_ENTITY_AUDIT_EVENTS";
-
- public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt");
- public static final byte[] COLUMN_ACTION = Bytes.toBytes("a");
- public static final byte[] COLUMN_DETAIL = Bytes.toBytes("d");
- public static final byte[] COLUMN_USER = Bytes.toBytes("u");
- public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("f");
+ public static final byte[] COLUMN_FAMILY = Bytes.toBytes("dt");
+ public static final byte[] COLUMN_ACTION = Bytes.toBytes("a");
+ public static final byte[] COLUMN_DETAIL = Bytes.toBytes("d");
+ public static final byte[] COLUMN_USER = Bytes.toBytes("u");
+ public static final byte[] COLUMN_DEFINITION = Bytes.toBytes("f");
+ public static final byte[] COLUMN_TYPE = Bytes.toBytes("t");
private TableName tableName;
private Connection connection;
+ private final AtlasInstanceConverter instanceConverter;
+
+ @Inject
+ public HBaseBasedAuditRepository(AtlasInstanceConverter instanceConverter) {
+ this.instanceConverter = instanceConverter;
+ }
/**
* Add events to the event repository
@@ -101,23 +125,32 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
}
Table table = null;
+
try {
- table = connection.getTable(tableName);
+ table = connection.getTable(tableName);
List<Put> puts = new ArrayList<>(events.size());
for (int index = 0; index < events.size(); index++) {
EntityAuditEvent event = events.get(index);
- LOG.debug("Adding entity audit event {}", event);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding entity audit event {}", event);
+ }
+
Put put = new Put(getKey(event.getEntityId(), event.getTimestamp(), index));
+
addColumn(put, COLUMN_ACTION, event.getAction());
addColumn(put, COLUMN_USER, event.getUser());
addColumn(put, COLUMN_DETAIL, event.getDetails());
+ addColumn(put, COLUMN_TYPE, ENTITY_AUDIT_V1);
+
if (persistEntityDefinition) {
addColumn(put, COLUMN_DEFINITION, event.getEntityDefinitionString());
}
+
puts.add(put);
}
+
table.put(puts);
} catch (IOException e) {
throw new AtlasException(e);
@@ -150,6 +183,7 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
addColumn(put, COLUMN_ACTION, event.getAction());
addColumn(put, COLUMN_USER, event.getUser());
addColumn(put, COLUMN_DETAIL, event.getDetails());
+ addColumn(put, COLUMN_TYPE, ENTITY_AUDIT_V2);
if (persistEntityDefinition) {
addColumn(put, COLUMN_DEFINITION, event.getEntity());
@@ -216,16 +250,14 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
if (!event.getEntityId().equals(entityId)) {
continue;
}
+
event.setUser(getResultString(result, COLUMN_USER));
- event.setAction(EntityAuditAction.fromString(getResultString(result, COLUMN_ACTION)));
- event.setDetails(getResultString(result, COLUMN_DETAIL));
+ event.setAction(EntityAuditActionV2.fromString(getResultString(result, COLUMN_ACTION)));
+ event.setDetails(getEntityDetails(result));
+ event.setType(getAuditType(result));
if (persistEntityDefinition) {
- String colDef = getResultString(result, COLUMN_DEFINITION);
-
- if (colDef != null) {
- event.setEntityDefinition(colDef);
- }
+ event.setEntityDefinition(getEntityDefinition(result));
}
events.add(event);
@@ -248,6 +280,91 @@ public class HBaseBasedAuditRepository extends AbstractStorageBasedAuditReposito
}
}
+ private String getEntityDefinition(Result result) throws AtlasBaseException {
+ String ret = getResultString(result, COLUMN_DEFINITION);
+
+ if (getAuditType(result) != ENTITY_AUDIT_V2) {
+ Referenceable referenceable = AtlasType.fromV1Json(ret, Referenceable.class);
+ AtlasEntity entity = toAtlasEntity(referenceable);
+
+ ret = AtlasType.toJson(entity);
+ }
+
+ return ret;
+ }
+
+ private String getEntityDetails(Result result) throws AtlasBaseException {
+ String ret;
+
+ if (getAuditType(result) == ENTITY_AUDIT_V2) {
+ ret = getResultString(result, COLUMN_DETAIL);
+ } else {
+ // convert v1 audit detail to v2
+ ret = getV2Details(result);
+ }
+
+ return ret;
+ }
+
+ private EntityAuditType getAuditType(Result result) {
+ String typeString = getResultString(result, COLUMN_TYPE);
+ EntityAuditType ret = (typeString != null) ? EntityAuditType.valueOf(typeString) : ENTITY_AUDIT_V1;
+
+ return ret;
+ }
+
+ private String getV2Details(Result result) throws AtlasBaseException {
+ String ret = null;
+ String v1DetailsWithPrefix = getResultString(result, COLUMN_DETAIL);
+
+ if (StringUtils.isNotEmpty(v1DetailsWithPrefix)) {
+ EntityAuditAction v1AuditAction = EntityAuditAction.fromString(getResultString(result, COLUMN_ACTION));
+ String v1AuditPrefix = EntityAuditListener.getV1AuditPrefix(v1AuditAction);
+ String[] split = v1DetailsWithPrefix.split(v1AuditPrefix);
+
+ if (ArrayUtils.isNotEmpty(split) && split.length == 2) {
+ String v1AuditDetails = split[1];
+ Referenceable referenceable = AtlasType.fromV1Json(v1AuditDetails, Referenceable.class);
+ String v2Json = (referenceable != null) ? toV2Json(referenceable, v1AuditAction) : v1AuditDetails;
+
+ if (v2Json != null) {
+ ret = getV2AuditPrefix(v1AuditAction) + v2Json;
+ }
+ } else {
+ ret = v1DetailsWithPrefix;
+ }
+ }
+
+ return ret;
+ }
+
+ private String toV2Json(Referenceable referenceable, EntityAuditAction action) throws AtlasBaseException {
+ String ret;
+
+ if (action == TAG_ADD || action == TAG_UPDATE || action == TAG_DELETE) {
+ AtlasClassification classification = instanceConverter.toAtlasClassification(referenceable);
+
+ ret = AtlasType.toJson(classification);
+ } else {
+ AtlasEntity entity = toAtlasEntity(referenceable);
+
+ ret = AtlasType.toJson(entity);
+ }
+
+ return ret;
+ }
+
+ private AtlasEntity toAtlasEntity(Referenceable referenceable) throws AtlasBaseException {
+ AtlasEntity ret = null;
+ AtlasEntitiesWithExtInfo entitiesWithExtInfo = instanceConverter.toAtlasEntity(referenceable);
+
+ if (entitiesWithExtInfo != null && CollectionUtils.isNotEmpty(entitiesWithExtInfo.getEntities())) {
+ ret = entitiesWithExtInfo.getEntities().get(0);
+ }
+
+ return ret;
+ }
+
private <T> void addColumn(Put put, byte[] columnName, T columnValue) {
if (columnValue != null && !columnValue.toString().isEmpty()) {
put.addColumn(COLUMN_FAMILY, columnName, Bytes.toBytes(columnValue.toString()));
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
index 7eda7e2..7d7e780 100644
--- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java
@@ -348,7 +348,7 @@ public class AtlasInstanceConverter {
return ret;
}
- private EntityAuditEvent.EntityAuditAction getV1AuditAction(EntityAuditEventV2.EntityAuditAction v2AuditAction) {
+ private EntityAuditEvent.EntityAuditAction getV1AuditAction(EntityAuditEventV2.EntityAuditActionV2 v2AuditAction) {
switch (v2AuditAction) {
case ENTITY_CREATE:
return EntityAuditEvent.EntityAuditAction.ENTITY_CREATE;
@@ -379,32 +379,32 @@ public class AtlasInstanceConverter {
return null;
}
- private EntityAuditEventV2.EntityAuditAction getV2AuditAction(EntityAuditEvent.EntityAuditAction v1AuditAction) {
+ private EntityAuditEventV2.EntityAuditActionV2 getV2AuditAction(EntityAuditEvent.EntityAuditAction v1AuditAction) {
switch (v1AuditAction) {
case ENTITY_CREATE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_CREATE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_CREATE;
case ENTITY_UPDATE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_UPDATE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE;
case ENTITY_DELETE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_DELETE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_DELETE;
case ENTITY_IMPORT_CREATE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_CREATE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_CREATE;
case ENTITY_IMPORT_UPDATE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_UPDATE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_UPDATE;
case ENTITY_IMPORT_DELETE:
- return EntityAuditEventV2.EntityAuditAction.ENTITY_IMPORT_DELETE;
+ return EntityAuditEventV2.EntityAuditActionV2.ENTITY_IMPORT_DELETE;
case TAG_ADD:
- return EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_ADD;
+ return EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_ADD;
case TAG_DELETE:
- return EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_DELETE;
+ return EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_DELETE;
case TAG_UPDATE:
- return EntityAuditEventV2.EntityAuditAction.CLASSIFICATION_UPDATE;
+ return EntityAuditEventV2.EntityAuditActionV2.CLASSIFICATION_UPDATE;
case PROPAGATED_TAG_ADD:
- return EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_ADD;
+ return EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
case PROPAGATED_TAG_DELETE:
- return EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_DELETE;
+ return EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_DELETE;
case PROPAGATED_TAG_UPDATE:
- return EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_UPDATE;
+ return EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_UPDATE;
}
return null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
index 46b17c0..083600e 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/AtlasEntityChangeNotifier.java
@@ -24,7 +24,7 @@ import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.EntityChangeListenerV2;
-import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction;
+import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
@@ -57,8 +57,8 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_ADD;
-import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditAction.PROPAGATED_CLASSIFICATION_DELETE;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_ADD;
+import static org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2.PROPAGATED_CLASSIFICATION_DELETE;
import static org.apache.atlas.util.AtlasRepositoryConfiguration.isV2EntityNotificationEnabled;
@@ -201,7 +201,7 @@ public class AtlasEntityChangeNotifier {
notifyPropagatedEntities(removedPropagations, PROPAGATED_CLASSIFICATION_DELETE);
}
- private void notifyPropagatedEntities(Map<String, List<AtlasClassification>> entityPropagationMap, EntityAuditAction action) throws AtlasBaseException {
+ private void notifyPropagatedEntities(Map<String, List<AtlasClassification>> entityPropagationMap, EntityAuditActionV2 action) throws AtlasBaseException {
if (MapUtils.isEmpty(entityPropagationMap) || action == null) {
return;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bf037c92/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
index 87ca849..aa175a2 100644
--- a/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
+++ b/repository/src/test/java/org/apache/atlas/repository/audit/AuditRepositoryTestBase.java
@@ -110,7 +110,7 @@ public class AuditRepositoryTestBase {
@Test
public void testAddEventsV2() throws Exception {
EntityAuditEventV2 event = new EntityAuditEventV2(rand(), System.currentTimeMillis(), "u1",
- EntityAuditEventV2.EntityAuditAction.ENTITY_CREATE, "d1", new AtlasEntity(rand()));
+ EntityAuditEventV2.EntityAuditActionV2.ENTITY_CREATE, "d1", new AtlasEntity(rand()));
eventRepository.putEventsV2(event);
@@ -131,12 +131,12 @@ public class AuditRepositoryTestBase {
for (int i = 0; i < 3; i++) {
//Add events for both ids
- EntityAuditEventV2 event = new EntityAuditEventV2(id2, ts - i, "user" + i, EntityAuditEventV2.EntityAuditAction.ENTITY_UPDATE, "details" + i, entity);
+ EntityAuditEventV2 event = new EntityAuditEventV2(id2, ts - i, "user" + i, EntityAuditEventV2.EntityAuditActionV2.ENTITY_UPDATE, "details" + i, entity);
eventRepository.putEventsV2(event);
expectedEvents.add(event);
- eventRepository.putEventsV2(new EntityAuditEventV2(id1, ts - i, "user" + i, EntityAuditEventV2.EntityAuditAction.ENTITY_DELETE, "details" + i, entity));
- eventRepository.putEventsV2(new EntityAuditEventV2(id3, ts - i, "user" + i, EntityAuditEventV2.EntityAuditAction.ENTITY_CREATE, "details" + i, entity));
+ eventRepository.putEventsV2(new EntityAuditEventV2(id1, ts - i, "user" + i, EntityAuditEventV2.EntityAuditActionV2.ENTITY_DELETE, "details" + i, entity));
+ eventRepository.putEventsV2(new EntityAuditEventV2(id3, ts - i, "user" + i, EntityAuditEventV2.EntityAuditActionV2.ENTITY_CREATE, "details" + i, entity));
}
//Use ts for which there is no event - ts + 2
[7/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
ATLAS-2637: migration-import updates for changes in collection attribute storage
Signed-off-by: Madhan Neethiraj <ma...@apache.org>
(cherry picked from commit 47ec9f7a8cc02346ac167849374bf088f55be74a)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/5cbfca24
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/5cbfca24
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/5cbfca24
Branch: refs/heads/branch-1.0
Commit: 5cbfca240311f0cb5da6e85c87115824f7b34235
Parents: e5e4558
Author: Ashutosh Mestry <am...@hortonworks.com>
Authored: Tue May 22 18:24:57 2018 -0700
Committer: Madhan Neethiraj <ma...@apache.org>
Committed: Tue May 22 23:11:43 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/atlas/AtlasConstants.java | 2 +-
.../atlas/repository/graphdb/AtlasGraph.java | 3 +-
.../graphdb/janus/AtlasJanusGraph.java | 262 +-
.../graphdb/janus/AtlasJanusGraphDatabase.java | 10 +-
.../janus/migration/AtlasGraphSONReader.java | 100 +-
.../janus/migration/ElementProcessors.java | 423 ++
.../janus/migration/GraphSONUtility.java | 107 +-
.../janus/migration/JsonNodeProcessManager.java | 1 +
.../janus/migration/MappedElementCache.java | 44 -
.../janus/migration/PostProcessManager.java | 100 +-
.../migration/RelationshipCacheGenerator.java | 98 +
.../janus/migration/RelationshipTypeCache.java | 37 -
.../migration/TypesWithCollectionsFinder.java | 123 +
.../postProcess/PostProcessListProperty.java | 68 +
.../graphdb/janus/migration/BaseUtils.java | 107 +-
.../GraphSONUtilityPostProcessTest.java | 95 -
.../janus/migration/GraphSONUtilityTest.java | 266 +-
.../janus/migration/JsonNodeParsersTest.java | 16 +-
.../janus/migration/MappedElementCacheTest.java | 26 +-
.../migration/PostProcessListPropertyTest.java | 142 +
.../janus/src/test/resources/col-2-legacy.json | 73 +
.../janus/src/test/resources/col-3-legacy.json | 86 +
.../src/test/resources/edge-legacy-col.json | 27 +
.../src/test/resources/edge-legacy-col2.json | 27 +
.../src/test/resources/edge-legacy-col3.json | 27 +
.../src/test/resources/edge-legacy-col4.json | 27 +
.../src/test/resources/edge-legacy-process.json | 27 +
.../src/test/resources/edge-legacy-tag.json | 27 +
.../src/test/resources/lineage-v-98312.json | 79 +
.../src/test/resources/table-v-147504.json | 83 +-
.../janus/src/test/resources/tag-163856752.json | 35 +
.../apache/atlas/store/AtlasTypeDefStore.java | 6 -
.../test/java/org/apache/atlas/TestUtilsV2.java | 1 -
.../migration/DataMigrationService.java | 48 +-
.../migration/RelationshipCacheGenerator.java | 73 -
.../graph/v1/AtlasTypeDefGraphStoreV1.java | 7 -
.../migration/ComplexAttributesTest.java | 61 +
.../migration/HiveParititionTest.java | 13 +-
.../repository/migration/HiveStocksTest.java | 3 +-
.../migration/MigrationBaseAsserts.java | 28 +-
.../RelationshipCacheGeneratorTest.java | 96 +
.../migration/RelationshipMappingTest.java | 87 -
.../TypesWithCollectionsFinderTest.java | 84 +
.../store/graph/v1/AtlasEntityTestBase.java | 5 +-
.../complex-attr_db/atlas-migration-data.json | 5569 ++++++++++++++++++
.../atlas-migration-typesdef.json | 2303 ++++++++
.../parts_db/atlas-migration-data.json | 3990 ++++++++++++-
47 files changed, 14107 insertions(+), 815 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/common/src/main/java/org/apache/atlas/AtlasConstants.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/atlas/AtlasConstants.java b/common/src/main/java/org/apache/atlas/AtlasConstants.java
index 2b92e6e..2b9f411 100644
--- a/common/src/main/java/org/apache/atlas/AtlasConstants.java
+++ b/common/src/main/java/org/apache/atlas/AtlasConstants.java
@@ -35,6 +35,6 @@ public final class AtlasConstants {
public static final int ATLAS_SHUTDOWN_HOOK_PRIORITY = 30;
public static final String DEFAULT_TYPE_VERSION = "1.0";
- public static final String ATLAS_MIGRATION_MODE_FILENAME = "atlas.migration.mode.filename";
+ public static final String ATLAS_MIGRATION_MODE_FILENAME = "atlas.migration.data.filename";
public static final String ATLAS_SERVICES_ENABLED = "atlas.services.enabled";
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
----------------------------------------------------------------------
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index 607baf6..e5316d8 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -30,6 +30,7 @@ import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.groovy.GroovyExpression;
import org.apache.atlas.model.impexp.MigrationStatus;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
/**
* Represents a graph.
@@ -320,7 +321,7 @@ public interface AtlasGraph<V, E> {
*/
boolean isMultiProperty(String name);
- void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException;
+ void importLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException;
MigrationStatus getMigrationStatus();
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index c0b9c17..b4d6b33 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -34,10 +34,10 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasSchemaViolationException;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.graphdb.GremlinVersion;
-import org.apache.atlas.repository.graphdb.janus.migration.ReaderStatusManager;
import org.apache.atlas.repository.graphdb.janus.query.AtlasJanusGraphQuery;
import org.apache.atlas.repository.graphdb.utils.IteratorToIterableAdapter;
import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.groovy.CompilerCustomizerProvider;
import org.apache.tinkerpop.gremlin.groovy.DefaultImportCustomizerProvider;
@@ -84,26 +84,20 @@ import static org.apache.atlas.repository.Constants.INDEX_SEARCH_VERTEX_PREFIX_P
*/
public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusEdge> {
- private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
-
private static Configuration APPLICATION_PROPERTIES = null;
- private final class ConvertGremlinValueFunction implements Function<Object, Object> {
- @Override
- public Object apply(Object input) {
- return convertGremlinValue(input);
- }
- }
-
- private final Set<String> multiProperties;
+ private final ConvertGremlinValueFunction GREMLIN_VALUE_CONVERSION_FUNCTION = new ConvertGremlinValueFunction();
+ private final Set<String> multiProperties = new HashSet<>();
public AtlasJanusGraph() {
//determine multi-properties once at startup
JanusGraphManagement mgmt = null;
+
try {
mgmt = AtlasJanusGraphDatabase.getGraphInstance().openManagement();
+
Iterable<PropertyKey> keys = mgmt.getRelationTypes(PropertyKey.class);
- multiProperties = new HashSet<>();
+
for (PropertyKey key : keys) {
if (key.cardinality() != Cardinality.SINGLE) {
multiProperties.add(key.name());
@@ -118,13 +112,13 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> addEdge(AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> outVertex,
- AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> inVertex,
- String edgeLabel) {
-
+ AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> inVertex,
+ String edgeLabel) {
try {
- Vertex oV = outVertex.getV().getWrappedElement();
- Vertex iV = inVertex.getV().getWrappedElement();
- Edge edge = oV.addEdge(edgeLabel, iV);
+ Vertex oV = outVertex.getV().getWrappedElement();
+ Vertex iV = inVertex.getV().getWrappedElement();
+ Edge edge = oV.addEdge(edgeLabel, iV);
+
return GraphDbObjectFactory.createEdge(this, edge);
} catch (SchemaViolationException e) {
throw new AtlasSchemaViolationException(e);
@@ -139,42 +133,43 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> getEdge(String edgeId) {
Iterator<Edge> it = getGraph().edges(edgeId);
- Edge e = getSingleElement(it, edgeId);
+ Edge e = getSingleElement(it, edgeId);
+
return GraphDbObjectFactory.createEdge(this, e);
}
@Override
public void removeEdge(AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> edge) {
-
Edge wrapped = edge.getE().getWrappedElement();
- wrapped.remove();
+ wrapped.remove();
}
@Override
public void removeVertex(AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> vertex) {
Vertex wrapped = vertex.getV().getWrappedElement();
+
wrapped.remove();
}
@Override
public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> getEdges() {
-
Iterator<Edge> edges = getGraph().edges();
- return wrapEdges(edges);
+ return wrapEdges(edges);
}
@Override
public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> getVertices() {
-
Iterator<Vertex> vertices = getGraph().vertices();
+
return wrapVertices(vertices);
}
@Override
public AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> addVertex() {
Vertex result = getGraph().addVertex();
+
return GraphDbObjectFactory.createVertex(this, result);
}
@@ -221,73 +216,32 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return getIndexKeys(Vertex.class);
}
- private Set<String> getIndexKeys(Class<? extends Element> janusGraphElementClass) {
-
- JanusGraphManagement mgmt = getGraph().openManagement();
- Iterable<JanusGraphIndex> indices = mgmt.getGraphIndexes(janusGraphElementClass);
- Set<String> result = new HashSet<String>();
- for (JanusGraphIndex index : indices) {
- result.add(index.name());
- }
- mgmt.commit();
- return result;
-
- }
-
@Override
public AtlasVertex<AtlasJanusVertex, AtlasJanusEdge> getVertex(String vertexId) {
- Iterator<Vertex> it = getGraph().vertices(vertexId);
- Vertex vertex = getSingleElement(it, vertexId);
- return GraphDbObjectFactory.createVertex(this, vertex);
- }
+ Iterator<Vertex> it = getGraph().vertices(vertexId);
+ Vertex vertex = getSingleElement(it, vertexId);
- public static <T> T getSingleElement(Iterator<T> it, String id) {
- if (!it.hasNext()) {
- return null;
- }
- T element = it.next();
- if (it.hasNext()) {
- throw new RuntimeException("Multiple items were found with the id " + id);
- }
- return element;
+ return GraphDbObjectFactory.createVertex(this, vertex);
}
@Override
public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> getVertices(String key, Object value) {
AtlasGraphQuery<AtlasJanusVertex, AtlasJanusEdge> query = query();
- query.has(key, value);
- return query.vertices();
- }
- private Object convertGremlinValue(Object rawValue) {
+ query.has(key, value);
- if (rawValue instanceof Vertex) {
- return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
- } else if (rawValue instanceof Edge) {
- return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
- } else if (rawValue instanceof Map) {
- Map<String,Object> rowValue = (Map<String,Object>)rawValue;
- return Maps.transformValues(rowValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
- } else if (rawValue instanceof ImmutablePath) {
- ImmutablePath path = (ImmutablePath) rawValue;
- return convertGremlinValue(path.objects());
- }
- else if (rawValue instanceof List) {
- return Lists.transform((List)rawValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
- } else if (rawValue instanceof Collection) {
- throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
- }
- return rawValue;
+ return query.vertices();
}
@Override
public GremlinVersion getSupportedGremlinVersion() {
-
return GremlinVersion.THREE;
}
+
@Override
public void clear() {
JanusGraph graph = getGraph();
+
if (graph.isOpen()) {
// only a shut down graph can be cleared
graph.close();
@@ -295,7 +249,8 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
try {
JanusGraphFactory.drop(graph);
- } catch (BackendException ignoreEx) {}
+ } catch (BackendException ignoreEx) {
+ }
}
private JanusGraph getGraph() {
@@ -304,25 +259,27 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public void exportToGson(OutputStream os) throws IOException {
-
- GraphSONMapper mapper = getGraph().io(IoCore.graphson()).mapper().create();
+ GraphSONMapper mapper = getGraph().io(IoCore.graphson()).mapper().create();
GraphSONWriter.Builder builder = GraphSONWriter.build();
+
builder.mapper(mapper);
+
GraphSONWriter writer = builder.create();
+
writer.writeGraph(os, getGraph());
}
@Override
public GremlinGroovyScriptEngine getGremlinScriptEngine() {
- Set<String> extraImports = new HashSet<String>();
- extraImports.add(java.util.function.Function.class.getName());
-
+ Set<String> extraImports = new HashSet<String>();
Set<String> extraStaticImports = new HashSet<String>();
+
+ extraImports.add(java.util.function.Function.class.getName());
extraStaticImports.add(P.class.getName() + ".*");
extraStaticImports.add(__.class.getName() + ".*");
- CompilerCustomizerProvider provider = new DefaultImportCustomizerProvider(extraImports, extraStaticImports);
- GremlinGroovyScriptEngine scriptEngine = new GremlinGroovyScriptEngine(provider);
+ CompilerCustomizerProvider provider = new DefaultImportCustomizerProvider(extraImports, extraStaticImports);
+ GremlinGroovyScriptEngine scriptEngine = new GremlinGroovyScriptEngine(provider);
return scriptEngine;
}
@@ -341,38 +298,20 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
@Override
public Object executeGremlinScript(String query, boolean isPath) throws AtlasBaseException {
Object result = executeGremlinScript(query);
- return convertGremlinValue(result);
- }
-
- private Object executeGremlinScript(String gremlinQuery) throws AtlasBaseException {
- GremlinGroovyScriptEngine scriptEngine = getGremlinScriptEngine();
-
- try {
- Bindings bindings = scriptEngine.createBindings();
-
- bindings.put("graph", getGraph());
- bindings.put("g", getGraph().traversal());
-
- Object result = scriptEngine.eval(gremlinQuery, bindings);
- return result;
- } catch (ScriptException e) {
- throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e, gremlinQuery);
- } finally {
- releaseGremlinScriptEngine(scriptEngine);
- }
+ return convertGremlinValue(result);
}
@Override
- public Object executeGremlinScript(ScriptEngine scriptEngine,
- Map<? extends String, ? extends Object> userBindings, String query, boolean isPath)
- throws ScriptException {
+ public Object executeGremlinScript(ScriptEngine scriptEngine, Map<? extends String, ? extends Object> userBindings,
+ String query, boolean isPath) throws ScriptException {
Bindings bindings = scriptEngine.createBindings();
bindings.putAll(userBindings);
bindings.put("g", getGraph().traversal());
Object result = scriptEngine.eval(query, bindings);
+
return convertGremlinValue(result);
}
@@ -402,39 +341,42 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return expr;
}
- public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterator<? extends Vertex> it) {
- Iterable<? extends Vertex> iterable = new IteratorToIterableAdapter<>(it);
- return wrapVertices(iterable);
+ @Override
+ public boolean isMultiProperty(String propertyName) {
+ return multiProperties.contains(propertyName);
}
- public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterable<? extends Vertex> it) {
+ @Override
+ public void importLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException {
+ AtlasJanusGraphDatabase.loadLegacyGraphSON(typeRegistry, fs);
+ }
- return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)).collect(Collectors.toList());
+ @Override
+ public MigrationStatus getMigrationStatus() {
+ return AtlasJanusGraphDatabase.getMigrationStatus();
+ }
+ public Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterable<? extends Vertex> it) {
+ return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createVertex(AtlasJanusGraph.this, input)).collect(Collectors.toList());
}
public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterator<? extends Edge> it) {
Iterable<? extends Edge> iterable = new IteratorToIterableAdapter<>(it);
+
return wrapEdges(iterable);
}
public Iterable<AtlasEdge<AtlasJanusVertex, AtlasJanusEdge>> wrapEdges(Iterable<? extends Edge> it) {
-
return StreamSupport.stream(it.spliterator(), false).map(input -> GraphDbObjectFactory.createEdge(AtlasJanusGraph.this, input)).collect(Collectors.toList());
-
- }
-
- @Override
- public boolean isMultiProperty(String propertyName) {
- return multiProperties.contains(propertyName);
}
public void addMultiProperties(Set<String> names) {
multiProperties.addAll(names);
}
- public String getIndexQueryPrefix() {
- String ret;
+
+ private String getIndexQueryPrefix() {
+ final String ret;
initApplicationProperties();
@@ -447,6 +389,82 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
return ret;
}
+ private Iterable<AtlasVertex<AtlasJanusVertex, AtlasJanusEdge>> wrapVertices(Iterator<? extends Vertex> it) {
+ Iterable<? extends Vertex> iterable = new IteratorToIterableAdapter<>(it);
+
+ return wrapVertices(iterable);
+ }
+
+ private static <T> T getSingleElement(Iterator<T> it, String id) {
+ if (!it.hasNext()) {
+ return null;
+ }
+
+ T element = it.next();
+
+ if (it.hasNext()) {
+ throw new RuntimeException("Multiple items were found with the id " + id);
+ }
+
+ return element;
+ }
+
+ private Object convertGremlinValue(Object rawValue) {
+ if (rawValue instanceof Vertex) {
+ return GraphDbObjectFactory.createVertex(this, (Vertex) rawValue);
+ } else if (rawValue instanceof Edge) {
+ return GraphDbObjectFactory.createEdge(this, (Edge) rawValue);
+ } else if (rawValue instanceof Map) {
+ Map<String,Object> rowValue = (Map<String,Object>)rawValue;
+
+ return Maps.transformValues(rowValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
+ } else if (rawValue instanceof ImmutablePath) {
+ ImmutablePath path = (ImmutablePath) rawValue;
+
+ return convertGremlinValue(path.objects());
+ } else if (rawValue instanceof List) {
+ return Lists.transform((List)rawValue, GREMLIN_VALUE_CONVERSION_FUNCTION);
+ } else if (rawValue instanceof Collection) {
+ throw new UnsupportedOperationException("Unhandled collection type: " + rawValue.getClass());
+ }
+
+ return rawValue;
+ }
+
+ private Set<String> getIndexKeys(Class<? extends Element> janusGraphElementClass) {
+ JanusGraphManagement mgmt = getGraph().openManagement();
+ Iterable<JanusGraphIndex> indices = mgmt.getGraphIndexes(janusGraphElementClass);
+ Set<String> result = new HashSet<String>();
+
+ for (JanusGraphIndex index : indices) {
+ result.add(index.name());
+ }
+
+ mgmt.commit();
+
+ return result;
+
+ }
+
+ private Object executeGremlinScript(String gremlinQuery) throws AtlasBaseException {
+ GremlinGroovyScriptEngine scriptEngine = getGremlinScriptEngine();
+
+ try {
+ Bindings bindings = scriptEngine.createBindings();
+
+ bindings.put("graph", getGraph());
+ bindings.put("g", getGraph().traversal());
+
+ Object result = scriptEngine.eval(gremlinQuery, bindings);
+
+ return result;
+ } catch (ScriptException e) {
+ throw new AtlasBaseException(AtlasErrorCode.GREMLIN_SCRIPT_EXECUTION_FAILED, e, gremlinQuery);
+ } finally {
+ releaseGremlinScriptEngine(scriptEngine);
+ }
+ }
+
private void initApplicationProperties() {
if (APPLICATION_PROPERTIES == null) {
try {
@@ -457,13 +475,11 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
}
- @Override
- public void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
- AtlasJanusGraphDatabase.loadLegacyGraphSON(relationshipCache, fs);
- }
- @Override
- public MigrationStatus getMigrationStatus() {
- return AtlasJanusGraphDatabase.getMigrationStatus();
+ private final class ConvertGremlinValueFunction implements Function<Object, Object> {
+ @Override
+ public Object apply(Object input) {
+ return convertGremlinValue(input);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
index 16aecd5..c9d6067 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraphDatabase.java
@@ -26,10 +26,12 @@ import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.GraphDatabase;
import org.apache.atlas.repository.graphdb.janus.migration.AtlasGraphSONReader;
import org.apache.atlas.repository.graphdb.janus.migration.ReaderStatusManager;
+import org.apache.atlas.repository.graphdb.janus.migration.ElementProcessors;
import org.apache.atlas.repository.graphdb.janus.serializer.BigDecimalSerializer;
import org.apache.atlas.repository.graphdb.janus.serializer.BigIntegerSerializer;
import org.apache.atlas.repository.graphdb.janus.serializer.TypeCategorySerializer;
import org.apache.atlas.runner.LocalSolrRunner;
+import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.utils.AtlasPerfTracer;
import org.apache.commons.configuration.Configuration;
@@ -47,7 +49,6 @@ import java.io.InputStream;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
-import java.util.Map;
/**
* Default implementation for Graph Provider that doles out JanusGraph.
@@ -231,7 +232,7 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
return ret;
}
- public static void loadLegacyGraphSON(Map<String, String> relationshipCache, InputStream fs) throws AtlasBaseException {
+ public static void loadLegacyGraphSON(AtlasTypeRegistry typeRegistry, InputStream fs) throws AtlasBaseException {
AtlasPerfTracer perf = null;
try {
@@ -242,9 +243,10 @@ public class AtlasJanusGraphDatabase implements GraphDatabase<AtlasJanusVertex,
}
AtlasGraphSONReader legacyGraphSONReader = AtlasGraphSONReader.build().
- relationshipCache(relationshipCache).
+ relationshipCache(new ElementProcessors(typeRegistry)).
schemaDB(getGraphInstance()).
- bulkLoadingDB(getBulkLoadingGraphInstance()).create();
+ bulkLoadingDB(getBulkLoadingGraphInstance()).
+ create();
legacyGraphSONReader.readGraph(fs);
} catch (Exception ex) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
index ae119b0..2d5bd8a 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/AtlasGraphSONReader.java
@@ -37,44 +37,46 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
public final class AtlasGraphSONReader {
private static final Logger LOG = LoggerFactory.getLogger(AtlasGraphSONReader.class);
- private final ObjectMapper mapper;
- private final RelationshipTypeCache relationshipCache;
- private final Graph graph;
- private final Graph bulkLoadGraph;
- private final int numWorkers;
- private final int batchSize;
- private final long suppliedStartIndex;
- private final String[] propertiesToPostProcess;
- private GraphSONUtility graphSONUtility;
- private ReaderStatusManager readerStatusManager;
- private AtomicLong counter;
-
- private AtlasGraphSONReader(ObjectMapper mapper, Map<String, String> relationshipLookup, Graph graph,
- Graph bulkLoadGraph, String[] propertiesToPostProcess, int numWorkers, int batchSize, long suppliedStartIndex) {
+ private static String APPLICATION_PROPERTY_MIGRATION_START_INDEX = "atlas.migration.mode.start.index";
+ private static String APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS = "atlas.migration.mode.workers";
+ private static String APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE = "atlas.migration.mode.batch.size";
+
+ private final ObjectMapper mapper;
+ private final ElementProcessors relationshipCache;
+ private final Graph graph;
+ private final Graph bulkLoadGraph;
+ private final int numWorkers;
+ private final int batchSize;
+ private final long suppliedStartIndex;
+ private final GraphSONUtility graphSONUtility;
+ private ReaderStatusManager readerStatusManager;
+ private AtomicLong counter;
+
+ private AtlasGraphSONReader(ObjectMapper mapper, ElementProcessors relationshipLookup, Graph graph,
+ Graph bulkLoadGraph, int numWorkers, int batchSize, long suppliedStartIndex) {
this.mapper = mapper;
- this.relationshipCache = new RelationshipTypeCache(relationshipLookup);
+ this.relationshipCache = relationshipLookup;
this.graph = graph;
this.bulkLoadGraph = bulkLoadGraph;
this.numWorkers = numWorkers;
this.batchSize = batchSize;
this.suppliedStartIndex = suppliedStartIndex;
- this.propertiesToPostProcess = propertiesToPostProcess;
+ this.graphSONUtility = new GraphSONUtility(relationshipCache);
}
public void readGraph(final InputStream inputStream) throws IOException {
- counter = new AtomicLong(0);
- graphSONUtility = new GraphSONUtility(relationshipCache);
+ counter = new AtomicLong(0);
final long startIndex = initStatusManager();
final JsonFactory factory = mapper.getFactory();
LOG.info("AtlasGraphSONReader.readGraph: numWorkers: {}: batchSize: {}: startIndex: {}", numWorkers, batchSize, startIndex);
+
try (JsonParser parser = factory.createParser(inputStream)) {
if (parser.nextToken() != JsonToken.START_OBJECT) {
throw new IOException("Expected data to start with an Object");
@@ -88,6 +90,7 @@ public final class AtlasGraphSONReader {
switch (fieldName) {
case GraphSONTokensTP2.MODE:
parser.nextToken();
+
final String mode = parser.getText();
if (!mode.equals("EXTENDED")) {
@@ -180,8 +183,7 @@ public final class AtlasGraphSONReader {
LOG.info("postProcess: Starting... : counter at: {}", counter.get());
try {
- PostProcessManager.WorkItemsManager wim = PostProcessManager.create(bulkLoadGraph, graphSONUtility,
- propertiesToPostProcess, batchSize, numWorkers);
+ PostProcessManager.WorkItemsManager wim = PostProcessManager.create(bulkLoadGraph, relationshipCache.getPropertiesToPostProcess(), batchSize, numWorkers);
GraphTraversal query = bulkLoadGraph.traversal().V();
while (query.hasNext()) {
@@ -228,30 +230,32 @@ public final class AtlasGraphSONReader {
}
}
- public static Builder build() throws AtlasException {
+ public static Builder build() {
return new Builder();
}
public final static class Builder {
- private int batchSize = 500;
- private Map<String, String> relationshipCache;
- private Graph graph;
- private Graph bulkLoadGraph;
- private int numWorkers;
- private long suppliedStartIndex;
- private String[] propertiesToPostProcess;
+ private int batchSize = 500;
+ private ElementProcessors relationshipCache;
+ private Graph graph;
+ private Graph bulkLoadGraph;
+ private int numWorkers;
+ private long suppliedStartIndex;
private Builder() {
}
- private void setDefaults() throws AtlasException {
- this.startIndex(ApplicationProperties.get().getLong("atlas.migration.mode.start.index", 0L))
- .numWorkers(ApplicationProperties.get().getInt("atlas.migration.mode.workers", 4))
- .batchSize(ApplicationProperties.get().getInt("atlas.migration.mode.batch.size", 3000))
- .propertiesToPostProcess(getPropertiesToPostProcess("atlas.migration.mode.postprocess.properties"));
+ private void setDefaults() {
+ try {
+ this.startIndex(ApplicationProperties.get().getLong(APPLICATION_PROPERTY_MIGRATION_START_INDEX, 0L))
+ .numWorkers(ApplicationProperties.get().getInt(APPLICATION_PROPERTY_MIGRATION_NUMER_OF_WORKERS, 4))
+ .batchSize(ApplicationProperties.get().getInt(APPLICATION_PROPERTY_MIGRATION_BATCH_SIZE, 3000));
+ } catch (AtlasException ex) {
+ LOG.error("setDefaults: failed!", ex);
+ }
}
- public AtlasGraphSONReader create() throws AtlasException {
+ public AtlasGraphSONReader create() {
setDefaults();
if(bulkLoadGraph == null) {
bulkLoadGraph = graph;
@@ -261,10 +265,10 @@ public final class AtlasGraphSONReader {
final GraphSONMapper mapper = builder.typeInfo(TypeInfo.NO_TYPES).create();
return new AtlasGraphSONReader(mapper.createMapper(), relationshipCache, graph, bulkLoadGraph,
- propertiesToPostProcess, numWorkers, batchSize, suppliedStartIndex);
+ numWorkers, batchSize, suppliedStartIndex);
}
- public Builder relationshipCache(Map<String, String> relationshipCache) {
+ public Builder relationshipCache(ElementProcessors relationshipCache) {
this.relationshipCache = relationshipCache;
return this;
@@ -305,27 +309,5 @@ public final class AtlasGraphSONReader {
return this;
}
-
- public Builder propertiesToPostProcess(String[] list) {
- this.propertiesToPostProcess = list;
- return this;
- }
-
- private static String[] getPropertiesToPostProcess(String applicationPropertyKey) throws AtlasException {
- final String HIVE_COLUMNS_PROPERTY = "hive_table.columns";
- final String HIVE_PARTITION_KEYS_PROPERTY = "hive_table.partitionKeys";
- final String PROCESS_INPUT_PROPERTY = "Process.inputs";
- final String PROCESS_OUTPUT_PROPERTY = "Process.outputs";
- final String USER_PROFILE_OUTPUT_PROPERTY = "__AtlasUserProfile.savedSearches";
-
- String[] defaultProperties = new String[] { HIVE_COLUMNS_PROPERTY, HIVE_PARTITION_KEYS_PROPERTY,
- PROCESS_INPUT_PROPERTY, PROCESS_OUTPUT_PROPERTY,
- USER_PROFILE_OUTPUT_PROPERTY
- };
-
- String[] userDefinedList = ApplicationProperties.get().getStringArray(applicationPropertyKey);
-
- return (userDefinedList == null || userDefinedList.length == 0) ? defaultProperties : userDefinedList;
- }
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
new file mode 100644
index 0000000..f51080a
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/ElementProcessors.java
@@ -0,0 +1,423 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.commons.lang.StringUtils;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_INDEX_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.ATTRIBUTE_KEY_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
+import static org.apache.atlas.repository.Constants.CLASSIFICATION_VERTEX_PROPAGATE_KEY;
+import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
+import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY;
+
+public class ElementProcessors {
+ private static final Logger LOG = LoggerFactory.getLogger(ElementProcessors.class);
+
+ public static final String PRIMITIVE_MAP_CATEGORY = "MAP_PRIMITIVE";
+ public static final String NON_PRIMITIVE_MAP_CATEGORY = "MAP";
+ public static final String NON_PRIMITIVE_ARRAY_CATEGORY = "ARRAY";
+ private static final String[] NON_PRIMITIVE_KEYS = { ElementProcessors.NON_PRIMITIVE_ARRAY_CATEGORY };
+
+ private final Map<String, RelationshipCacheGenerator.TypeInfo> relationshipLookup;
+ private final Map<String, Map<String, List<String>>> postProcessMap;
+
+ private final NonPrimitiveListPropertyProcessor nonPrimitiveListPropertyProcessor = new NonPrimitiveListPropertyProcessor();
+ private final NonPrimitiveMapPropertyProcessor nonPrimitiveMapPropertyProcessor = new NonPrimitiveMapPropertyProcessor();
+ private final PrimitiveMapPropertyProcessor primitiveMapPropertyProcessor = new PrimitiveMapPropertyProcessor();
+ private final EdgeCollectionPropertyProcessor edgeCollectionPropertyProcessor = new EdgeCollectionPropertyProcessor();
+ private final EdgeRelationshipPropertyProcessor edgeRelationshipPropertyProcessor = new EdgeRelationshipPropertyProcessor();
+
+ public ElementProcessors(AtlasTypeRegistry typeRegistry) {
+ this(RelationshipCacheGenerator.get(typeRegistry), TypesWithCollectionsFinder.getVertexPropertiesForCollectionAttributes(typeRegistry));
+ }
+
+ ElementProcessors(Map<String, RelationshipCacheGenerator.TypeInfo> lookup, Map<String, Map<String, List<String>>> postProcessMap) {
+ this.relationshipLookup = lookup;
+ this.postProcessMap = postProcessMap;
+ }
+
+ public static String[] getNonPrimitiveCategoryKeys() {
+ return NON_PRIMITIVE_KEYS;
+ }
+
+ public Map<String,Map<String, List<String>>> getPropertiesToPostProcess() {
+ return postProcessMap;
+ }
+
+ public String addIndexKeysForCollections(Vertex out, Object edgeId, String label, Map<String, Object> edgeProperties) {
+ return edgeCollectionPropertyProcessor.update(out, edgeId, label, edgeProperties);
+ }
+
+ public void processCollections(String typeNameKey, Map<String,Object> vertexProperties) {
+ if (!vertexProperties.containsKey(typeNameKey)) {
+ return;
+ }
+
+ String typeName = (String) vertexProperties.get(typeNameKey);
+
+ if (!postProcessMap.containsKey(typeName)) {
+ return;
+ }
+
+ primitiveMapPropertyProcessor.update(typeName, vertexProperties);
+ nonPrimitiveMapPropertyProcessor.update(typeName, vertexProperties);
+ nonPrimitiveListPropertyProcessor.update(typeName, vertexProperties);
+ }
+
+ public String updateEdge(Vertex in, Vertex out, Object edgeId, String label, Map<String,Object> props) {
+ return edgeRelationshipPropertyProcessor.update(in, out, edgeId, label, props);
+ }
+
+ private class NonPrimitiveMapPropertyProcessor {
+ final String category = NON_PRIMITIVE_MAP_CATEGORY;
+
+ public void update(String typeName, Map<String,Object> vertexProperties) {
+ if (!postProcessMap.containsKey(typeName)) {
+ return;
+ }
+
+ if (!postProcessMap.get(typeName).containsKey(category)) {
+ return;
+ }
+
+ List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+
+ for (String property : propertyTypeList) {
+ if (!vertexProperties.containsKey(property)) {
+ continue;
+ }
+
+ List<Object> list = (List<Object>) vertexProperties.get(property);
+
+ if (list == null) {
+ continue;
+ }
+
+ for (Object listEntry : list) {
+ String key = (String) listEntry;
+ String valueKey = getMapKey(property, key);
+
+ if (vertexProperties.containsKey(valueKey)) {
+ vertexProperties.remove(valueKey);
+ }
+ }
+
+ vertexProperties.remove(property);
+ }
+ }
+
+ private String getMapKey(String property, String key) {
+ return String.format("%s.%s", property, key);
+ }
+ }
+
+ private class PrimitiveMapPropertyProcessor {
+ final String category = PRIMITIVE_MAP_CATEGORY;
+
+ public void update(String typeName, Map<String, Object> vertexProperties) {
+ if (!postProcessMap.get(typeName).containsKey(category)) {
+ return;
+ }
+
+ List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+
+ for (String property : propertyTypeList) {
+ if (!vertexProperties.containsKey(property)) {
+ continue;
+ }
+
+ List<Object> list = (List<Object>) vertexProperties.get(property);
+
+ if (list == null) {
+ continue;
+ }
+
+ Map<String, Object> map = getAggregatedMap(vertexProperties, property, list);
+
+ vertexProperties.put(property, map);
+ }
+ }
+
+ private Map<String, Object> getAggregatedMap(Map<String, Object> vertexProperties, String property, List<Object> list) {
+ Map<String, Object> map = new HashMap<>();
+
+ for (Object listEntry : list) {
+ String key = (String) listEntry;
+ String valueKey = getMapKey(property, key);
+
+ if (vertexProperties.containsKey(valueKey)) {
+ Object value = getValueFromProperties(valueKey, vertexProperties);
+
+ vertexProperties.remove(valueKey);
+
+ map.put(key, value);
+ }
+ }
+
+ return map;
+ }
+
+ private String getMapKey(String property, String key) {
+ return String.format("%s.%s", property, key);
+ }
+
+ private Object getValueFromProperties(String key, Map<String, Object> vertexProperties) {
+ if (!vertexProperties.containsKey(key)) {
+ return null;
+ }
+
+ return vertexProperties.get(key);
+ }
+ }
+
+ private class NonPrimitiveListPropertyProcessor {
+ private final String category = NON_PRIMITIVE_ARRAY_CATEGORY;
+
+ private void update(String typeName, Map<String,Object> props) {
+ if(!postProcessMap.get(typeName).containsKey(category)) {
+ return;
+ }
+
+ List<String> propertyTypeList = postProcessMap.get(typeName).get(category);
+ for (String property : propertyTypeList) {
+ if(!props.containsKey(property)) {
+ continue;
+ }
+
+ Map<String, String> listMap = getUpdatedEdgeList(props.get(property));
+
+ if(listMap == null) {
+ continue;
+ }
+
+ props.put(property, listMap);
+ }
+ }
+
+ private Map<String, String> getUpdatedEdgeList(Object o) {
+ Map<String, String> listMap = new HashMap<>();
+
+ if(!(o instanceof List)) {
+ return null;
+ }
+
+ List list = (List) o;
+
+ for (int i = 0; i < list.size(); i++) {
+ listMap.put((String) list.get(i), Integer.toString(i));
+ }
+
+ return listMap;
+ }
+ }
+
+ private class EdgeRelationshipPropertyProcessor {
+ public String update(Vertex in, Vertex out, Object edgeId, String label, Map<String, Object> props) {
+ if(addRelationshipTypeForClassification(in, out, label, props)) {
+ label = Constants.CLASSIFICATION_LABEL;
+ } else {
+ addRelationshipTypeName(label, props);
+
+ label = addIndexKeysForCollections(out, edgeId, label, props);
+ }
+
+ addMandatoryRelationshipProperties(label, props);
+
+ return label;
+ }
+
+ private String getRelationshipTypeName(String label) {
+ return relationshipLookup.containsKey(label) ? relationshipLookup.get(label).getTypeName() : "";
+ }
+
+ private PropagateTags getDefaultPropagateValue(String label) {
+ return relationshipLookup.containsKey(label) ?
+ relationshipLookup.get(label).getPropagateTags() :
+ AtlasRelationshipDef.PropagateTags.NONE;
+ }
+
+ private boolean addRelationshipTypeForClassification(Vertex in, Vertex out, String label, Map<String, Object> props) {
+ if (in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+ String inTypeName = (String) in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+ if (StringUtils.isNotEmpty(inTypeName)) {
+ if (inTypeName.equals(label)) {
+ props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, inTypeName);
+
+ addEntityGuidToTrait(in, out);
+
+ return true;
+ }
+ } else {
+ LOG.info("Could not find typeName for trait: {}", label);
+ }
+ }
+
+ return false;
+ }
+
+ private void addEntityGuidToTrait(Vertex in, Vertex out) {
+ String entityGuid = "";
+
+ if (out.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
+ entityGuid = (String) out.property(Constants.GUID_PROPERTY_KEY).value();
+ }
+
+ if(StringUtils.isNotEmpty(entityGuid)) {
+ in.property(CLASSIFICATION_ENTITY_GUID, entityGuid);
+ in.property(CLASSIFICATION_VERTEX_PROPAGATE_KEY, false);
+ }
+ }
+
+ private void addRelationshipTypeName(String edgeLabel, Map<String, Object> props) {
+ String typeName = getRelationshipTypeName(edgeLabel);
+
+ if (StringUtils.isNotEmpty(typeName)) {
+ props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not find relationship type for: {}", edgeLabel);
+ }
+ }
+ }
+
+ private void addMandatoryRelationshipProperties(String label, Map<String, Object> props) {
+ props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
+
+ props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, String.valueOf(getDefaultPropagateValue(label)));
+ props.put(STATE_PROPERTY_KEY, "ACTIVE");
+ }
+ }
+
+ private class EdgeCollectionPropertyProcessor {
+ private static final int LABEL_INDEX = 0;
+ private static final int KEY_INDEX = 1;
+
+ public String update(Vertex out, Object edgeId, String label, Map<String, Object> edgeProperties) {
+ String[] labelKeyPair = getNonPrimitiveArrayFromLabel(out, (String) edgeId, label);
+
+ if (labelKeyPair != null) {
+ edgeProperties.put(ATTRIBUTE_INDEX_PROPERTY_KEY, Integer.valueOf(labelKeyPair[KEY_INDEX]));
+
+ return label;
+ }
+
+ labelKeyPair = getNonPrimitiveMapKeyFromLabel(out, label);
+
+ if (labelKeyPair != null) {
+ label = labelKeyPair[LABEL_INDEX];
+
+ edgeProperties.put(ATTRIBUTE_KEY_PROPERTY_KEY, labelKeyPair[KEY_INDEX]);
+ }
+
+ return label;
+ }
+
+ private String[] getNonPrimitiveArrayFromLabel(Vertex v, String edgeId, String label) {
+ if (!v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+ return null;
+ }
+
+ String typeName = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+ String propertyName = StringUtils.remove(label, Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+
+ if(!containsNonPrimitiveCollectionProperty(typeName, propertyName, NON_PRIMITIVE_ARRAY_CATEGORY)) {
+ return null;
+ }
+
+ Map<String, String> edgeIdIndexList = (Map<String, String>) v.property(propertyName).value();
+
+ if (edgeIdIndexList.containsKey(edgeId)) {
+ return getLabelKeyPair(label, edgeIdIndexList.get(edgeId));
+ }
+
+ return null;
+ }
+
+ // legacy edge label is in format: __<type name>.<key>
+ // label: in new format which is type name
+ // this method extracts:
+ // key: what remains of the legacy label string when '__' and type name are removed
+ private String[] getNonPrimitiveMapKeyFromLabel(Vertex v, String label) {
+ if (!v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+ return null;
+ }
+
+ String typeName = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+ if(!postProcessMap.containsKey(typeName)) {
+ return null;
+ }
+
+ if(!postProcessMap.get(typeName).containsKey(NON_PRIMITIVE_MAP_CATEGORY)) {
+ return null;
+ }
+
+ String propertyName = StringUtils.remove(label, Constants.INTERNAL_PROPERTY_KEY_PREFIX);
+ List<String> properties = postProcessMap.get(typeName).get(NON_PRIMITIVE_MAP_CATEGORY);
+
+ for (String p : properties) {
+ if (propertyName.startsWith(p)) {
+ return getLabelKeyPair(
+ String.format("%s%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, p),
+ StringUtils.remove(propertyName, p).substring(1).trim());
+ }
+ }
+
+ return null;
+ }
+
+ private boolean containsNonPrimitiveCollectionProperty(String typeName, String propertyName, String categoryType) {
+ if (!postProcessMap.containsKey(typeName)) {
+ return false;
+ }
+
+ if (!postProcessMap.get(typeName).containsKey(categoryType)) {
+ return false;
+ }
+
+ List<String> properties = postProcessMap.get(typeName).get(categoryType);
+
+ for (String p : properties) {
+ if (p.equals(propertyName)) {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ private String[] getLabelKeyPair(String label, String value) {
+ return new String[] { label, value };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java
index ec320b0..f1bbfcf 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/GraphSONUtility.java
@@ -18,10 +18,9 @@
package org.apache.atlas.repository.graphdb.janus.migration;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.type.AtlasBuiltInTypes;
-import org.apache.commons.lang.StringUtils;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasBigDecimalType;
+import org.apache.atlas.type.AtlasBuiltInTypes.AtlasBigIntegerType;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Graph.Features.EdgeFeatures;
@@ -34,21 +33,17 @@ import org.slf4j.LoggerFactory;
import java.util.*;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_EDGE_STATE_PROPERTY_KEY;
-import static org.apache.atlas.repository.Constants.CLASSIFICATION_ENTITY_GUID;
-import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY;
-
class GraphSONUtility {
private static final Logger LOG = LoggerFactory.getLogger(GraphSONUtility.class);
- private static final String EMPTY_STRING = "";
+ private static final String EMPTY_STRING = "";
+ private static final AtlasBigIntegerType bigIntegerType = new AtlasBigIntegerType();
+ private static final AtlasBigDecimalType bigDecimalType = new AtlasBigDecimalType();
- private final RelationshipTypeCache relationshipTypeCache;
- private static AtlasBuiltInTypes.AtlasBigIntegerType bigIntegerType = new AtlasBuiltInTypes.AtlasBigIntegerType();
- private static AtlasBuiltInTypes.AtlasBigDecimalType bigDecimalType = new AtlasBuiltInTypes.AtlasBigDecimalType();
+ private final ElementProcessors elementProcessors;
- public GraphSONUtility(final RelationshipTypeCache relationshipTypeCache) {
- this.relationshipTypeCache = relationshipTypeCache;
+ public GraphSONUtility(final ElementProcessors elementProcessors) {
+ this.elementProcessors = elementProcessors;
}
public Map<String, Object> vertexFromJson(Graph g, final JsonNode json) {
@@ -64,6 +59,7 @@ class GraphSONUtility {
Vertex vertex = vertexFeatures.willAllowId(vertexId) ? g.addVertex(T.id, vertexId) : g.addVertex();
props.put(Constants.VERTEX_ID_IN_IMPORT_KEY, vertexId);
+ elementProcessors.processCollections(Constants.ENTITY_TYPE_PROPERTY_KEY, props);
for (Map.Entry<String, Object> entry : props.entrySet()) {
try {
@@ -107,17 +103,11 @@ class GraphSONUtility {
props.put(Constants.EDGE_ID_IN_IMPORT_KEY, edgeId.toString());
- if(addRelationshipTypeForClassification(in, out, label, props)) {
- label = Constants.CLASSIFICATION_LABEL;
- } else {
- addRelationshipTypeName(label, props);
- }
+ label = elementProcessors.updateEdge(in, out, edgeId, label, props);
EdgeFeatures edgeFeatures = g.features().edge();
final Edge edge = edgeFeatures.willAllowId(edgeId) ? out.addEdge(label, in, T.id, edgeId) : out.addEdge(label, in);
- addMandatoryRelationshipProperties(props);
-
for (Map.Entry<String, Object> entry : props.entrySet()) {
try {
edge.property(entry.getKey(), entry.getValue());
@@ -153,82 +143,6 @@ class GraphSONUtility {
return cache.getMappedVertex(gr, inVId);
}
- private boolean addRelationshipTypeForClassification(Vertex in, Vertex out, String label, Map<String, Object> props) {
- if (in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
- String inTypeName = (String) in.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
-
- if (inTypeName.equals(label)) {
- if (StringUtils.isNotEmpty(inTypeName)) {
- props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, inTypeName);
-
- addEntityGuidToTrait(in, out);
-
- return true;
- } else {
- LOG.info("Could not find typeName for trait: {}", label);
- }
- }
- }
-
- return false;
- }
-
- private void addEntityGuidToTrait(Vertex in, Vertex out) {
- String entityGuid = "";
- if (out.property(Constants.GUID_PROPERTY_KEY).isPresent()) {
- entityGuid = (String) out.property(Constants.GUID_PROPERTY_KEY).value();
- }
-
- if(StringUtils.isNotEmpty(entityGuid)) {
- in.property(CLASSIFICATION_ENTITY_GUID, entityGuid);
- }
- }
-
- private void addRelationshipTypeName(String edgeLabel, Map<String, Object> props) {
- String typeName = relationshipTypeCache.get(edgeLabel);
-
- if (StringUtils.isNotEmpty(typeName)) {
- props.put(Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
- } else {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Could not find relationship type for: {}", edgeLabel);
- }
- }
- }
-
- private void addMandatoryRelationshipProperties(Map<String, Object> props) {
- props.put(Constants.RELATIONSHIP_GUID_PROPERTY_KEY, UUID.randomUUID().toString());
- props.put(RELATIONSHIPTYPE_TAG_PROPAGATION_KEY, "NONE");
- props.put(CLASSIFICATION_EDGE_STATE_PROPERTY_KEY, "ACTIVE");
- }
-
- public void replaceReferencedEdgeIdForList(Graph g, MappedElementCache cache, Vertex v, String propertyName) {
- try {
- if (v.property(Constants.TYPENAME_PROPERTY_KEY).isPresent() || !v.property(propertyName).isPresent()) {
- return;
- }
-
- List list = (List) v.property(propertyName).value();
- for (int i = 0; i < list.size(); i++) {
- String id = list.get(i).toString();
- Object newId = cache.getMappedEdge(g, id);
-
- if (newId == null) {
- continue;
- }
-
- list.set(i, newId.toString());
- }
-
- v.property(propertyName, list);
- } catch (IllegalArgumentException ex) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("processItem: IllegalArgumentException: v[{}] error!", v.id(), ex);
- }
- }
- }
-
- @VisibleForTesting
static Map<String, Object> readProperties(final JsonNode node) {
final Map<String, Object> map = new HashMap<>();
final Iterator<Map.Entry<String, JsonNode>> iterator = node.fields();
@@ -303,7 +217,6 @@ class GraphSONUtility {
return array;
}
- @VisibleForTesting
static Object getTypedValueFromJsonNode(final JsonNode node) {
Object theValue = null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java
index e4b6ee2..e2f418e 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/JsonNodeProcessManager.java
@@ -101,6 +101,7 @@ public class JsonNodeProcessManager {
private void commitRegular() {
commit(graph, nodes.size());
+ cache.clearAll();
}
private void commit(Graph g, int size) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java
index cca72ad..817e8dc 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/MappedElementCache.java
@@ -18,9 +18,7 @@
package org.apache.atlas.repository.graphdb.janus.migration;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.atlas.utils.LruCache;
-import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.slf4j.Logger;
@@ -28,18 +26,13 @@ import org.slf4j.LoggerFactory;
import java.util.Map;
-import static org.apache.atlas.repository.Constants.EDGE_ID_IN_IMPORT_KEY;
import static org.apache.atlas.repository.Constants.VERTEX_ID_IN_IMPORT_KEY;
public class MappedElementCache {
private static final Logger LOG = LoggerFactory.getLogger(MappedElementCache.class);
- @VisibleForTesting
final Map<Object, Vertex> lruVertexCache = new LruCache<>(500, 100000);
- @VisibleForTesting
- final Map<String, String> lruEdgeCache = new LruCache<>(500, 100000);
-
public Vertex getMappedVertex(Graph gr, Object key) {
try {
Vertex ret = lruVertexCache.get(key);
@@ -62,32 +55,6 @@ public class MappedElementCache {
}
}
- public String getMappedEdge(Graph gr, String key) {
- try {
- String ret = lruEdgeCache.get(key);
-
- if (ret == null) {
- synchronized (lruEdgeCache) {
- ret = lruEdgeCache.get(key);
-
- if (ret == null) {
- Edge e = fetchEdge(gr, key);
-
- ret = e.id().toString();
-
- lruEdgeCache.put(key, ret);
- }
- }
- }
-
- return ret;
- } catch (Exception ex) {
- LOG.error("getMappedEdge: {}", key, ex);
- return null;
- }
- }
-
- @VisibleForTesting
Vertex fetchVertex(Graph gr, Object key) {
try {
return gr.traversal().V().has(VERTEX_ID_IN_IMPORT_KEY, key).next();
@@ -97,18 +64,7 @@ public class MappedElementCache {
}
}
- @VisibleForTesting
- Edge fetchEdge(Graph gr, String key) {
- try {
- return gr.traversal().E().has(EDGE_ID_IN_IMPORT_KEY, key).next();
- } catch (Exception ex) {
- LOG.error("fetchEdge: fetchFromDB failed: {}", key);
- return null;
- }
- }
-
public void clearAll() {
lruVertexCache.clear();
- lruEdgeCache.clear();
}
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java
index d0a65f7..7046f8c 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/PostProcessManager.java
@@ -21,36 +21,42 @@ package org.apache.atlas.repository.graphdb.janus.migration;
import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemBuilder;
import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemConsumer;
import org.apache.atlas.repository.graphdb.janus.migration.pc.WorkItemManager;
+import org.apache.atlas.repository.graphdb.janus.migration.postProcess.PostProcessListProperty;
import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.List;
+import java.util.Map;
import java.util.concurrent.BlockingQueue;
+import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
+import static org.apache.atlas.repository.Constants.TYPENAME_PROPERTY_KEY;
+
public class PostProcessManager {
static class Consumer extends WorkItemConsumer<Object> {
private static final Logger LOG = LoggerFactory.getLogger(Consumer.class);
- private final Graph bulkLoadGraph;
- private final GraphSONUtility utility;
- private final String[] properties;
- private final MappedElementCache cache;
- private final int batchSize;
- private long counter;
- private long batchCounter;
+ private final Graph bulkLoadGraph;
+ private final Map<String, Map<String, List<String>>> typePropertiesMap;
+ private final int batchSize;
+ private long counter;
+ private long batchCounter;
+ private final PostProcessListProperty processor;
+ private final String[] nonPrimitiveCategoryKeys;
- public Consumer(BlockingQueue<Object> queue, Graph bulkLoadGraph, GraphSONUtility utility,
- String[] properties, MappedElementCache cache, int batchSize) {
+ public Consumer(BlockingQueue<Object> queue, Graph bulkLoadGraph, Map<String, Map<String, List<String>>> typePropertiesMap, int batchSize) {
super(queue);
- this.bulkLoadGraph = bulkLoadGraph;
- this.utility = utility;
- this.properties = properties;
- this.cache = cache;
- this.batchSize = batchSize;
- this.counter = 0;
- this.batchCounter = 0;
+ this.bulkLoadGraph = bulkLoadGraph;
+ this.typePropertiesMap = typePropertiesMap;
+ this.batchSize = batchSize;
+ this.counter = 0;
+ this.batchCounter = 0;
+ this.processor = new PostProcessListProperty();
+ this.nonPrimitiveCategoryKeys = ElementProcessors.getNonPrimitiveCategoryKeys();
}
@Override
@@ -59,23 +65,44 @@ public class PostProcessManager {
counter++;
try {
- Vertex v = bulkLoadGraph.traversal().V(vertexId).next();
-
- for (String p : properties) {
- utility.replaceReferencedEdgeIdForList(bulkLoadGraph, cache, v, p);
+ Vertex vertex = bulkLoadGraph.traversal().V(vertexId).next();
+ boolean isTypeVertex = vertex.property(TYPENAME_PROPERTY_KEY).isPresent();
+ VertexProperty typeNameProperty = vertex.property(ENTITY_TYPE_PROPERTY_KEY);
+
+ if (!isTypeVertex && typeNameProperty.isPresent()) {
+ String typeName = (String) typeNameProperty.value();
+ if (!typePropertiesMap.containsKey(typeName)) {
+ return;
+ }
+
+ Map<String, List<String>> collectionTypeProperties = typePropertiesMap.get(typeName);
+ for (String key : nonPrimitiveCategoryKeys) {
+ if (!collectionTypeProperties.containsKey(key)) {
+ continue;
+ }
+
+ for(String propertyName : collectionTypeProperties.get(key)) {
+ processor.process(vertex, typeName, propertyName);
+ }
+ }
}
- if (batchCounter >= batchSize) {
- LOG.info("[{}]: batch: {}: commit", counter, batchCounter);
- commit();
- batchCounter = 0;
- }
- }
- catch (Exception ex) {
+ commitBatch();
+ } catch (Exception ex) {
LOG.error("processItem: v[{}] error!", vertexId, ex);
}
}
+ private void commitBatch() {
+ if (batchCounter >= batchSize) {
+ LOG.info("[{}]: batch: {}: commit", counter, batchCounter);
+
+ commit();
+
+ batchCounter = 0;
+ }
+ }
+
@Override
protected void doCommit() {
bulkLoadGraph.tx().commit();
@@ -83,23 +110,19 @@ public class PostProcessManager {
}
private static class ConsumerBuilder implements WorkItemBuilder<Consumer, Object> {
- private final Graph bulkLoadGraph;
- private final GraphSONUtility utility;
- private final int batchSize;
- private final MappedElementCache cache;
- private final String[] vertexPropertiesToPostProcess;
+ private final Graph bulkLoadGraph;
+ private final int batchSize;
+ private final Map<String, Map<String, List<String>>> vertexPropertiesToPostProcess;
- public ConsumerBuilder(Graph bulkLoadGraph, GraphSONUtility utility, String[] propertiesToPostProcess, int batchSize) {
+ public ConsumerBuilder(Graph bulkLoadGraph, Map<String, Map<String, List<String>>> propertiesToPostProcess, int batchSize) {
this.bulkLoadGraph = bulkLoadGraph;
- this.utility = utility;
this.batchSize = batchSize;
- this.cache = new MappedElementCache();
this.vertexPropertiesToPostProcess = propertiesToPostProcess;
}
@Override
public Consumer build(BlockingQueue<Object> queue) {
- return new Consumer(queue, bulkLoadGraph, utility, vertexPropertiesToPostProcess, cache, batchSize);
+ return new Consumer(queue, bulkLoadGraph, vertexPropertiesToPostProcess, batchSize);
}
}
@@ -109,8 +132,9 @@ public class PostProcessManager {
}
}
- public static WorkItemsManager create(Graph bGraph, GraphSONUtility utility, String[] propertiesToPostProcess, int batchSize, int numWorkers) {
- ConsumerBuilder cb = new ConsumerBuilder(bGraph, utility, propertiesToPostProcess, batchSize);
+ public static WorkItemsManager create(Graph bGraph, Map<String, Map<String, List<String>>> propertiesToPostProcess,
+ int batchSize, int numWorkers) {
+ ConsumerBuilder cb = new ConsumerBuilder(bGraph, propertiesToPostProcess, batchSize);
return new WorkItemsManager(cb, batchSize, numWorkers);
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipCacheGenerator.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipCacheGenerator.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipCacheGenerator.java
new file mode 100644
index 0000000..c4ee179
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipCacheGenerator.java
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.model.typedef.AtlasRelationshipDef;
+import org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags;
+import org.apache.atlas.model.typedef.AtlasRelationshipEndDef;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.type.AtlasRelationshipType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.v1.typesystem.types.utils.TypesUtil;
+import org.apache.commons.lang.StringUtils;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class RelationshipCacheGenerator {
+
+ public static class TypeInfo extends TypesUtil.Pair<String, PropagateTags> {
+
+ public TypeInfo(String typeName, PropagateTags propagateTags) {
+ super(typeName, propagateTags);
+ }
+
+ public String getTypeName() {
+ return left;
+ }
+
+ public PropagateTags getPropagateTags() {
+ return right;
+ }
+ }
+
+
+ public static Map<String, TypeInfo> get(AtlasTypeRegistry typeRegistry) {
+ Map<String, TypeInfo> ret = new HashMap<>();
+
+ for (AtlasRelationshipType relType : typeRegistry.getAllRelationshipTypes()) {
+ AtlasRelationshipDef relDef = relType.getRelationshipDef();
+ String relTypeName = relType.getTypeName();
+
+ add(ret, getKey(relDef.getEndDef1()), relTypeName, relDef.getPropagateTags());
+ add(ret, getKey(relDef.getEndDef2()), relTypeName, getEnd2PropagateTag(relDef.getPropagateTags()));
+ }
+
+ return ret;
+ }
+
+ private static String getKey(AtlasRelationshipEndDef endDef) {
+ return getKey(endDef.getIsLegacyAttribute(), endDef.getType(), endDef.getName());
+ }
+
+ private static String getKey(String lhs, String rhs) {
+ return String.format("%s%s.%s", Constants.INTERNAL_PROPERTY_KEY_PREFIX, lhs, rhs);
+ }
+
+ private static String getKey(boolean isLegacy, String entityTypeName, String relEndName) {
+ if (!isLegacy) {
+ return "";
+ }
+
+ return getKey(entityTypeName, relEndName);
+ }
+
+ private static void add(Map<String, TypeInfo> map, String key, String relationTypeName, PropagateTags propagateTags) {
+ if (StringUtils.isEmpty(key) || map.containsKey(key)) {
+ return;
+ }
+
+ map.put(key, new TypeInfo(relationTypeName, propagateTags));
+ }
+
+ private static PropagateTags getEnd2PropagateTag(PropagateTags end1PropagateTags) {
+ if (end1PropagateTags == PropagateTags.ONE_TO_TWO) {
+ return PropagateTags.TWO_TO_ONE;
+ } else if (end1PropagateTags == PropagateTags.TWO_TO_ONE) {
+ return PropagateTags.ONE_TO_TWO;
+ } else {
+ return end1PropagateTags;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipTypeCache.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipTypeCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipTypeCache.java
deleted file mode 100644
index e4e8264..0000000
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/RelationshipTypeCache.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graphdb.janus.migration;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.Map;
-
-public class RelationshipTypeCache {
- private static final Logger LOG = LoggerFactory.getLogger(RelationshipTypeCache.class);
- private final Map<String, String> relationshipLookup;
-
- public RelationshipTypeCache(Map<String, String> lookup) {
- relationshipLookup = lookup;
- }
-
- public String get(String label) {
- return relationshipLookup.get(label);
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
new file mode 100644
index 0000000..55aa9c9
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/TypesWithCollectionsFinder.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration;
+
+import org.apache.atlas.model.TypeCategory;
+import org.apache.atlas.type.AtlasArrayType;
+import org.apache.atlas.type.AtlasMapType;
+import org.apache.atlas.type.AtlasStructType;
+import org.apache.atlas.type.AtlasStructType.AtlasAttribute;
+import org.apache.atlas.type.AtlasType;
+import org.apache.atlas.type.AtlasTypeRegistry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.atlas.model.TypeCategory.*;
+
+public class TypesWithCollectionsFinder {
+ private static final Logger LOG = LoggerFactory.getLogger(TypesWithCollectionsFinder.class);
+
+ static final EnumSet<TypeCategory> nonPrimitives = EnumSet.of(ENTITY, STRUCT, OBJECT_ID_TYPE);
+
+ public static Map<String, Map<String, List<String>>> getVertexPropertiesForCollectionAttributes(AtlasTypeRegistry typeRegistry) {
+ Map<String, Map<String, List<String>>> ret = new HashMap<>();
+
+ addVertexPropertiesForCollectionAttributes(typeRegistry.getAllEntityTypes(), ret);
+ addVertexPropertiesForCollectionAttributes(typeRegistry.getAllStructTypes(), ret);
+
+ displayInfo("types with properties: ", ret);
+
+ return ret;
+ }
+
+ private static void addVertexPropertiesForCollectionAttributes(Collection<? extends AtlasStructType> types, Map<String, Map<String, List<String>>> typeAttrMap) {
+ for (AtlasStructType type : types) {
+ Map<String, List<String>> collectionProperties = getVertexPropertiesForCollectionAttributes(type);
+
+ if(collectionProperties != null && collectionProperties.size() > 0) {
+ typeAttrMap.put(type.getTypeName(), collectionProperties);
+ }
+ }
+ }
+
+ static Map<String, List<String>> getVertexPropertiesForCollectionAttributes(AtlasStructType type) {
+ try {
+ Map<String, List<String>> collectionProperties = new HashMap<>();
+
+ for (AtlasAttribute attr : type.getAllAttributes().values()) {
+ addIfCollectionAttribute(attr, collectionProperties);
+ }
+
+ return collectionProperties;
+ } catch (Exception e) {
+ LOG.error("addVertexPropertiesForCollectionAttributes", e);
+ }
+
+ return null;
+ }
+
+ private static void addIfCollectionAttribute(AtlasAttribute attr, Map<String, List<String>> collectionProperties) {
+ AtlasType attrType = attr.getAttributeType();
+ TypeCategory attrTypeCategory = attrType.getTypeCategory();
+
+ switch (attrTypeCategory) {
+ case ARRAY: {
+ TypeCategory arrayElementType = ((AtlasArrayType) attrType).getElementType().getTypeCategory();
+
+ if (nonPrimitives.contains(arrayElementType)) {
+ addVertexProperty(attrTypeCategory.toString(), attr.getVertexPropertyName(), collectionProperties);
+ }
+ }
+ break;
+
+ case MAP: {
+ TypeCategory mapValueType = ((AtlasMapType) attrType).getValueType().getTypeCategory();
+
+ if (nonPrimitives.contains(mapValueType)) {
+ addVertexProperty(attrTypeCategory.toString(), attr.getVertexPropertyName(), collectionProperties);
+ } else {
+ addVertexProperty(attrTypeCategory.toString() + "_PRIMITIVE", attr.getVertexPropertyName(), collectionProperties);
+ }
+ }
+ break;
+ }
+ }
+
+ private static void addVertexProperty(String collectionType, String propertyName, Map<String, List<String>> collectionProperties) {
+ if(!collectionProperties.containsKey(collectionType)) {
+ collectionProperties.put(collectionType, new ArrayList<>());
+ }
+
+ collectionProperties.get(collectionType).add(propertyName);
+ }
+
+ static void displayInfo(String message, Map<String, Map<String, List<String>>> map) {
+ LOG.info(message);
+ for (Map.Entry<String, Map<String, List<String>>> e : map.entrySet()) {
+ LOG.info(" type: {} : {}", e.getKey(), e.getValue());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/postProcess/PostProcessListProperty.java
----------------------------------------------------------------------
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/postProcess/PostProcessListProperty.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/postProcess/PostProcessListProperty.java
new file mode 100644
index 0000000..8270ae9
--- /dev/null
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/migration/postProcess/PostProcessListProperty.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.graphdb.janus.migration.postProcess;
+
+import org.apache.atlas.repository.Constants;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PostProcessListProperty {
+ private static final Logger LOG = LoggerFactory.getLogger(PostProcessListProperty.class);
+
+ public void process(Vertex v, String typeName, String propertyName) {
+ try {
+ if (doesNotHaveProperty(v, typeName) || !hasProperty(v, propertyName)) {
+ return;
+ }
+
+ removeProperty(v, propertyName);
+ } catch (IllegalArgumentException ex) {
+ LOG.error("process: IllegalArgumentException: v[{}] error!", v.id(), ex);
+ }
+ }
+
+ protected void removeProperty(Vertex v, String propertyName) {
+ v.property(propertyName).remove();
+ }
+
+ protected boolean doesNotHaveProperty(Vertex v, String typeName) {
+ return v.property(Constants.TYPENAME_PROPERTY_KEY).isPresent() || !isInstanceVertexOfType(v, typeName);
+ }
+
+ private boolean hasProperty(Vertex v, String propertyName) {
+ try {
+ return v.property(propertyName).isPresent();
+ } catch(Exception ex) {
+ // ...
+ }
+
+ return false;
+ }
+
+ private boolean isInstanceVertexOfType(Vertex v, String typeName) {
+ if(v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).isPresent()) {
+ String s = (String) v.property(Constants.ENTITY_TYPE_PROPERTY_KEY).value();
+
+ return s.equals(typeName);
+ }
+
+ return false;
+ }
+}
[4/8] atlas git commit: ATLAS-2637: migration-import updates for
changes in collection attribute storage
Posted by ma...@apache.org.
http://git-wip-us.apache.org/repos/asf/atlas/blob/5cbfca24/repository/src/test/resources/complex-attr_db/atlas-migration-data.json
----------------------------------------------------------------------
diff --git a/repository/src/test/resources/complex-attr_db/atlas-migration-data.json b/repository/src/test/resources/complex-attr_db/atlas-migration-data.json
new file mode 100644
index 0000000..e7330e2
--- /dev/null
+++ b/repository/src/test/resources/complex-attr_db/atlas-migration-data.json
@@ -0,0 +1,5569 @@
+{
+ "mode": "EXTENDED",
+ "vertices": [
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "TaxonomyTerm"
+ },
+ "__type.TaxonomyTerm.atlas.taxonomy": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"atlas.taxonomy\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "5af3e7e8-68a6-4cf6-b88e-4c619731f3e2"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.TaxonomyTerm": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "atlas.taxonomy"
+ }
+ ]
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "TRAIT"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656711303
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "TaxonomyTerm"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656711303
+ },
+ "_id": 16400,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "Referenceable"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.Referenceable": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "qualifiedName"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "f4b80d9b-e72c-419f-ab40-9a3256a541b9"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656711776
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.Referenceable.qualifiedName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"qualifiedName\",\"isUnique\":true}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "Referenceable"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656711776
+ },
+ "_id": 32784,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "__internal"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656711806
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "__internal"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "25e22509-b6fd-42ed-b5bc-8e1726f37103"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656711806
+ },
+ "_id": 49168,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "DataSet"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656712591
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "DataSet"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "38c97017-cb6c-4f64-9610-6930d04c7645"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656712591
+ },
+ "_id": 65552,
+ "_type": "vertex"
+ },
+ {
+ "__type.file_action.READ_EXECUTE": {
+ "type": "integer",
+ "value": 5
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "file_action"
+ },
+ "__type.file_action.NONE": {
+ "type": "integer",
+ "value": 0
+ },
+ "__type.file_action.ALL": {
+ "type": "integer",
+ "value": 7
+ },
+ "__type.file_action.READ_WRITE": {
+ "type": "integer",
+ "value": 6
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "df00ab81-0791-4dbb-bd56-e4183e039111"
+ },
+ "__type.file_action.WRITE_EXECUTE": {
+ "type": "integer",
+ "value": 3
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.file_action.EXECUTE": {
+ "type": "integer",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "ENUM"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__type.file_action": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "NONE"
+ },
+ {
+ "type": "string",
+ "value": "EXECUTE"
+ },
+ {
+ "type": "string",
+ "value": "WRITE"
+ },
+ {
+ "type": "string",
+ "value": "WRITE_EXECUTE"
+ },
+ {
+ "type": "string",
+ "value": "READ"
+ },
+ {
+ "type": "string",
+ "value": "READ_EXECUTE"
+ },
+ {
+ "type": "string",
+ "value": "READ_WRITE"
+ },
+ {
+ "type": "string",
+ "value": "ALL"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656718218
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.file_action.READ": {
+ "type": "integer",
+ "value": 4
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "file_action"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656718218
+ },
+ "__type.file_action.WRITE": {
+ "type": "integer",
+ "value": 2
+ },
+ "_id": 81936,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_table"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_table.tableType": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"tableType\",\"isUnique\":false}"
+ },
+ "__type.hive_table.retention": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"retention\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 2
+ },
+ "__type.hive_table.db": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_db\",\"name\":\"db\",\"isUnique\":false}"
+ },
+ "__type.hive_table.viewOriginalText": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"viewOriginalText\",\"isUnique\":false}"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.1"
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hive_table.partitionKeys": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hive_column>\",\"name\":\"partitionKeys\",\"isUnique\":false}"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656722469
+ },
+ "__type.hive_table.aliases": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"aliases\",\"isUnique\":false}"
+ },
+ "__type.hive_table.sd": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"hive_storagedesc\",\"name\":\"sd\",\"isUnique\":false}"
+ },
+ "__type.hive_table.columns": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hive_column>\",\"name\":\"columns\",\"isUnique\":false}"
+ },
+ "__type.hive_table.temporary": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"temporary\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "{\"schemaElementsAttribute\":\"columns\"}"
+ },
+ "__type.hive_table.comment": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"comment\",\"isUnique\":false}"
+ },
+ "__type.hive_table.viewExpandedText": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"viewExpandedText\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "dfab14ca-1978-4157-8a9d-3c4e1d608830"
+ },
+ "__type.hive_table.createTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"
+ },
+ "__type.hive_table.lastAccessTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"lastAccessTime\",\"isUnique\":false}"
+ },
+ "__type.hive_table": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "db"
+ },
+ {
+ "type": "string",
+ "value": "createTime"
+ },
+ {
+ "type": "string",
+ "value": "lastAccessTime"
+ },
+ {
+ "type": "string",
+ "value": "comment"
+ },
+ {
+ "type": "string",
+ "value": "retention"
+ },
+ {
+ "type": "string",
+ "value": "sd"
+ },
+ {
+ "type": "string",
+ "value": "partitionKeys"
+ },
+ {
+ "type": "string",
+ "value": "aliases"
+ },
+ {
+ "type": "string",
+ "value": "columns"
+ },
+ {
+ "type": "string",
+ "value": "parameters"
+ },
+ {
+ "type": "string",
+ "value": "viewOriginalText"
+ },
+ {
+ "type": "string",
+ "value": "viewExpandedText"
+ },
+ {
+ "type": "string",
+ "value": "tableType"
+ },
+ {
+ "type": "string",
+ "value": "temporary"
+ }
+ ]
+ },
+ "__type.hive_table.parameters": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656731237
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_table"
+ },
+ "_id": 98320,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "falcon_feed_replication"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656725295
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "falcon_feed_replication"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "861f6201-6bbd-4a94-ac04-6b4982c4f94e"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656725295
+ },
+ "_id": 114704,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "falcon_cluster"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "f252485f-992a-4588-9b54-f96cf98ef3d2"
+ },
+ "__type.falcon_cluster.colo": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"colo\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.falcon_cluster.tags": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"tags\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.falcon_cluster": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "colo"
+ },
+ {
+ "type": "string",
+ "value": "tags"
+ }
+ ]
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656725304
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "falcon_cluster"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656725304
+ },
+ "_id": 131088,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__type.entity_type": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "name"
+ },
+ {
+ "type": "string",
+ "value": "isReplicated"
+ }
+ ]
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.entity_type.name": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":true}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "e37409c4-6e2d-4ee7-b87a-f1dc32bb8aee"
+ },
+ "__type.entity_type.isReplicated": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"isReplicated\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1526494658119
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "entity_type_description"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494658119
+ },
+ "_id": 147472,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": true
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityMapValue1"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated true name entityMapValue1 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "00448b65-b8bf-49f5-a478-d4c10f440464"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494659545
+ },
+ "_id": 163856,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityMapValue11-edit"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityMapValue11-edit "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "892387e9-9492-4c6d-a1f5-68697fb356a5"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494664276
+ },
+ "_id": 180240,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": true
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityArray33"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated true name entityArray33 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "7c6200d0-8b58-47ef-a279-64c67e25021d"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494662554
+ },
+ "_id": 196624,
+ "_type": "vertex"
+ },
+ {
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "struct_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "struct_type.name": {
+ "type": "string",
+ "value": "structMap22"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "_id": 213008,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "__AtlasUserProfile"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.__AtlasUserProfile.name": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":true}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "69331a04-420b-4610-9892-398e18e1fa5f"
+ },
+ "__type.__AtlasUserProfile.fullName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"fullName\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.__AtlasUserProfile.savedSearches": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<__AtlasUserSavedSearch>\",\"name\":\"savedSearches\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__type.__AtlasUserProfile": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "name"
+ },
+ {
+ "type": "string",
+ "value": "fullName"
+ },
+ {
+ "type": "string",
+ "value": "savedSearches"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656714170
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "__AtlasUserProfile"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656714170
+ },
+ "_id": 16448,
+ "_type": "vertex"
+ },
+ {
+ "__type.hdfs_path.extendedAttributes": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"extendedAttributes\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "hdfs_path"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hdfs_path.clusterName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hdfs_path": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "numberOfReplicas"
+ },
+ {
+ "type": "string",
+ "value": "extendedAttributes"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "cc19bd6e-fe24-4499-ac74-ae40ca934eef"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hdfs_path.numberOfReplicas": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"numberOfReplicas\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656718524
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hdfs_path"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656718524
+ },
+ "_id": 32832,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_column_lineage"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hive_column_lineage.expression": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"expression\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "8d0bf90b-0a3c-4e66-ba32-5349732c0967"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hive_column_lineage": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "query"
+ },
+ {
+ "type": "string",
+ "value": "depenendencyType"
+ },
+ {
+ "type": "string",
+ "value": "expression"
+ }
+ ]
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656722705
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hive_column_lineage.depenendencyType": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"depenendencyType\",\"isUnique\":false}"
+ },
+ "__type.hive_column_lineage.query": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_process\",\"name\":\"query\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_column_lineage"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656722705
+ },
+ "_id": 49216,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "kafka_topic"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "35619cb1-5e95-43a0-b51d-b8f228290f1c"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.kafka_topic.topic": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"topic\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656727066
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.kafka_topic": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "topic"
+ },
+ {
+ "type": "string",
+ "value": "uri"
+ }
+ ]
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "kafka_topic"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656727066
+ },
+ "__type.kafka_topic.uri": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"uri\",\"isUnique\":false}"
+ },
+ "_id": 65600,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityMapValue0"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityMapValue0 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "a6fcae0e-60a8-4950-a757-59b5a4d8b220"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494659545
+ },
+ "_id": 81984,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityArray11-edit"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityArray11-edit "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "4715142d-9058-4075-a307-869b6655f016"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494662768
+ },
+ "_id": 98368,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": true
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityArray00"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated true name entityArray00 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "2ae5b5ee-836e-4c1d-9be3-c94390c75436"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494662376
+ },
+ "_id": 114752,
+ "_type": "vertex"
+ },
+ {
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "struct_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "struct_type.name": {
+ "type": "string",
+ "value": "structMap11-edit"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "_id": 131136,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "Asset"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.Asset.name": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":false}"
+ },
+ "__type.Asset.owner": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"owner\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "4dffd142-fd58-4103-80ef-6a773b25fcb2"
+ },
+ "__type.Asset": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "name"
+ },
+ {
+ "type": "string",
+ "value": "description"
+ },
+ {
+ "type": "string",
+ "value": "owner"
+ }
+ ]
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656712492
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.Asset.description": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"description\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "Asset"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656712492
+ },
+ "_id": 16488,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "Process"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.Process.inputs": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<DataSet>\",\"name\":\"inputs\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "e273b345-4c6b-4272-8b2c-f2936eb1941b"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656713377
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.Process": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "inputs"
+ },
+ {
+ "type": "string",
+ "value": "outputs"
+ }
+ ]
+ },
+ "__type.Process.outputs": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<DataSet>\",\"name\":\"outputs\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "Process"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656713377
+ },
+ "_id": 32872,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_order"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_order.order": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"order\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "30993e57-4d69-46ab-b531-02371eace3d9"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hive_order": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "order"
+ },
+ {
+ "type": "string",
+ "value": "col"
+ }
+ ]
+ },
+ "__type.hive_order.col": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"col\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "STRUCT"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656721304
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_order"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656721304
+ },
+ "_id": 49256,
+ "_type": "vertex"
+ },
+ {
+ "__type.hive_process.operationType": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"operationType\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "hive_process"
+ },
+ "__type.hive_process.userName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"userName\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_process.queryId": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryId\",\"isUnique\":false}"
+ },
+ "__type.hive_process.queryPlan": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryPlan\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hive_process.recentQueries": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"recentQueries\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "a382861a-9dd2-4310-897e-20f8666198b4"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.hive_process": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "startTime"
+ },
+ {
+ "type": "string",
+ "value": "endTime"
+ },
+ {
+ "type": "string",
+ "value": "userName"
+ },
+ {
+ "type": "string",
+ "value": "operationType"
+ },
+ {
+ "type": "string",
+ "value": "queryText"
+ },
+ {
+ "type": "string",
+ "value": "queryPlan"
+ },
+ {
+ "type": "string",
+ "value": "queryId"
+ },
+ {
+ "type": "string",
+ "value": "recentQueries"
+ },
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "queryGraph"
+ }
+ ]
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hive_process.clusterName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656722390
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hive_process.endTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"endTime\",\"isUnique\":false}"
+ },
+ "__type.hive_process.queryText": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryText\",\"isUnique\":false}"
+ },
+ "__type.hive_process.queryGraph": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"queryGraph\",\"isUnique\":false}"
+ },
+ "__type.hive_process.startTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"startTime\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_process"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656722390
+ },
+ "_id": 65640,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__type.hive_column.table": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hive_table\",\"name\":\"table\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "{\"schemaAttributes\":\"[\\\"name\\\", \\\"description\\\", \\\"owner\\\", \\\"type\\\", \\\"comment\\\", \\\"position\\\"]\"}"
+ },
+ "__type.hive_column": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "type"
+ },
+ {
+ "type": "string",
+ "value": "comment"
+ },
+ {
+ "type": "string",
+ "value": "position"
+ },
+ {
+ "type": "string",
+ "value": "table"
+ }
+ ]
+ },
+ "__type.hive_column.type": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"type\",\"isUnique\":false}"
+ },
+ "__type.hive_column.comment": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"comment\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "210330a4-4777-4fa4-a3a0-a455e8d2a959"
+ },
+ "__version": {
+ "type": "long",
+ "value": 4
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.3"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656731493
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hive_column.position": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"int\",\"name\":\"position\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hive_column"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656722634
+ },
+ "_id": 82024,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "sqoop_process"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "3eb69f70-1569-41a9-b827-c16a77946b41"
+ },
+ "__type.sqoop_process": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "operation"
+ },
+ {
+ "type": "string",
+ "value": "commandlineOpts"
+ },
+ {
+ "type": "string",
+ "value": "startTime"
+ },
+ {
+ "type": "string",
+ "value": "endTime"
+ },
+ {
+ "type": "string",
+ "value": "userName"
+ }
+ ]
+ },
+ "__type.sqoop_process.userName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"userName\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.sqoop_process.endTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"endTime\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.sqoop_process.startTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"startTime\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656724359
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.sqoop_process.operation": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"operation\",\"isUnique\":false}"
+ },
+ "__type.sqoop_process.commandlineOpts": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"commandlineOpts\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "sqoop_process"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656724359
+ },
+ "_id": 98408,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hbase_column_family"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hbase_column_family.table": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":\"column_families\",\"dataType\":\"hbase_table\",\"name\":\"table\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "55673891-cb91-4169-8e77-65d402943ce2"
+ },
+ "__type.hbase_column_family.createTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 2
+ },
+ "__type.hbase_column_family.columns": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hbase_column>\",\"name\":\"columns\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.1"
+ },
+ "__type.hbase_column_family": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "table"
+ },
+ {
+ "type": "string",
+ "value": "columns"
+ },
+ {
+ "type": "string",
+ "value": "createTime"
+ },
+ {
+ "type": "string",
+ "value": "modifiedTime"
+ }
+ ]
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hbase_column_family.modifiedTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"modifiedTime\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656729584
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hbase_column_family"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656726223
+ },
+ "_id": 114792,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "storm_topology"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.storm_topology.startTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"startTime\",\"isUnique\":false}"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.storm_topology.id": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"id\",\"isUnique\":true}"
+ },
+ "__type.storm_topology": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "id"
+ },
+ {
+ "type": "string",
+ "value": "startTime"
+ },
+ {
+ "type": "string",
+ "value": "endTime"
+ },
+ {
+ "type": "string",
+ "value": "conf"
+ },
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "nodes"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "86313734-80ae-469f-b4c2-f505e1d68846"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__type.storm_topology.endTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"endTime\",\"isUnique\":false}"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656727932
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.storm_topology.clusterName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"
+ },
+ "__type.storm_topology.nodes": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<storm_node>\",\"name\":\"nodes\",\"isUnique\":false}"
+ },
+ "__type.storm_topology.conf": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"conf\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "storm_topology"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656727932
+ },
+ "_id": 131176,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "storm_spout"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.storm_spout": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "outputs"
+ }
+ ]
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "1ac62577-4ec0-4932-8913-67a4f51cfaa7"
+ },
+ "__type.storm_spout.outputs": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"outputs\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656727994
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "storm_spout"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656727994
+ },
+ "_id": 147560,
+ "_type": "vertex"
+ },
+ {
+ "__type.storm_bolt": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "inputs"
+ },
+ {
+ "type": "string",
+ "value": "outputs"
+ }
+ ]
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "storm_bolt"
+ },
+ "__type.storm_bolt.inputs": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"inputs\",\"isUnique\":false}"
+ },
+ "__type.storm_bolt.outputs": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"array<string>\",\"name\":\"outputs\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "d1e4da7c-83ea-48f1-b098-1f3201c4d58e"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656728002
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "storm_bolt"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656728002
+ },
+ "_id": 163944,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityMapValue22"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityMapValue22 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "4458482e-4890-45be-9320-364566f40f09"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494663395
+ },
+ "_id": 180328,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": true
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityArray22"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated true name entityArray22 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "b0f396f3-eeef-4ca1-ba5c-345e790c0d0c"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494662376
+ },
+ "_id": 196712,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "Infrastructure"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656713347
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "Infrastructure"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "887f7f1f-c16a-44af-a399-b81a30502f52"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656713347
+ },
+ "_id": 16528,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "fs_path"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.fs_path": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "path"
+ },
+ {
+ "type": "string",
+ "value": "createTime"
+ },
+ {
+ "type": "string",
+ "value": "modifiedTime"
+ },
+ {
+ "type": "string",
+ "value": "isFile"
+ },
+ {
+ "type": "string",
+ "value": "isSymlink"
+ },
+ {
+ "type": "string",
+ "value": "fileSize"
+ },
+ {
+ "type": "string",
+ "value": "group"
+ },
+ {
+ "type": "string",
+ "value": "posixPermissions"
+ }
+ ]
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "c24bf9da-6c38-4f69-bf72-9017a7a32707"
+ },
+ "__type.fs_path.path": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"path\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.fs_path.modifiedTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"modifiedTime\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.fs_path.fileSize": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"long\",\"name\":\"fileSize\",\"isUnique\":false}"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656718424
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.fs_path.isFile": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"isFile\",\"isUnique\":false}"
+ },
+ "__type.fs_path.createTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"
+ },
+ "__type.fs_path.isSymlink": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"boolean\",\"name\":\"isSymlink\",\"isUnique\":false}"
+ },
+ "__type.fs_path.group": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"group\",\"isUnique\":false}"
+ },
+ "__type.fs_path.posixPermissions": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"fs_permissions\",\"name\":\"posixPermissions\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "fs_path"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656718424
+ },
+ "_id": 32912,
+ "_type": "vertex"
+ },
+ {
+ "__type.hbase_namespace.createTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "hbase_namespace"
+ },
+ "__type.hbase_namespace.parameters": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"
+ },
+ "__type.hbase_namespace.modifiedTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"modifiedTime\",\"isUnique\":false}"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hbase_namespace": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "clusterName"
+ },
+ {
+ "type": "string",
+ "value": "parameters"
+ },
+ {
+ "type": "string",
+ "value": "createTime"
+ },
+ {
+ "type": "string",
+ "value": "modifiedTime"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "ff0890ec-c80d-4a35-bd7c-b0145cb714a9"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.hbase_namespace.clusterName": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"clusterName\",\"isUnique\":false}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656726191
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hbase_namespace"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656726191
+ },
+ "_id": 49296,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hbase_table"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.hbase_table": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "uri"
+ },
+ {
+ "type": "string",
+ "value": "column_families"
+ },
+ {
+ "type": "string",
+ "value": "namespace"
+ },
+ {
+ "type": "string",
+ "value": "parameters"
+ },
+ {
+ "type": "string",
+ "value": "createTime"
+ },
+ {
+ "type": "string",
+ "value": "modifiedTime"
+ }
+ ]
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "e82bc222-1bd2-429b-a73d-6a4b45d1e104"
+ },
+ "__type.hbase_table.namespace": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"hbase_namespace\",\"name\":\"namespace\",\"isUnique\":false}"
+ },
+ "__version": {
+ "type": "long",
+ "value": 2
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.2"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656728900
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.hbase_table.column_families": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<hbase_column_family>\",\"name\":\"column_families\",\"isUnique\":false}"
+ },
+ "__type.hbase_table.uri": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"uri\",\"isUnique\":false}"
+ },
+ "__type.hbase_table.parameters": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"map<string,string>\",\"name\":\"parameters\",\"isUnique\":false}"
+ },
+ "__type.hbase_table.createTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"createTime\",\"isUnique\":false}"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hbase_table"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656726210
+ },
+ "__type.hbase_table.modifiedTime": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"date\",\"name\":\"modifiedTime\",\"isUnique\":false}"
+ },
+ "_id": 65680,
+ "_type": "vertex"
+ },
+ {
+ "__type.name": {
+ "type": "string",
+ "value": "hbase_column"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.hbase_column.column_family": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":false,\"reverseAttributeName\":\"columns\",\"dataType\":\"hbase_column_family\",\"name\":\"column_family\",\"isUnique\":false}"
+ },
+ "__guid": {
+ "type": "string",
+ "value": "b44eaa9e-3727-4b0c-aad1-c0e59eea4d73"
+ },
+ "__type.hbase_column": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "column_family"
+ }
+ ]
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "root"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523656726233
+ },
+ "__type": {
+ "type": "string",
+ "value": "typeSystem"
+ },
+ "__type.description": {
+ "type": "string",
+ "value": "hbase_column"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523656726233
+ },
+ "_id": 82064,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityMapValue00"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityMapValue00 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "d265086f-60a8-49f6-906e-1738c42f59f3"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494663395
+ },
+ "_id": 98448,
+ "_type": "vertex"
+ },
+ {
+ "__version": {
+ "type": "integer",
+ "value": 0
+ },
+ "entity_type.isReplicated": {
+ "type": "boolean",
+ "value": false
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "DELETED"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "testUser"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "entity_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "entity_type.name": {
+ "type": "string",
+ "value": "entityArray11"
+ },
+ "entityText": {
+ "type": "string",
+ "value": "entity_type isReplicated false name entityArray11 "
+ },
+ "__guid": {
+ "type": "string",
+ "value": "a4bc28c8-d4a0-4ae6-ad44-e28069cf3b2d"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1526494662376
+ },
+ "_id": 114832,
+ "_type": "vertex"
+ },
+ {
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__state": {
+ "type": "string",
+ "value": "ACTIVE"
+ },
+ "__createdBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__typeName": {
+ "type": "string",
+ "value": "struct_type"
+ },
+ "__modificationTimestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "struct_type.name": {
+ "type": "string",
+ "value": "structMap33"
+ },
+ "__timestamp": {
+ "type": "long",
+ "value": 1523657064098
+ },
+ "_id": 131216,
+ "_type": "vertex"
+ },
+ {
+ "__type.entity_with_complex_collection_attr.listOfEntities": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":0,\\\"upper\\\":2147483647,\\\"isUnique\\\":false}\",\"isIndexable\":false,\"isComposite\":true,\"reverseAttributeName\":null,\"dataType\":\"array<entity_type>\",\"name\":\"listOfEntities\",\"isUnique\":false}"
+ },
+ "__type.name": {
+ "type": "string",
+ "value": "entity_with_complex_collection_attr"
+ },
+ "__modifiedBy": {
+ "type": "string",
+ "value": "admin"
+ },
+ "__type.options": {
+ "type": "string",
+ "value": "null"
+ },
+ "__type.entity_with_complex_collection_attr": {
+ "type": "list",
+ "value": [
+ {
+ "type": "string",
+ "value": "name"
+ },
+ {
+ "type": "string",
+ "value": "listOfStructs"
+ },
+ {
+ "type": "string",
+ "value": "listOfEntities"
+ },
+ {
+ "type": "string",
+ "value": "mapOfStructs"
+ },
+ {
+ "type": "string",
+ "value": "mapOfEntities"
+ }
+ ]
+ },
+ "__guid": {
+ "type": "string",
+ "value": "3da46128-6e98-4bce-b3d1-cbc4bf9b9055"
+ },
+ "__version": {
+ "type": "long",
+ "value": 1
+ },
+ "__type.entity_with_complex_collection_attr.name": {
+ "type": "string",
+ "value": "{\"multiplicity\":\"{\\\"lower\\\":1,\\\"upper\\\":1,\\\"isUnique\\\":false}\",\"isIndexable\":true,\"isComposite\":false,\"reverseAttributeName\":null,\"dataType\":\"string\",\"name\":\"name\",\"isUnique\":true}"
+ },
+ "__type.category": {
+ "type": "string",
+ "value": "CLASS"
+ },
+ "__type.version": {
+ "type": "string",
+ "value": "1.0"
+ },
+ "__created
<TRUNCATED>