You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by jn...@apache.org on 2017/02/03 17:00:36 UTC
incubator-atlas git commit: ATLAS-1510: Consolidate/batch calls to
GraphBackedTypeStore.findVertex()
Repository: incubator-atlas
Updated Branches:
refs/heads/master 09089e09f -> 02cf8c488
ATLAS-1510: Consolidate/batch calls to GraphBackedTypeStore.findVertex()
Project: http://git-wip-us.apache.org/repos/asf/incubator-atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-atlas/commit/02cf8c48
Tree: http://git-wip-us.apache.org/repos/asf/incubator-atlas/tree/02cf8c48
Diff: http://git-wip-us.apache.org/repos/asf/incubator-atlas/diff/02cf8c48
Branch: refs/heads/master
Commit: 02cf8c4884eb346f7b993bd46d6e789bd73f006d
Parents: 09089e0
Author: Jeff Hagelberg <jn...@us.ibm.com>
Authored: Fri Feb 3 12:00:09 2017 -0500
Committer: Jeff Hagelberg <jn...@us.ibm.com>
Committed: Fri Feb 3 12:00:09 2017 -0500
----------------------------------------------------------------------
release-log.txt | 1 +
.../atlas/repository/graph/GraphHelper.java | 2 +-
.../typestore/GraphBackedTypeStore.java | 156 +++++++++++++------
.../typestore/TypePersistenceVisitor.java | 116 ++++++++++++++
.../repository/typestore/TypeVertexFinder.java | 103 ++++++++++++
.../repository/typestore/TypeVertexInfo.java | 94 +++++++++++
.../atlas/repository/typestore/TypeVisitor.java | 96 ++++++++++++
.../typestore/GraphBackedTypeStoreTest.java | 6 +-
8 files changed, 520 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 991557b..e6272fb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -9,6 +9,7 @@ ATLAS-1060 Add composite indexes for exact match performance improvements for al
ATLAS-1127 Modify creation and modification timestamps to Date instead of Long(sumasai)
ALL CHANGES:
+ATLAS-1510 Consolidate/batch calls to GraphBackedTypeStore.findVertex() (jnhagelb)
ATLAS-1388 Cache entities that are created/updated (jnhagelb)
ATLAS-1369 Optimize Gremlin queries generated by DSL translator (jnhagelb)
ATLAS-1517: updated hive_model to include schema related attributes (sarath.kum4r@gmail.com via mneethiraj)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index ce8d4c7..89e978d 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -538,7 +538,7 @@ public final class GraphHelper {
*
* @return propertyValue to AtlasVertex map with the result.
*/
- private Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values)
+ public Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values)
throws RepositoryException {
if(values.isEmpty()) {
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
old mode 100755
new mode 100644
index c1c2a94..39e7a31
--- a/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/GraphBackedTypeStore.java
@@ -21,14 +21,17 @@ package org.apache.atlas.repository.typestore;
import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.atlas.AtlasException;
import org.apache.atlas.GraphTransaction;
import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -57,8 +60,10 @@ import org.codehaus.jettison.json.JSONException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Function;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
import com.google.inject.Inject;
import com.google.inject.Singleton;
@@ -82,25 +87,37 @@ public class GraphBackedTypeStore implements ITypeStore {
@Override
@GraphTransaction
public void store(TypeSystem typeSystem, ImmutableList<String> typeNames) throws AtlasException {
+
+ //Pre-create the vertices that are needed for the types. This allows us to execute
+ //one query to determine all of the vertices that already exist.
+ Map<String, AtlasVertex> typeVertices = getOrCreateTypeVertices(typeSystem, typeNames);
+
+ //Complete the storage process by adding properties and edges to the vertices
+ //that were created.
+ TypePersistenceVisitor visitor = new TypePersistenceVisitor(this, typeVertices, typeSystem);
+ processTypes(typeNames, typeSystem, visitor);
+ }
+
+ private void processTypes(ImmutableList<String> typeNames, TypeSystem typeSystem, TypeVisitor visitor) throws AtlasException {
for (String typeName : typeNames) {
IDataType dataType = typeSystem.getDataType(IDataType.class, typeName);
LOG.debug("Processing {}.{}.{} in type store", dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
switch (dataType.getTypeCategory()) {
case ENUM:
- storeInGraph((EnumType) dataType);
+ visitor.visitEnumeration((EnumType)dataType);
break;
case STRUCT:
StructType structType = (StructType) dataType;
- storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
- ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of());
+ processType(typeSystem, dataType.getTypeCategory(), dataType.getName(), dataType.getDescription(),
+ ImmutableList.copyOf(structType.infoToNameMap.keySet()), ImmutableSet.<String>of(), visitor);
break;
case TRAIT:
case CLASS:
HierarchicalType type = (HierarchicalType) dataType;
- storeInGraph(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs,
- type.superTypes);
+ processType(typeSystem, dataType.getTypeCategory(), dataType.getName(), type.getDescription(), type.immediateAttrs,
+ type.superTypes, visitor);
break;
default: //Ignore primitive/collection types as they are covered under references
@@ -109,63 +126,70 @@ public class GraphBackedTypeStore implements ITypeStore {
}
}
- private void storeInGraph(EnumType dataType) {
- AtlasVertex AtlasVertex = createVertex(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
- List<String> values = new ArrayList<>(dataType.values().size());
- for (EnumValue enumValue : dataType.values()) {
- String key = getPropertyKey(dataType.getName(), enumValue.value);
- setProperty(AtlasVertex, key, enumValue.ordinal);
- values.add(enumValue.value);
+ private Map<String, AtlasVertex> getOrCreateTypeVertices(TypeSystem typeSystem, ImmutableList<String> typeNames) throws AtlasException {
+
+ //examine the types to determine what type vertices are needed
+ TypeVertexFinder vertexFinder = new TypeVertexFinder(typeSystem);
+ processTypes(typeNames, typeSystem, vertexFinder);
+ List<TypeVertexInfo> typeVerticesNeeded = vertexFinder.getVerticesToCreate();
+
+ //find or create the type vertices
+ List<AtlasVertex> vertices = createVertices(typeVerticesNeeded);
+
+ //Create a type name->AtlasVertex map with the result
+ Map<String, AtlasVertex> result = new HashMap<String,AtlasVertex>(typeVerticesNeeded.size());
+ for(int i = 0 ; i < typeVerticesNeeded.size(); i++) {
+ TypeVertexInfo createdVertexInfo = typeVerticesNeeded.get(i);
+ AtlasVertex createdVertex = vertices.get(i);
+ result.put(createdVertexInfo.getTypeName(), createdVertex);
}
- setProperty(AtlasVertex, getPropertyKey(dataType.getName()), values);
+ return result;
+
}
- private String getPropertyKey(String name) {
+
+ static String getPropertyKey(String name) {
return PROPERTY_PREFIX + name;
}
- private String getPropertyKey(String parent, String child) {
+ static String getPropertyKey(String parent, String child) {
return PROPERTY_PREFIX + parent + "." + child;
}
- String getEdgeLabel(String parent, String child) {
+ static String getEdgeLabel(String parent, String child) {
return PROPERTY_PREFIX + "edge." + parent + "." + child;
}
- private void storeInGraph(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription,
- ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes) throws AtlasException {
- AtlasVertex vertex = createVertex(category, typeName, typeDescription);
+ private void processType(TypeSystem typeSystem, DataTypes.TypeCategory category, String typeName, String typeDescription,
+ ImmutableList<AttributeInfo> attributes, ImmutableSet<String> superTypes, TypeVisitor visitor) throws AtlasException {
+
+ visitor.visitDataType(category, typeName, typeDescription);
+
List<String> attrNames = new ArrayList<>();
if (attributes != null) {
for (AttributeInfo attribute : attributes) {
- String propertyKey = getPropertyKey(typeName, attribute.name);
- try {
- setProperty(vertex, propertyKey, attribute.toJson());
- } catch (JSONException e) {
- throw new StorageException(typeName, e);
- }
+ visitor.visitAttribute(typeName, attribute);
attrNames.add(attribute.name);
- addReferencesForAttribute(typeSystem, vertex, attribute);
+ processsAttribute(typeSystem, typeName, attribute, visitor);
}
}
- setProperty(vertex, getPropertyKey(typeName), attrNames);
+ visitor.visitAttributeNames(typeName, attrNames);
//Add edges for hierarchy
if (superTypes != null) {
for (String superTypeName : superTypes) {
- HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
- AtlasVertex superVertex = createVertex(superType.getTypeCategory(), superTypeName, superType.getDescription());
- graphHelper.getOrCreateEdge(vertex, superVertex, SUPERTYPE_EDGE_LABEL);
+ visitor.visitSuperType(typeName, superTypeName);
}
}
}
- private void addReferencesForAttribute(TypeSystem typeSystem, AtlasVertex vertex, AttributeInfo attribute)
+ private void processsAttribute(TypeSystem typeSystem, String typeName, AttributeInfo attribute, TypeVisitor visitor)
throws AtlasException {
+
ImmutableList<String> coreTypes = typeSystem.getCoreTypes();
List<IDataType> attrDataTypes = new ArrayList<>();
IDataType attrDataType = attribute.dataType();
- String vertexTypeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, String.class);
+
switch (attrDataType.getTypeCategory()) {
case ARRAY:
@@ -200,11 +224,10 @@ public class GraphBackedTypeStore implements ITypeStore {
"Attribute cannot reference instances of type : " + attrDataType.getTypeCategory());
}
+
for (IDataType attrType : attrDataTypes) {
if (!coreTypes.contains(attrType.getName())) {
- AtlasVertex attrVertex = createVertex(attrType.getTypeCategory(), attrType.getName(), attrType.getDescription());
- String label = getEdgeLabel(vertexTypeName, attribute.name);
- graphHelper.getOrCreateEdge(vertex, attrVertex, label);
+ visitor.visitAttributeDataType(typeName, attribute, attrType);
}
}
}
@@ -328,23 +351,54 @@ public class GraphBackedTypeStore implements ITypeStore {
return vertex;
}
- private AtlasVertex createVertex(DataTypes.TypeCategory category, String typeName, String typeDescription) {
- AtlasVertex vertex = findVertex(category, typeName);
- if (vertex == null) {
- LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, typeName);
- vertex = graph.addVertex();
- setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type AtlasVertex
- setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, category);
- setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, typeName);
- }
- if (typeDescription != null) {
- String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
- if (!typeDescription.equals(oldDescription)) {
- setProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, typeDescription);
+ //package-private for testing
+ Map<String, AtlasVertex> findVertices(List<String> typeNames) throws RepositoryException {
+ LOG.debug("Finding vertices for {}", typeNames.toString());
+ Map<String, AtlasVertex> foundVertices = graphHelper.getVerticesForPropertyValues(Constants.TYPENAME_PROPERTY_KEY, typeNames);
+ return foundVertices;
+
+ }
+
+
+ /**
+ * Finds or creates type vertices with the information specified.
+ *
+ * @param infoList
+ * @return list with the vertices corresponding to the types in the list.
+ * @throws AtlasException
+ */
+ private List<AtlasVertex> createVertices(List<TypeVertexInfo> infoList) throws AtlasException {
+
+ List<AtlasVertex> result = new ArrayList<>(infoList.size());
+ List<String> typeNames = Lists.transform(infoList, new Function<TypeVertexInfo,String>() {
+
+ @Override
+ public String apply(TypeVertexInfo input) {
+ return input.getTypeName();
+ }
+ });
+ Map<String, AtlasVertex> vertices = findVertices(typeNames);
+
+ for(TypeVertexInfo info : infoList) {
+ AtlasVertex vertex = vertices.get(info.getTypeName());
+ if (! GraphHelper.elementExists(vertex)) {
+ LOG.debug("Adding vertex {}{}", PROPERTY_PREFIX, info.getTypeName());
+ vertex = graph.addVertex();
+ setProperty(vertex, Constants.VERTEX_TYPE_PROPERTY_KEY, VERTEX_TYPE); // Mark as type AtlasVertex
+ setProperty(vertex, Constants.TYPE_CATEGORY_PROPERTY_KEY, info.getCategory());
+ setProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, info.getTypeName());
}
- } else {
- LOG.debug(" type description is null ");
+ String newDescription = info.getTypeDescription();
+ if (newDescription != null) {
+ String oldDescription = getPropertyKey(Constants.TYPEDESCRIPTION_PROPERTY_KEY);
+ if (!newDescription.equals(oldDescription)) {
+ setProperty(vertex, Constants.TYPEDESCRIPTION_PROPERTY_KEY, newDescription);
+ }
+ } else {
+ LOG.debug(" type description is null ");
+ }
+ result.add(vertex);
}
- return vertex;
+ return result;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
new file mode 100644
index 0000000..bfb1bfc
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypePersistenceVisitor.java
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.typestore;
+
+import static org.apache.atlas.repository.graph.GraphHelper.setProperty;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.codehaus.jettison.json.JSONException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TypeVisitor implementation that completes the type storage process by
+ * adding the required properties and edges to the type vertices
+ * that were created.
+ */
+public class TypePersistenceVisitor implements TypeVisitor {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TypePersistenceVisitor.class);
+ private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+ private final GraphBackedTypeStore typeStore_;
+ private final Map<String,AtlasVertex> typeVertices;
+ private final TypeSystem typeSystem;
+
+ /**
+ * @param graphBackedTypeStore
+ */
+ public TypePersistenceVisitor(GraphBackedTypeStore graphBackedTypeStore, Map<String,AtlasVertex> typeVertices, TypeSystem typeSystem) {
+ typeStore_ = graphBackedTypeStore;
+ this.typeVertices = typeVertices;
+ this.typeSystem = typeSystem;
+ }
+
+ @Override
+ public void visitEnumeration(EnumType dataType) throws AtlasException {
+ AtlasVertex vertex = typeVertices.get(dataType.getName());
+ List<String> values = new ArrayList<>(dataType.values().size());
+ for (EnumValue enumValue : dataType.values()) {
+ String key = GraphBackedTypeStore.getPropertyKey(dataType.getName(), enumValue.value);
+ setProperty(vertex, key, enumValue.ordinal);
+ values.add(enumValue.value);
+ }
+ setProperty(vertex, GraphBackedTypeStore.getPropertyKey(dataType.getName()), values);
+
+ }
+ @Override
+ public void visitAttributeDataType(String typeName, AttributeInfo attribute, IDataType attrType) throws AtlasException {
+ AtlasVertex vertex = typeVertices.get(typeName);
+ String vertexTypeName = GraphHelper.getSingleValuedProperty(vertex, Constants.TYPENAME_PROPERTY_KEY, String.class);
+ AtlasVertex attrVertex = typeVertices.get(attrType.getName());
+ String label = GraphBackedTypeStore.getEdgeLabel(vertexTypeName, attribute.name);
+ graphHelper.getOrCreateEdge(vertex, attrVertex, label);
+ }
+ @Override
+ public void visitSuperType(String typeName, String superTypeName) throws AtlasException {
+ AtlasVertex vertex = typeVertices.get(typeName);
+ HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
+ AtlasVertex superVertex = typeVertices.get(superTypeName);
+ graphHelper.getOrCreateEdge(vertex, superVertex, GraphBackedTypeStore.SUPERTYPE_EDGE_LABEL);
+ }
+
+ @Override
+ public void visitAttributeNames(String typeName, List<String> attrNames) throws AtlasException {
+ AtlasVertex vertex = typeVertices.get(typeName);
+ setProperty(vertex, GraphBackedTypeStore.getPropertyKey(typeName), attrNames);
+
+ }
+
+ @Override
+ public void visitAttribute(String typeName, AttributeInfo attribute) throws AtlasException {
+ AtlasVertex vertex = typeVertices.get(typeName);
+ String propertyKey = GraphBackedTypeStore.getPropertyKey(typeName, attribute.name);
+ try {
+ setProperty(vertex, propertyKey, attribute.toJson());
+ } catch (JSONException e) {
+ throw new StorageException(typeName, e);
+ }
+ }
+
+ @Override
+ public void visitDataType(TypeCategory category, String typeName, String typeDescription) {
+ //nothing to do
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
new file mode 100644
index 0000000..8b38152
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexFinder.java
@@ -0,0 +1,103 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.HierarchicalType;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+
+/**
+ * TypeVisitor implementation that builds up a list of type vertices
+ * that need to be created for the types that are being stored.
+ *
+ */
+public class TypeVertexFinder implements TypeVisitor {
+
+ private final List<TypeVertexInfo> toCreate = new ArrayList<TypeVertexInfo>();
+ private final Set<String> typesIncluded = new HashSet<String>();
+ private final TypeSystem typeSystem;
+
+ public TypeVertexFinder(TypeSystem ts) {
+ typeSystem = ts;
+ }
+
+
+ @Override
+ public void visitEnumeration(EnumType dataType) {
+ visitDataType(dataType);
+ }
+
+ private void addTypeIfNeeded(TypeVertexInfo info) {
+ if(! typesIncluded.contains(info.getTypeName())) {
+ toCreate.add(info);
+ typesIncluded.add(info.getTypeName());
+ }
+ }
+
+ @Override
+ public void visitAttributeDataType(String typeName, AttributeInfo sourceAttr, IDataType attrType) throws AtlasException {
+ visitDataType(attrType);
+ }
+
+ @Override
+ public void visitSuperType(String typeName, String superTypeName) throws AtlasException {
+ HierarchicalType superType = typeSystem.getDataType(HierarchicalType.class, superTypeName);
+ visitDataType(superType);
+ }
+
+ @Override
+ public void visitAttributeNames(String typeName, List<String> attrNames) throws AtlasException {
+ //nothing to do
+
+ }
+
+ @Override
+ public void visitAttribute(String typeName, AttributeInfo attribute) throws StorageException, AtlasException {
+ //nothing to do
+ }
+
+
+ private void visitDataType(IDataType dataType) {
+ TypeVertexInfo info = null;
+ info = new TypeVertexInfo(dataType.getTypeCategory(), dataType.getName(), dataType.getDescription());
+ addTypeIfNeeded(info);
+
+ }
+
+
+ public List<TypeVertexInfo> getVerticesToCreate() {
+ return toCreate;
+ }
+
+ @Override
+ public void visitDataType(TypeCategory category, String typeName, String typeDescription) {
+ TypeVertexInfo info = new TypeVertexInfo(category, typeName, typeDescription);
+ addTypeIfNeeded(info);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
new file mode 100644
index 0000000..32a9a19
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVertexInfo.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.Objects;
+
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+
+/**
+ * Records the information needed to create a particular type vertex.
+ */
+public class TypeVertexInfo {
+
+ private DataTypes.TypeCategory category;
+ private String typeName;
+ private String typeDescription;
+
+ public TypeVertexInfo(TypeCategory category, String typeName, String typeDescription) {
+ super();
+ this.category = category;
+ this.typeName = typeName;
+ this.typeDescription = typeDescription;
+ }
+
+ public DataTypes.TypeCategory getCategory() {
+ return category;
+ }
+
+ public void setCategory(DataTypes.TypeCategory category) {
+ this.category = category;
+ }
+
+ public String getTypeName() {
+ return typeName;
+ }
+
+ public void setTypeName(String typeName) {
+ this.typeName = typeName;
+ }
+
+ public String getTypeDescription() {
+ return typeDescription;
+ }
+
+ public void setTypeDescription(String typeDescription) {
+ this.typeDescription = typeDescription;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(category, typeName);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+
+ if (this == obj) {
+ return true;
+ }
+
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+
+ TypeVertexInfo other = (TypeVertexInfo)obj;
+ if(! Objects.equals(category, other.category)) {
+ return false;
+ }
+
+ if(! Objects.equals(typeName, other.typeName)) {
+ return false;
+ }
+
+ return true;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
new file mode 100644
index 0000000..a6e353c
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/typestore/TypeVisitor.java
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.atlas.repository.typestore;
+
+import java.util.List;
+
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
+import org.apache.atlas.typesystem.types.EnumType;
+import org.apache.atlas.typesystem.types.IDataType;
+
+/**
+ * Callback mechanism used when storing types. As {@link GraphBackedTypeStore} traverses
+ * through the types being persisted, these methods are called with the information that
+ * it finds.
+ */
+public interface TypeVisitor {
+
+ /**
+ * Called when an enumeration type is found
+ * @param type
+ * @throws AtlasException
+ */
+ void visitEnumeration(EnumType type) throws AtlasException;
+
+ /**
+ * Called with a data type that is associated with a given attribute. There can
+ * be more than one. For example, map types have both a key and a value type.
+ * This is called once for each type. This is called once for each datatype
+ * associated with the given attribute.
+ *
+ * @param typeName The name of the type being processed.
+ * @param sourceAttr The attribute in that type that we are processing.
+ * @param attrType A dataType associated with that attribute.
+ * @throws AtlasException
+ */
+ void visitAttributeDataType(String typeName, AttributeInfo sourceAttr, IDataType attrType) throws AtlasException;
+
+ /**
+ * Called when a super type is found. It is called once for each superType.
+ *
+ * @param typeName The type being processed.
+ * @param superType The name of the super type that was found.
+ * @throws RepositoryException
+ * @throws AtlasException
+ */
+ void visitSuperType(String typeName, String superType) throws RepositoryException, AtlasException;
+
+ /**
+ * Called with the list of immediate attribute names that were found for the given type. It
+ * is called once per type.
+ *
+ * @param typeName The name of the type that is being processed.
+ * @param attrNames The names of all of the immediate attributes in the type.
+ * @throws AtlasException
+ */
+ void visitAttributeNames(String typeName, List<String> attrNames) throws AtlasException;
+
+ /**
+ * Called once for each immediate attribute in a type.
+ * @param typeName The name of the type that is being procesed
+ * @param attribute The immediate attribute that was found
+ *
+ * @throws StorageException
+ * @throws AtlasException
+ */
+ void visitAttribute(String typeName, AttributeInfo attribute) throws StorageException, AtlasException;
+
+ /**
+ * Called once for each struct, class, and trait type that was found. It is
+ * called when we start processing that type.
+ *
+ * @param category The category of the type
+ * @param typeName The name of the type
+ * @param typeDescription The description of the type.
+ */
+ void visitDataType(TypeCategory category, String typeName, String typeDescription);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/02cf8c48/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
index 6b83aa3..265b316 100755
--- a/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/typestore/GraphBackedTypeStoreTest.java
@@ -23,6 +23,7 @@ import static org.apache.atlas.typesystem.types.utils.TypesUtil.createOptionalAt
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createRequiredAttrDef;
import static org.apache.atlas.typesystem.types.utils.TypesUtil.createStructTypeDef;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -32,6 +33,7 @@ import javax.inject.Inject;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
+import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graph.GraphHelper;
import org.apache.atlas.repository.graphdb.AtlasEdge;
@@ -200,11 +202,11 @@ public class GraphBackedTypeStoreTest {
verifyEdges();
}
- private void verifyEdges() {
+ private void verifyEdges() throws RepositoryException {
// ATLAS-474: verify that type update did not write duplicate edges to the type store.
if (typeStore instanceof GraphBackedTypeStore) {
GraphBackedTypeStore gbTypeStore = (GraphBackedTypeStore) typeStore;
- AtlasVertex typeVertex = gbTypeStore.findVertex(TypeCategory.CLASS, "Department");
+ AtlasVertex typeVertex = gbTypeStore.findVertices(Collections.singletonList("Department")).get("Department");
int edgeCount = countOutgoingEdges(typeVertex, gbTypeStore.getEdgeLabel("Department", "employees"));
Assert.assertEquals(edgeCount, 1, "Should only be 1 edge for employees attribute on Department type AtlasVertex");
}