You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by su...@apache.org on 2015/12/18 11:44:07 UTC
[07/26] incubator-atlas git commit: ATLAS-47 Entity mutations for
complex types (sumasai via shwethags)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
new file mode 100644
index 0000000..1fabafa
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphToTypedInstanceMapper.java
@@ -0,0 +1,419 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import com.thinkaurelius.titan.core.TitanGraph;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.StructType;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.math.BigDecimal;
+import java.math.BigInteger;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public final class GraphToTypedInstanceMapper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GraphToTypedInstanceMapper.class);
+ private static TypeSystem typeSystem = TypeSystem.getInstance();
+ private final TitanGraph titanGraph;
+
+ public GraphToTypedInstanceMapper(TitanGraph titanGraph) {
+ this.titanGraph = titanGraph;
+ }
+
+ public ITypedReferenceableInstance mapGraphToTypedInstance(String guid, Vertex instanceVertex)
+ throws AtlasException {
+
+ LOG.debug("Mapping graph root vertex {} to typed instance for guid {}", instanceVertex, guid);
+ String typeName = instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY);
+ List<String> traits = GraphHelper.getTraitNames(instanceVertex);
+
+ Id id = new Id(guid, instanceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY), typeName);
+ LOG.debug("Created id {} for instance type {}", id, typeName);
+
+ ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
+ ITypedReferenceableInstance typedInstance =
+ classType.createInstance(id, traits.toArray(new String[traits.size()]));
+
+ mapVertexToInstance(instanceVertex, typedInstance, classType.fieldMapping().fields);
+ mapVertexToInstanceTraits(instanceVertex, typedInstance, traits);
+
+ return typedInstance;
+ }
+
+ private void mapVertexToInstanceTraits(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
+ List<String> traits) throws AtlasException {
+ for (String traitName : traits) {
+ LOG.debug("mapping trait {} to instance", traitName);
+ TraitType traitType = typeSystem.getDataType(TraitType.class, traitName);
+ mapVertexToTraitInstance(instanceVertex, typedInstance, traitName, traitType);
+ }
+ }
+
+ public void mapVertexToInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ Map<String, AttributeInfo> fields) throws AtlasException {
+
+ LOG.debug("Mapping vertex {} to instance {} for fields", instanceVertex, typedInstance.getTypeName(),
+ fields);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ mapVertexToAttribute(instanceVertex, typedInstance, attributeInfo);
+ }
+ }
+
+
+ private void mapVertexToAttribute(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("Mapping attributeInfo {}", attributeInfo.name);
+ final IDataType dataType = attributeInfo.dataType();
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+
+ switch (dataType.getTypeCategory()) {
+ case PRIMITIVE:
+ mapVertexToPrimitive(instanceVertex, typedInstance, attributeInfo);
+ break; // add only if vertex has this attribute
+
+ case ENUM:
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ typedInstance.set(attributeInfo.name,
+ dataType.convert(instanceVertex.<String>getProperty(vertexPropertyName),
+ Multiplicity.REQUIRED));
+ break;
+
+ case ARRAY:
+ mapVertexToArrayInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case MAP:
+ mapVertexToMapInstance(instanceVertex, typedInstance, attributeInfo, vertexPropertyName);
+ break;
+
+ case STRUCT:
+ mapVertexToStructInstance(instanceVertex, typedInstance, attributeInfo);
+ break;
+
+ case TRAIT:
+ // do NOTHING - handled in class
+ break;
+
+ case CLASS:
+ String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+ Object idOrInstance = mapVertexToClassReference(instanceVertex, attributeInfo, relationshipLabel,
+ attributeInfo.dataType());
+ if (idOrInstance != null) {
+ typedInstance.set(attributeInfo.name, idOrInstance);
+ }
+ break;
+
+ default:
+ break;
+ }
+ }
+
+ private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
+ String relationshipLabel, IDataType dataType) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ Iterator<Edge> results = instanceVertex.getEdges(Direction.OUT, relationshipLabel).iterator();
+ if (results.hasNext()) {
+ final Vertex referenceVertex = results.next().getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel, guid);
+ if (attributeInfo.isComposite) {
+ //Also, when you retrieve a type's instance, you get the complete object graph of the composites
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId =
+ new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapVertexToArrayInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo, String propertyName) throws AtlasException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List list = instanceVertex.getProperty(propertyName);
+ if (list == null || list.size() == 0) {
+ return;
+ }
+ DataTypes.ArrayType arrayType = (DataTypes.ArrayType) attributeInfo.dataType();
+ final IDataType elementType = arrayType.getElemType();
+
+ String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
+ ArrayList values = new ArrayList();
+ for (int index = 0; index < list.size(); index++) {
+ values.add(mapVertexToCollectionEntry(instanceVertex, attributeInfo, elementType, list.get(index),
+ edgeLabel));
+ }
+
+ if (values.size() > 0) {
+ typedInstance.set(attributeInfo.name, values);
+ }
+ }
+
+ private Object mapVertexToCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
+ IDataType elementType, Object value, String edgeLabel) throws AtlasException {
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return value;
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ break;
+
+ case STRUCT:
+ return getStructInstanceFromVertex(instanceVertex, elementType, attributeInfo.name, edgeLabel,
+ (String) value);
+
+ case CLASS:
+ return mapVertexToClassReference(instanceVertex, attributeInfo, edgeLabel, elementType, (String) value);
+
+ default:
+ break;
+ }
+
+ throw new IllegalArgumentException();
+ }
+
+ @SuppressWarnings("unchecked")
+ private void mapVertexToMapInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo, final String propertyName) throws AtlasException {
+ LOG.debug("mapping vertex {} to array {}", instanceVertex, attributeInfo.name);
+ List<String> keys = instanceVertex.getProperty(propertyName);
+ if (keys == null || keys.size() == 0) {
+ return;
+ }
+ DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
+ final IDataType valueType = mapType.getValueType();
+
+ HashMap values = new HashMap();
+ for (String key : keys) {
+ final String keyPropertyName = propertyName + "." + key;
+ final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + keyPropertyName;
+ final Object keyValue = instanceVertex.getProperty(keyPropertyName);
+ Object mapValue = mapVertexToCollectionEntry(instanceVertex, attributeInfo, valueType, keyValue, edgeLabel);
+ if (mapValue != null) {
+ values.put(key, mapValue);
+ }
+ }
+
+ if (!values.isEmpty()) {
+ typedInstance.set(attributeInfo.name, values);
+ }
+ }
+
+ private ITypedStruct getStructInstanceFromVertex(Vertex instanceVertex, IDataType elemType,
+ String attributeName, String relationshipLabel, String edgeId) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ LOG.debug("mapping vertex {} to struct {}", structInstanceVertex, attributeName);
+
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
+ elemType.getName());
+ StructType structType = typeSystem.getDataType(StructType.class, elemType.getName());
+ ITypedStruct structInstance = structType.createInstance();
+ mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
+ return structInstance;
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ private Object mapVertexToClassReference(Vertex instanceVertex, AttributeInfo attributeInfo,
+ String relationshipLabel, IDataType dataType, String edgeId) throws AtlasException {
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ if (edgeId.equals(String.valueOf(edge.getId()))) {
+ final Vertex referenceVertex = edge.getVertex(Direction.IN);
+ if (referenceVertex != null) {
+ final String guid = referenceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ LOG.debug("Found vertex {} for label {} with guid {}", referenceVertex, relationshipLabel,
+ guid);
+ if (attributeInfo.isComposite) {
+ //Also, when you retrieve a type's instance, you get the complete object graph of the composites
+ LOG.debug("Found composite, mapping vertex to instance");
+ return mapGraphToTypedInstance(guid, referenceVertex);
+ } else {
+ Id referenceId =
+ new Id(guid, referenceVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ dataType.getName());
+ LOG.debug("Found non-composite, adding id {} ", referenceId);
+ return referenceId;
+ }
+ }
+
+ break;
+ }
+ }
+
+ return null;
+ }
+
+ private void mapVertexToStructInstance(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("mapping vertex {} to struct {}", instanceVertex, attributeInfo.name);
+ StructType structType = typeSystem.getDataType(StructType.class, attributeInfo.dataType().getName());
+ ITypedStruct structInstance = null;
+
+ String relationshipLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ final Iterable<Edge> edges = instanceVertex.getEdges(Direction.OUT, relationshipLabel);
+ if (edges.iterator().hasNext()) {
+ structInstance = structType.createInstance();
+ typedInstance.set(attributeInfo.name, structInstance);
+ }
+
+ for (Edge edge : edges) {
+ final Vertex structInstanceVertex = edge.getVertex(Direction.IN);
+ if (structInstanceVertex != null) {
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", structInstanceVertex,
+ structInstance.getTypeName());
+ mapVertexToInstance(structInstanceVertex, structInstance, structType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToTraitInstance(Vertex instanceVertex, ITypedReferenceableInstance typedInstance,
+ String traitName, TraitType traitType) throws AtlasException {
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ mapVertexToTraitInstance(instanceVertex, typedInstance.getTypeName(), traitName, traitType, traitInstance);
+ }
+
+ private void mapVertexToTraitInstance(Vertex instanceVertex, String typedInstanceTypeName, String traitName,
+ TraitType traitType, ITypedStruct traitInstance) throws AtlasException {
+ String relationshipLabel = GraphHelper.getTraitLabel(typedInstanceTypeName, traitName);
+ LOG.debug("Finding edge for {} -> label {} ", instanceVertex, relationshipLabel);
+ for (Edge edge : instanceVertex.getEdges(Direction.OUT, relationshipLabel)) {
+ final Vertex traitInstanceVertex = edge.getVertex(Direction.IN);
+ if (traitInstanceVertex != null) {
+ LOG.debug("Found trait instance vertex {}, mapping to instance {} ", traitInstanceVertex,
+ traitInstance.getTypeName());
+ mapVertexToInstance(traitInstanceVertex, traitInstance, traitType.fieldMapping().fields);
+ break;
+ }
+ }
+ }
+
+ private void mapVertexToPrimitive(Vertex instanceVertex, ITypedInstance typedInstance,
+ AttributeInfo attributeInfo) throws AtlasException {
+ LOG.debug("Adding primitive {} from vertex {}", attributeInfo, instanceVertex);
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ if (instanceVertex.getProperty(vertexPropertyName) == null) {
+ return;
+ }
+
+ if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ typedInstance.setString(attributeInfo.name, instanceVertex.<String>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ typedInstance.setShort(attributeInfo.name, instanceVertex.<Short>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ typedInstance.setInt(attributeInfo.name, instanceVertex.<Integer>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ typedInstance.setBigInt(attributeInfo.name, instanceVertex.<BigInteger>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ typedInstance.setBoolean(attributeInfo.name, instanceVertex.<Boolean>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ typedInstance.setByte(attributeInfo.name, instanceVertex.<Byte>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ typedInstance.setLong(attributeInfo.name, instanceVertex.<Long>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ typedInstance.setFloat(attributeInfo.name, instanceVertex.<Float>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ typedInstance.setDouble(attributeInfo.name, instanceVertex.<Double>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ typedInstance
+ .setBigDecimal(attributeInfo.name, instanceVertex.<BigDecimal>getProperty(vertexPropertyName));
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ final Long dateVal = instanceVertex.<Long>getProperty(vertexPropertyName);
+ typedInstance.setDate(attributeInfo.name, new Date(dateVal));
+ }
+ }
+
+ public ITypedInstance getReferredEntity(String edgeId, IDataType<?> referredType) throws AtlasException {
+ final Edge edge = titanGraph.getEdge(edgeId);
+ if (edge != null) {
+ final Vertex referredVertex = edge.getVertex(Direction.IN);
+ if (referredVertex != null) {
+ switch (referredType.getTypeCategory()) {
+ case STRUCT:
+ LOG.debug("Found struct instance vertex {}, mapping to instance {} ", referredVertex,
+ referredType.getName());
+ StructType structType = (StructType) referredType;
+ ITypedStruct instance = structType.createInstance();
+ Map<String, AttributeInfo> fields = structType.fieldMapping().fields;
+ mapVertexToInstance(referredVertex, instance, fields);
+ return instance;
+ case CLASS:
+ //TODO isComposite handling for class loads
+ final String guid = referredVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ Id referenceId =
+ new Id(guid, referredVertex.<Integer>getProperty(Constants.VERSION_PROPERTY_KEY),
+ referredType.getName());
+ return referenceId;
+ default:
+ throw new UnsupportedOperationException("Loading " + referredType.getTypeCategory() + " is not supported");
+ }
+ }
+ }
+ return null;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
new file mode 100644
index 0000000..7ef5c50
--- /dev/null
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/TypedInstanceToGraphMapper.java
@@ -0,0 +1,633 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.atlas.repository.graph;
+
+import com.thinkaurelius.titan.core.SchemaViolationException;
+import com.tinkerpop.blueprints.Direction;
+import com.tinkerpop.blueprints.Edge;
+import com.tinkerpop.blueprints.Vertex;
+import org.apache.atlas.AtlasException;
+import org.apache.atlas.repository.Constants;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.IReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedInstance;
+import org.apache.atlas.typesystem.ITypedReferenceableInstance;
+import org.apache.atlas.typesystem.ITypedStruct;
+import org.apache.atlas.typesystem.exception.EntityExistsException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.persistence.Id;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.typesystem.types.AttributeInfo;
+import org.apache.atlas.typesystem.types.ClassType;
+import org.apache.atlas.typesystem.types.DataTypes;
+import org.apache.atlas.typesystem.types.EnumValue;
+import org.apache.atlas.typesystem.types.IDataType;
+import org.apache.atlas.typesystem.types.Multiplicity;
+import org.apache.atlas.typesystem.types.ObjectGraphWalker;
+import org.apache.atlas.typesystem.types.TraitType;
+import org.apache.atlas.typesystem.types.TypeSystem;
+import org.apache.atlas.utils.MD5Utils;
+import org.apache.commons.lang3.tuple.Pair;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.security.MessageDigest;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+public final class TypedInstanceToGraphMapper {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TypedInstanceToGraphMapper.class);
+ private final Map<Id, Vertex> idToVertexMap = new HashMap<>();
+ private final TypeSystem typeSystem = TypeSystem.getInstance();
+
+ private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
+
+ private static final GraphHelper graphHelper = GraphHelper.getInstance();
+
+ private final String SIGNATURE_HASH_PROPERTY_KEY = Constants.INTERNAL_PROPERTY_KEY_PREFIX + "signature";
+
+ public enum Operation {
+ CREATE,
+ UPDATE_PARTIAL,
+ UPDATE_FULL,
+ DELETE
+ }
+
+ public TypedInstanceToGraphMapper(GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
+ this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
+ }
+
+ String[] mapTypedInstanceToGraph(Operation operation, ITypedReferenceableInstance... typedInstances)
+ throws AtlasException {
+ List<String> guids = new ArrayList<>();
+ for (ITypedReferenceableInstance typedInstance : typedInstances) {
+ Collection<IReferenceableInstance> newInstances = walkClassInstances(typedInstance);
+ Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> instancesPair =
+ createVerticesAndDiscoverInstances(newInstances);
+
+ switch (operation) {
+ case CREATE:
+ addOrUpdateAttributesAndTraits(operation, instancesPair.getLeft());
+ addFullTextProperty(instancesPair.getLeft());
+ break;
+
+ case UPDATE_FULL:
+ case UPDATE_PARTIAL:
+ List<ITypedReferenceableInstance> instancesForUpdate = instancesPair.getLeft();
+ instancesForUpdate.addAll(instancesPair.getRight());
+ addOrUpdateAttributesAndTraits(operation, instancesForUpdate);
+ addFullTextProperty(instancesForUpdate);
+ break;
+
+ case DELETE:
+ throw new UnsupportedOperationException("Not handled - " + operation);
+ }
+
+ //Return guid for
+ addToGuids(typedInstance, guids);
+ }
+ return guids.toArray(new String[guids.size()]);
+ }
+
+ private Collection<IReferenceableInstance> walkClassInstances(ITypedReferenceableInstance typedInstance)
+ throws RepositoryException {
+
+ EntityProcessor entityProcessor = new EntityProcessor();
+ try {
+ LOG.debug("Walking the object graph for instance {}", typedInstance.getTypeName());
+ new ObjectGraphWalker(typeSystem, entityProcessor, typedInstance).walk();
+ } catch (AtlasException me) {
+ throw new RepositoryException("TypeSystem error when walking the ObjectGraph", me);
+ }
+
+ entityProcessor.addInstanceIfNotExists(typedInstance);
+ return entityProcessor.getInstances();
+ }
+
+ private void addOrUpdateAttributesAndTraits(Operation operation, List<ITypedReferenceableInstance> instances) throws AtlasException {
+ for (ITypedReferenceableInstance instance : instances) {
+ try {
+ //new vertex, set all the properties
+ addOrUpdateAttributesAndTraits(operation, instance);
+ } catch (SchemaViolationException e) {
+ throw new EntityExistsException(instance, e);
+ }
+ }
+ }
+
+ private void addOrUpdateAttributesAndTraits(Operation operation, ITypedReferenceableInstance typedInstance)
+ throws AtlasException {
+ LOG.debug("Adding/Updating typed instance {}", typedInstance.getTypeName());
+
+ Id id = typedInstance.getId();
+ if (id == null) { // oops
+ throw new RepositoryException("id cannot be null");
+ }
+
+ Vertex instanceVertex = idToVertexMap.get(id);
+
+ // add the attributes for the instance
+ ClassType classType = typeSystem.getDataType(ClassType.class, typedInstance.getTypeName());
+ final Map<String, AttributeInfo> fields = classType.fieldMapping().fields;
+
+ mapInstanceToVertex(typedInstance, instanceVertex, fields, false, operation);
+
+ if (Operation.CREATE.equals(operation)) {
+ //TODO - Handle Trait updates
+ addTraits(typedInstance, instanceVertex, classType);
+ }
+ }
+
+ private void mapInstanceToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ Map<String, AttributeInfo> fields, boolean mapOnlyUniqueAttributes, Operation operation)
+ throws AtlasException {
+ LOG.debug("Mapping instance {} of {} to vertex {}", typedInstance, typedInstance.getTypeName(),
+ instanceVertex);
+ for (AttributeInfo attributeInfo : fields.values()) {
+ if (mapOnlyUniqueAttributes && !attributeInfo.isUnique) {
+ continue;
+ }
+ mapAttributesToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ }
+ }
+
+ void mapAttributesToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+ LOG.debug("mapping attribute {} = {}", attributeInfo.name, attrValue);
+ final String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo);
+
+ if (attrValue != null || operation == Operation.UPDATE_FULL) {
+ switch (attributeInfo.dataType().getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ mapPrimitiveOrEnumToVertex(typedInstance, instanceVertex, attributeInfo);
+ break;
+
+ case ARRAY:
+ mapArrayCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ break;
+
+ case MAP:
+ mapMapCollectionToVertex(typedInstance, instanceVertex, attributeInfo, operation);
+ break;
+
+ case STRUCT:
+ case CLASS:
+ Iterator<Edge> outGoingEdgesIterator =
+ GraphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel).iterator();
+ String currentEntry =
+ outGoingEdgesIterator.hasNext() ? outGoingEdgesIterator.next().getId().toString() : null;
+ addOrUpdateCollectionEntry(instanceVertex, attributeInfo, attributeInfo.dataType(), attrValue,
+ currentEntry, propertyName, operation);
+ break;
+
+ case TRAIT:
+ // do NOTHING - this is taken care of earlier
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: " + attributeInfo.dataType().getTypeCategory());
+ }
+ }
+ }
+
+ private Pair<List<ITypedReferenceableInstance>, List<ITypedReferenceableInstance>> createVerticesAndDiscoverInstances(
+ Collection<IReferenceableInstance> instances) throws AtlasException {
+
+ List<ITypedReferenceableInstance> instancesToCreate = new ArrayList<>();
+ List<ITypedReferenceableInstance> instancesToUpdate = new ArrayList<>();
+
+ for (IReferenceableInstance instance : instances) {
+ Id id = instance.getId();
+ if (!idToVertexMap.containsKey(id)) {
+ Vertex instanceVertex;
+ if (id.isAssigned()) { // has a GUID
+ instanceVertex = graphHelper.getVertexForGUID(id.id);
+ if (!(instance instanceof ReferenceableInstance)) {
+ throw new IllegalStateException(
+ String.format("%s is not of type ITypedReferenceableInstance", instance));
+ }
+ instancesToUpdate.add((ITypedReferenceableInstance) instance);
+ } else {
+ //Check if there is already an instance with the same unique attribute value
+ ClassType classType = typeSystem.getDataType(ClassType.class, instance.getTypeName());
+ instanceVertex = graphHelper.getVertexForInstanceByUniqueAttribute(classType, instance);
+
+ //no entity with the given unique attribute, create new
+ if (instanceVertex == null) {
+ ITypedReferenceableInstance newInstance = classType.convert(instance, Multiplicity.REQUIRED);
+ instanceVertex = graphHelper.createVertexWithIdentity(newInstance, classType.getAllSuperTypeNames());
+ instancesToCreate.add(newInstance);
+
+ //Map only unique attributes for cases of circular references
+ mapInstanceToVertex(newInstance, instanceVertex, classType.fieldMapping().fields, true, Operation.CREATE);
+ } else {
+ if (!(instance instanceof ReferenceableInstance)) {
+ throw new IllegalStateException(
+ String.format("%s is not of type ITypedReferenceableInstance", instance));
+ }
+ instancesToUpdate.add((ITypedReferenceableInstance) instance);
+ }
+ }
+
+ idToVertexMap.put(id, instanceVertex);
+ }
+ }
+ return Pair.of(instancesToCreate, instancesToUpdate);
+ }
+
+ private void addToGuids(ITypedReferenceableInstance typedInstance, List<String> guids) {
+ Vertex instanceVertex = idToVertexMap.get(typedInstance.getId());
+ String guid = instanceVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ guids.add(guid);
+ }
+
+ private void addFullTextProperty(List<ITypedReferenceableInstance> instances) throws AtlasException {
+ FullTextMapper fulltextMapper = new FullTextMapper(graphToTypedInstanceMapper);
+ for (ITypedReferenceableInstance typedInstance : instances) { // Traverse
+ Vertex instanceVertex = getClassVertex(typedInstance);
+ String fullText = fulltextMapper.mapRecursive(instanceVertex, true);
+ GraphHelper.setProperty(instanceVertex, Constants.ENTITY_TEXT_PROPERTY_KEY, fullText);
+ }
+ }
+
+ private void addTraits(ITypedReferenceableInstance typedInstance, Vertex instanceVertex, ClassType classType) throws AtlasException {
+ for (String traitName : typedInstance.getTraits()) {
+ LOG.debug("mapping trait {}", traitName);
+ GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
+ ITypedStruct traitInstance = (ITypedStruct) typedInstance.getTrait(traitName);
+
+ // add the attributes for the trait instance
+ mapTraitInstanceToVertex(traitInstance, classType, instanceVertex);
+ }
+ }
+
+ /******************************************** STRUCT **************************************************/
+
+ private Pair<Vertex, Edge> updateStructVertex(ITypedStruct structInstance, Edge relEdge, Operation operation) throws AtlasException {
+ //Already existing vertex. Update
+ Vertex structInstanceVertex = relEdge.getVertex(Direction.IN);
+
+ // Update attributes
+ final MessageDigest digester = MD5Utils.getDigester();
+ String newSignature = structInstance.getSignatureHash(digester);
+ String curSignature = structInstanceVertex.getProperty(SIGNATURE_HASH_PROPERTY_KEY);
+
+ if (!newSignature.equals(curSignature)) {
+ //Update struct vertex instance only if there is a change
+ LOG.debug("Updating struct {} since signature has changed {} {} ", structInstance, curSignature, newSignature);
+ mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, operation);
+ GraphHelper.setProperty(structInstanceVertex, SIGNATURE_HASH_PROPERTY_KEY, String.valueOf(newSignature));
+ }
+ return Pair.of(structInstanceVertex, relEdge);
+ }
+
+ private Pair<Vertex, Edge> addStructVertex(ITypedStruct structInstance, Vertex instanceVertex, AttributeInfo attributeInfo, String edgeLabel) throws AtlasException {
+ // add a new vertex for the struct or trait instance
+ Vertex structInstanceVertex = graphHelper.createVertexWithoutIdentity(structInstance.getTypeName(), null,
+ Collections.<String>emptySet()); // no super types for struct type
+ LOG.debug("created vertex {} for struct {} value {}", structInstanceVertex, attributeInfo.name, structInstance);
+
+ // map all the attributes to this new vertex
+ mapInstanceToVertex(structInstance, structInstanceVertex, structInstance.fieldMapping().fields, false, Operation.CREATE);
+ // add an edge to the newly created vertex from the parent
+ Edge relEdge = graphHelper.addEdge(instanceVertex, structInstanceVertex, edgeLabel);
+
+ return Pair.of(structInstanceVertex, relEdge);
+ }
+
+ /******************************************** ARRAY **************************************************/
+
+ private void mapArrayCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
+ attributeInfo.name);
+ List newElements = (List) typedInstance.get(attributeInfo.name);
+ boolean empty = (newElements == null || newElements.isEmpty());
+ if (!empty || operation == Operation.UPDATE_FULL) {
+ String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ List<String> currentEntries = instanceVertex.getProperty(propertyName);
+
+ IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
+ List<String> newEntries = new ArrayList<>();
+
+ if (newElements != null && !newElements.isEmpty()) {
+ int index = 0;
+ for (; index < newElements.size(); index++) {
+ String currentEntry =
+ (currentEntries != null && index < currentEntries.size()) ? currentEntries.get(index) : null;
+ String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
+ newElements.get(index), currentEntry, propertyName, operation);
+ newEntries.add(newEntry);
+ }
+
+ //Remove extra entries in the list
+ if (currentEntries != null) {
+ if (index < currentEntries.size()) {
+ for (; index < currentEntries.size(); index++) {
+ removeUnusedReference(currentEntries.get(index), attributeInfo, elementType);
+ }
+ }
+ }
+ }
+
+ // for dereference on way out
+ GraphHelper.setProperty(instanceVertex, propertyName, newEntries);
+ }
+ }
+
+ /******************************************** MAP **************************************************/
+
+ private void mapMapCollectionToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo, Operation operation) throws AtlasException {
+ LOG.debug("Mapping instance {} to vertex {} for name {}", typedInstance.getTypeName(), instanceVertex,
+ attributeInfo.name);
+ @SuppressWarnings("unchecked") Map<Object, Object> collection =
+ (Map<Object, Object>) typedInstance.get(attributeInfo.name);
+ boolean empty = (collection == null || collection.isEmpty());
+ if (!empty || operation == Operation.UPDATE_FULL) {
+
+ String propertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ IDataType elementType = ((DataTypes.MapType) attributeInfo.dataType()).getValueType();
+
+ if (!empty) {
+ for (Map.Entry entry : collection.entrySet()) {
+ String myPropertyName = propertyName + "." + entry.getKey().toString();
+
+ String currentEntry = instanceVertex.getProperty(myPropertyName);
+ String newEntry = addOrUpdateCollectionEntry(instanceVertex, attributeInfo, elementType,
+ entry.getValue(), currentEntry, myPropertyName, operation);
+
+ //Add/Update/Remove property value
+ GraphHelper.setProperty(instanceVertex, myPropertyName, newEntry);
+ }
+
+ //Remove unused key references
+ List<Object> origKeys = instanceVertex.getProperty(propertyName);
+ if (origKeys != null) {
+ if (collection != null) {
+ origKeys.removeAll(collection.keySet());
+ }
+ for (Object unusedKey : origKeys) {
+ String edgeLabel = GraphHelper.getEdgeLabel(typedInstance, attributeInfo) + "." + unusedKey;
+ if (instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().hasNext()) {
+ Edge edge = instanceVertex.getEdges(Direction.OUT, edgeLabel).iterator().next();
+ removeUnusedReference(edge.getId().toString(), attributeInfo,
+ ((DataTypes.MapType) attributeInfo.dataType()).getValueType());
+ }
+ }
+ }
+
+ }
+
+ // for dereference on way out
+ GraphHelper.setProperty(instanceVertex, propertyName, collection == null ? null : new ArrayList(collection.keySet()));
+ }
+ }
+
+ /******************************************** ARRAY & MAP **************************************************/
+
+ private String addOrUpdateCollectionEntry(Vertex instanceVertex, AttributeInfo attributeInfo,
+ IDataType elementType, Object newVal, String curVal, String propertyName,
+ Operation operation)
+ throws AtlasException {
+
+ final String edgeLabel = GraphHelper.EDGE_LABEL_PREFIX + propertyName;
+ switch (elementType.getTypeCategory()) {
+ case PRIMITIVE:
+ case ENUM:
+ return newVal != null ? newVal.toString() : null;
+
+ case ARRAY:
+ case MAP:
+ case TRAIT:
+ // do nothing
+ return null;
+
+ case STRUCT:
+ return addOrUpdateStruct(instanceVertex, attributeInfo, elementType, (ITypedStruct) newVal, curVal, edgeLabel, operation);
+
+ case CLASS:
+ return addOrUpdateClassVertex(instanceVertex, attributeInfo, elementType,
+ (ITypedReferenceableInstance) newVal, curVal, edgeLabel, operation);
+
+ default:
+ throw new IllegalArgumentException("Unknown type category: " + elementType.getTypeCategory());
+ }
+ }
+
+ private String addOrUpdateStruct(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
+ ITypedStruct structAttr, String curVal,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Pair<Vertex, Edge> vertexEdgePair = null;
+ if (curVal != null && structAttr == null) {
+ //remove edge
+ removeUnusedReference(curVal, attributeInfo, elementType);
+ } else if (curVal != null && structAttr != null) {
+ //update
+ Edge edge = graphHelper.getOutGoingEdgeById(curVal);
+ vertexEdgePair = updateStructVertex(structAttr, edge, operation);
+ } else if (structAttr != null) {
+ //add
+ vertexEdgePair = addStructVertex(structAttr, instanceVertex, attributeInfo, edgeLabel);
+ }
+
+ return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
+ }
+
+ private String addOrUpdateClassVertex(Vertex instanceVertex, AttributeInfo attributeInfo, IDataType elementType,
+ ITypedReferenceableInstance newVal, String curVal,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Vertex toVertex = getClassVertex(newVal);
+ if(toVertex == null && newVal != null) {
+ LOG.error("Could not find vertex for Class Reference " + newVal);
+ throw new EntityNotFoundException("Could not find vertex for Class Reference " + newVal);
+ }
+
+ Pair<Vertex, Edge> vertexEdgePair = null;
+ if (curVal != null && newVal == null) {
+ //remove edge
+ removeUnusedReference(curVal, attributeInfo, elementType);
+ } else if (curVal != null && newVal != null) {
+ Edge edge = graphHelper.getOutGoingEdgeById(curVal);
+ Id classRefId = getId(newVal);
+ vertexEdgePair = updateClassEdge(classRefId, newVal, instanceVertex, edge, toVertex, attributeInfo, elementType, edgeLabel, operation);
+ } else if (newVal != null){
+ vertexEdgePair = addClassEdge(instanceVertex, toVertex, edgeLabel);
+ }
+
+ return (vertexEdgePair != null) ? vertexEdgePair.getRight().getId().toString() : null;
+ }
+
+ /******************************************** CLASS **************************************************/
+
+ private Pair<Vertex, Edge> addClassEdge(Vertex instanceVertex, Vertex toVertex, String edgeLabel) throws AtlasException {
+ // add an edge to the class vertex from the instance
+ Edge edge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
+ return Pair.of(toVertex, edge);
+ }
+
+ private Vertex getClassVertex(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
+ Vertex referenceVertex = null;
+ Id id = null;
+ if (typedReference != null) {
+ id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
+ if (id.isAssigned()) {
+ referenceVertex = graphHelper.getVertexForGUID(id.id);
+ } else {
+ referenceVertex = idToVertexMap.get(id);
+ }
+ }
+
+ return referenceVertex;
+ }
+
+ private Id getId(ITypedReferenceableInstance typedReference) throws EntityNotFoundException {
+ Id id = null;
+ if (typedReference != null) {
+ id = typedReference instanceof Id ? (Id) typedReference : typedReference.getId();
+ }
+
+ if (id.isUnassigned()) {
+ Vertex classVertex = idToVertexMap.get(id);
+ String guid = classVertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ id = new Id(guid, 0, typedReference.getTypeName());
+ }
+ return id;
+ }
+
+
+ private Pair<Vertex, Edge> updateClassEdge(Id id, final ITypedReferenceableInstance typedInstance,
+ Vertex instanceVertex, Edge edge, Vertex toVertex,
+ AttributeInfo attributeInfo, IDataType dataType,
+ String edgeLabel, Operation operation) throws AtlasException {
+ Pair<Vertex, Edge> result = Pair.of(toVertex, edge);
+ Edge newEdge = edge;
+ // Update edge if it exists
+ Vertex invertex = edge.getVertex(Direction.IN);
+ String currentGUID = invertex.getProperty(Constants.GUID_PROPERTY_KEY);
+ Id currentId = new Id(currentGUID, 0, (String) invertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY));
+ if (!currentId.equals(id)) {
+ // add an edge to the class vertex from the instance
+ if(toVertex != null) {
+ newEdge = graphHelper.addEdge(instanceVertex, toVertex, edgeLabel);
+ result = Pair.of(toVertex, newEdge);
+ }
+ removeUnusedReference(edge.getId().toString(), attributeInfo, dataType);
+ }
+
+ if (attributeInfo.isComposite) {
+ //Update the attributes also if composite
+ if (typedInstance.fieldMapping() != null) {
+ //In case of Id instance, fieldMapping is null
+ mapInstanceToVertex(typedInstance, toVertex, typedInstance.fieldMapping().fields , false, operation);
+ //Update full text for the updated composite vertex
+ addFullTextProperty(new ArrayList<ITypedReferenceableInstance>() {{ add(typedInstance); }});
+ }
+ }
+
+ return result;
+ }
+
+ /******************************************** TRAITS ****************************************************/
+
+ void mapTraitInstanceToVertex(ITypedStruct traitInstance, IDataType entityType, Vertex parentInstanceVertex)
+ throws AtlasException {
+ // add a new vertex for the struct or trait instance
+ final String traitName = traitInstance.getTypeName();
+ Vertex traitInstanceVertex = graphHelper.createVertexWithoutIdentity(traitInstance.getTypeName(), null,
+ typeSystem.getDataType(TraitType.class, traitName).getAllSuperTypeNames());
+ LOG.debug("created vertex {} for trait {}", traitInstanceVertex, traitName);
+
+ // map all the attributes to this newly created vertex
+ mapInstanceToVertex(traitInstance, traitInstanceVertex, traitInstance.fieldMapping().fields, false, Operation.CREATE);
+
+ // add an edge to the newly created vertex from the parent
+ String relationshipLabel = GraphHelper.getTraitLabel(entityType.getName(), traitName);
+ graphHelper.addEdge(parentInstanceVertex, traitInstanceVertex, relationshipLabel);
+ }
+
+ /******************************************** PRIMITIVES **************************************************/
+
+ private void mapPrimitiveOrEnumToVertex(ITypedInstance typedInstance, Vertex instanceVertex,
+ AttributeInfo attributeInfo) throws AtlasException {
+ Object attrValue = typedInstance.get(attributeInfo.name);
+
+ final String vertexPropertyName = GraphHelper.getQualifiedFieldName(typedInstance, attributeInfo);
+ Object propertyValue = null;
+
+ if (attrValue == null ) {
+ propertyValue = null;
+ } else if (attributeInfo.dataType() == DataTypes.STRING_TYPE) {
+ propertyValue = typedInstance.getString(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.SHORT_TYPE) {
+ propertyValue = typedInstance.getShort(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.INT_TYPE) {
+ propertyValue = typedInstance.getInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGINTEGER_TYPE) {
+ propertyValue = typedInstance.getBigInt(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BOOLEAN_TYPE) {
+ propertyValue = typedInstance.getBoolean(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BYTE_TYPE) {
+ propertyValue = typedInstance.getByte(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.LONG_TYPE) {
+ propertyValue = typedInstance.getLong(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.FLOAT_TYPE) {
+ propertyValue = typedInstance.getFloat(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DOUBLE_TYPE) {
+ propertyValue = typedInstance.getDouble(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.BIGDECIMAL_TYPE) {
+ propertyValue = typedInstance.getBigDecimal(attributeInfo.name);
+ } else if (attributeInfo.dataType() == DataTypes.DATE_TYPE) {
+ final Date dateVal = typedInstance.getDate(attributeInfo.name);
+ //Convert Property value to Long while persisting
+ propertyValue = dateVal.getTime();
+ } else if (attributeInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.ENUM) {
+ if (attrValue != null) {
+ propertyValue = ((EnumValue)attrValue).value;
+ }
+ }
+
+
+ GraphHelper.setProperty(instanceVertex, vertexPropertyName, propertyValue);
+ }
+
+ private Edge removeUnusedReference(String edgeId, AttributeInfo attributeInfo, IDataType<?> elementType) {
+ //Remove edges for property values which do not exist any more
+ Edge removedRelation = null;
+ switch (elementType.getTypeCategory()) {
+ case STRUCT:
+ removedRelation = graphHelper.removeRelation(edgeId, true);
+ //Remove the vertex from state so that further processing no longer uses this
+ break;
+ case CLASS:
+ removedRelation = graphHelper.removeRelation(edgeId, attributeInfo.isComposite);
+ break;
+ }
+ return removedRelation;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
index 86141e2..9351be9 100755
--- a/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
+++ b/repository/src/main/java/org/apache/atlas/repository/memory/ReplaceIdWithInstance.java
@@ -55,7 +55,6 @@ public class ReplaceIdWithInstance implements ObjectGraphWalker.NodeProcessor {
} else if (!nd.aInfo.isComposite || nd.value == null) {
// do nothing
} else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
-
if (nd.value != null && nd.value instanceof Id) {
Id id = (Id) nd.value;
ITypedReferenceableInstance r = getInstance(id);
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
index 0c90709..fb782a2 100755
--- a/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
+++ b/repository/src/main/java/org/apache/atlas/services/DefaultMetadataService.java
@@ -23,8 +23,11 @@ import com.google.common.collect.ImmutableList;
import com.google.inject.Provider;
import org.apache.atlas.AtlasClient;
import org.apache.atlas.AtlasException;
-import org.apache.atlas.ParamChecker;
-import org.apache.atlas.TypeNotFoundException;
+import org.apache.atlas.repository.RepositoryException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.TypeNotFoundException;
+import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
+import org.apache.atlas.utils.ParamChecker;
import org.apache.atlas.classification.InterfaceAudience;
import org.apache.atlas.listener.EntityChangeListener;
import org.apache.atlas.listener.TypesChangeListener;
@@ -39,6 +42,7 @@ import org.apache.atlas.typesystem.Struct;
import org.apache.atlas.typesystem.TypesDef;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.json.TypesSerialization;
+import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.types.AttributeDefinition;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
@@ -61,6 +65,7 @@ import scala.actors.threadpool.Arrays;
import javax.inject.Inject;
import javax.inject.Singleton;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
@@ -282,7 +287,7 @@ public class DefaultMetadataService implements MetadataService {
entitites.add(repository.getEntityDefinition(guid));
}
- onEntitiesAddedToRepo(entitites);
+ onEntitiesAdded(entitites);
return new JSONArray(Arrays.asList(guids)).toString();
}
@@ -325,12 +330,17 @@ public class DefaultMetadataService implements MetadataService {
return InstanceSerialization.toJson(instance, true);
}
- @Override
- public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
+ private ITypedReferenceableInstance getEntityDefinitionReference(String entityType, String attribute, String value)
+ throws AtlasException {
validateTypeExists(entityType);
validateUniqueAttribute(entityType, attribute);
- final ITypedReferenceableInstance instance = repository.getEntityDefinition(entityType, attribute, value);
+ return repository.getEntityDefinition(entityType, attribute, value);
+ }
+
+ @Override
+ public String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException {
+ final ITypedReferenceableInstance instance = getEntityDefinitionReference(entityType, attribute, value);
return InstanceSerialization.toJson(instance, true);
}
@@ -361,15 +371,146 @@ public class DefaultMetadataService implements MetadataService {
return repository.getEntityList(entityType);
}
+ /**
+ * Updates an entity, instance of the type based on the guid set.
+ *
+ * @param entityInstanceDefinition json array of entity definitions
+ * @return guids - json array of guids
+ */
@Override
- public void updateEntity(String guid, String property, String value) throws AtlasException {
+ public String updateEntities(String entityInstanceDefinition) throws AtlasException {
+
+ ParamChecker.notEmpty(entityInstanceDefinition, "Entity instance definition cannot be empty");
+ ITypedReferenceableInstance[] typedInstances = deserializeClassInstances(entityInstanceDefinition);
+
+ String[] guids = repository.updateEntities(typedInstances);
+ onEntitiesAdded(Arrays.asList(typedInstances));
+
+ return new JSONArray(Arrays.asList(guids)).toString();
+ }
+
+ @Override
+ public void updateEntityAttributeByGuid(final String guid, String attributeName, String value) throws AtlasException {
ParamChecker.notEmpty(guid, "guid cannot be null");
- ParamChecker.notEmpty(property, "property cannot be null");
+ ParamChecker.notEmpty(attributeName, "property cannot be null");
ParamChecker.notEmpty(value, "property value cannot be null");
- repository.updateEntity(guid, property, value);
+ ITypedReferenceableInstance existInstance = validateEntityExists(guid);
+ ClassType type = typeSystem.getDataType(ClassType.class, existInstance.getTypeName());
+ ITypedReferenceableInstance newInstance = type.createInstance();
+
+ AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
+ if (attributeInfo == null) {
+ throw new AtlasException("Invalid property " + attributeName + " for entity " + existInstance.getTypeName());
+ }
+
+ DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
+
+ switch(attrTypeCategory) {
+ case PRIMITIVE:
+ newInstance.set(attributeName, value);
+ break;
+ case CLASS:
+ Id id = new Id(value, 0, attributeInfo.dataType().getName());
+ newInstance.set(attributeName, id);
+ break;
+ default:
+ throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
+ }
+
+ ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
+ repository.updatePartial(newInstance);
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(repository.getEntityDefinition(guid));
+ }});
+ }
+
+ private ITypedReferenceableInstance validateEntityExists(String guid)
+ throws EntityNotFoundException, RepositoryException {
+ final ITypedReferenceableInstance instance = repository.getEntityDefinition(guid);
+ if (instance == null) {
+ throw new EntityNotFoundException(String.format("Entity with guid %s not found ", guid));
+ }
+ return instance;
+ }
+
+ @Override
+ public void updateEntityPartialByGuid(final String guid, Referenceable newEntity) throws AtlasException {
+ ParamChecker.notEmpty(guid, "guid cannot be null");
+ ParamChecker.notNull(newEntity, "updatedEntity cannot be null");
+ ITypedReferenceableInstance existInstance = validateEntityExists(guid);
- onEntityUpdated(repository.getEntityDefinition(guid));
+ ITypedReferenceableInstance newInstance = convertToTypedInstance(newEntity, existInstance.getTypeName());
+ ((ReferenceableInstance)newInstance).replaceWithNewId(new Id(guid, 0, newInstance.getTypeName()));
+
+ repository.updatePartial(newInstance);
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(repository.getEntityDefinition(guid));
+ }});
+ }
+
+ private ITypedReferenceableInstance convertToTypedInstance(Referenceable updatedEntity, String typeName) throws AtlasException {
+ ClassType type = typeSystem.getDataType(ClassType.class, typeName);
+ ITypedReferenceableInstance newInstance = type.createInstance();
+
+ for (String attributeName : updatedEntity.getValuesMap().keySet()) {
+ AttributeInfo attributeInfo = type.fieldMapping.fields.get(attributeName);
+ if (attributeInfo == null) {
+ throw new AtlasException("Invalid property " + attributeName + " for entity " + updatedEntity);
+ }
+
+ DataTypes.TypeCategory attrTypeCategory = attributeInfo.dataType().getTypeCategory();
+ Object value = updatedEntity.get(attributeName);
+ if (value != null) {
+ switch (attrTypeCategory) {
+ case CLASS:
+ if (value instanceof Referenceable) {
+ newInstance.set(attributeName, value);
+ } else {
+ Id id = new Id((String) value, 0, attributeInfo.dataType().getName());
+ newInstance.set(attributeName, id);
+ }
+ break;
+
+ case ENUM:
+ case PRIMITIVE:
+ case ARRAY:
+ case STRUCT:
+ case MAP:
+ newInstance.set(attributeName, value);
+ break;
+
+ case TRAIT:
+ //TODO - handle trait updates as well?
+ default:
+ throw new AtlasException("Update of " + attrTypeCategory + " is not supported");
+ }
+ }
+ }
+
+ return newInstance;
+ }
+
+ @Override
+ public String updateEntityByUniqueAttribute(String typeName, String uniqueAttributeName, String attrValue,
+ Referenceable updatedEntity) throws AtlasException {
+ ParamChecker.notEmpty(typeName, "typeName cannot be null");
+ ParamChecker.notEmpty(uniqueAttributeName, "uniqueAttributeName cannot be null");
+ ParamChecker.notNull(attrValue, "value cannot be null");
+ ParamChecker.notNull(updatedEntity, "updatedEntity cannot be null");
+
+ ITypedReferenceableInstance oldInstance = getEntityDefinitionReference(typeName, uniqueAttributeName, attrValue);
+
+ final ITypedReferenceableInstance newInstance = convertToTypedInstance(updatedEntity, typeName);
+ ((ReferenceableInstance)newInstance).replaceWithNewId(oldInstance.getId());
+
+ repository.updatePartial(newInstance);
+
+ onEntitiesUpdated(new ArrayList<ITypedReferenceableInstance>() {{
+ add(newInstance);
+ }});
+
+ return newInstance.getId()._getId();
}
private void validateTypeExists(String entityType) throws AtlasException {
@@ -485,8 +626,7 @@ public class DefaultMetadataService implements MetadataService {
}
}
- private void onEntitiesAddedToRepo(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
-
+ private void onEntitiesAdded(Collection<ITypedReferenceableInstance> entities) throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
listener.onEntitiesAdded(entities);
}
@@ -509,10 +649,10 @@ public class DefaultMetadataService implements MetadataService {
}
}
- private void onEntityUpdated(ITypedReferenceableInstance entity)
+ private void onEntitiesUpdated(Collection<ITypedReferenceableInstance> entities)
throws AtlasException {
for (EntityChangeListener listener : entityChangeListeners) {
- listener.onEntityUpdated(entity);
+ listener.onEntitiesUpdated(entities);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/java/org/apache/atlas/services/MetadataService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/services/MetadataService.java b/repository/src/main/java/org/apache/atlas/services/MetadataService.java
deleted file mode 100755
index f027b79..0000000
--- a/repository/src/main/java/org/apache/atlas/services/MetadataService.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.services;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.listener.EntityChangeListener;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.codehaus.jettison.json.JSONObject;
-
-import java.util.List;
-
-/**
- * Metadata service.
- */
-public interface MetadataService {
-
- /**
- * Creates a new type based on the type system to enable adding
- * entities (instances for types).
- *
- * @param typeDefinition definition as json
- * @return a unique id for this type
- */
- JSONObject createType(String typeDefinition) throws AtlasException;
-
- /**
- * Updates the given types in the type definition
- * @param typeDefinition
- * @return
- * @throws AtlasException
- */
- JSONObject updateType(String typeDefinition) throws AtlasException;
-
- /**
- * Return the definition for the given type.
- *
- * @param typeName name for this type, must be unique
- * @return type definition as JSON
- */
- String getTypeDefinition(String typeName) throws AtlasException;
-
- /**
- * Return the list of types in the type system.
- *
- * @return list of type names in the type system
- */
- List<String> getTypeNamesList() throws AtlasException;
-
- /**
- * Return the list of trait type names in the type system.
- *
- * @return list of trait type names in the type system
- */
- List<String> getTypeNamesByCategory(DataTypes.TypeCategory typeCategory) throws AtlasException;
-
- /**
- * Creates an entity, instance of the type.
- *
- * @param entityDefinition definition
- * @return guid
- */
- String createEntities(String entityDefinition) throws AtlasException;
-
- /**
- * Return the definition for the given guid.
- *
- * @param guid guid
- * @return entity definition as JSON
- */
- String getEntityDefinition(String guid) throws AtlasException;
-
- /**
- * Return the definition given type and attribute. The attribute has to be unique attribute for the type
- * @param entityType - type name
- * @param attribute - attribute name
- * @param value - attribute value
- * @return
- * @throws AtlasException
- */
- String getEntityDefinition(String entityType, String attribute, String value) throws AtlasException;
-
- /**
- * Return the list of entity names for the given type in the repository.
- *
- * @param entityType type
- * @return list of entity names for the given type in the repository
- */
- List<String> getEntityList(String entityType) throws AtlasException;
-
- /**
- * Adds the property to the given entity id(guid).
- *
- * @param guid entity id
- * @param property property name
- * @param value property value
- */
- void updateEntity(String guid, String property, String value) throws AtlasException;
-
- // Trait management functions
-
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws AtlasException
- */
- List<String> getTraitNames(String guid) throws AtlasException;
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstanceDefinition trait instance that needs to be added to entity
- * @throws AtlasException
- */
- void addTrait(String guid, String traitInstanceDefinition) throws AtlasException;
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws AtlasException
- */
- void deleteTrait(String guid, String traitNameToBeDeleted) throws AtlasException;
-
- /**
- * Register a listener for entity change.
- *
- * @param listener the listener to register
- */
- void registerListener(EntityChangeListener listener);
-
- /**
- * Unregister an entity change listener.
- *
- * @param listener the listener to unregister
- */
- void unregisterListener(EntityChangeListener listener);
-}
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
----------------------------------------------------------------------
diff --git a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
index d857a66..34d101a 100755
--- a/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
+++ b/repository/src/main/scala/org/apache/atlas/query/GraphPersistenceStrategies.scala
@@ -26,7 +26,7 @@ import com.tinkerpop.blueprints.{Vertex, Direction}
import org.apache.atlas.AtlasException
import org.apache.atlas.query.Expressions.{ComparisonExpression, ExpressionException}
import org.apache.atlas.query.TypeUtils.FieldInfo
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository
+import org.apache.atlas.repository.graph.{GraphHelper, GraphBackedMetadataRepository}
import org.apache.atlas.typesystem.persistence.Id
import org.apache.atlas.typesystem.types.DataTypes._
import org.apache.atlas.typesystem.types._
@@ -199,7 +199,7 @@ object GraphPersistenceStrategy1 extends GraphPersistenceStrategies {
def traitLabel(cls: IDataType[_], traitName: String) = s"${cls.getName}.$traitName"
- def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphBackedMetadataRepository.getQualifiedName(dataType, aInfo.name)
+ def fieldNameInVertex(dataType: IDataType[_], aInfo: AttributeInfo) = GraphHelper.getQualifiedFieldName(dataType, aInfo.name)
def getIdFromVertex(dataTypeNm: String, v: TitanVertex): Id =
new Id(v.getId.toString, 0, dataTypeNm)
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
index f9378e4..1075d85 100644
--- a/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
+++ b/repository/src/test/java/org/apache/atlas/BaseHiveRepositoryTest.java
@@ -21,10 +21,10 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.thinkaurelius.titan.core.TitanGraph;
import com.thinkaurelius.titan.core.util.TitanCleanup;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphBackedSearchIndexer;
import org.apache.atlas.repository.graph.GraphProvider;
-import org.apache.atlas.services.DefaultMetadataService;
+import org.apache.atlas.services.MetadataService;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.TypesDef;
@@ -55,10 +55,10 @@ import java.util.List;
public class BaseHiveRepositoryTest {
@Inject
- protected DefaultMetadataService metadataService;
+ protected MetadataService metadataService;
@Inject
- protected GraphBackedMetadataRepository repository;
+ protected MetadataRepository repository;
@Inject
protected GraphProvider<TitanGraph> graphProvider;
@@ -67,7 +67,7 @@ public class BaseHiveRepositoryTest {
setUpTypes();
new GraphBackedSearchIndexer(graphProvider);
setupInstances();
- // TestUtils.dumpGraph(graphProvider.get());
+ TestUtils.dumpGraph(graphProvider.get());
}
protected void tearDown() throws Exception {
@@ -190,17 +190,20 @@ public class BaseHiveRepositoryTest {
Id salesDB = database("Sales", "Sales Database", "John ETL", "hdfs://host:8000/apps/warehouse/sales");
Referenceable sd =
- storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(column("time_id", "int", "time id")));
+ storageDescriptor("hdfs://host:8000/apps/warehouse/sales", "TextInputFormat", "TextOutputFormat", true, ImmutableList.of(
+ column("time_id", "int", "time id")));
List<Referenceable> salesFactColumns = ImmutableList
- .of(column("time_id", "int", "time id"), column("product_id", "int", "product id"),
+ .of(column("time_id", "int", "time id"),
+ column("product_id", "int", "product id"),
column("customer_id", "int", "customer id", "PII"),
column("sales", "double", "product id", "Metric"));
Id salesFact = table("sales_fact", "sales fact table", salesDB, sd, "Joe", "Managed", salesFactColumns, "Fact");
List<Referenceable> timeDimColumns = ImmutableList
- .of(column("time_id", "int", "time id"), column("dayOfYear", "int", "day Of Year"),
+ .of(column("time_id", "int", "time id"),
+ column("dayOfYear", "int", "day Of Year"),
column("weekDay", "int", "week Day"));
Id timeDim = table("time_dim", "time dimension table", salesDB, sd, "John Doe", "External", timeDimColumns,
@@ -217,7 +220,8 @@ public class BaseHiveRepositoryTest {
ImmutableList.of(salesFactDaily), "create table as select ", "plan", "id", "graph", "ETL");
List<Referenceable> productDimColumns = ImmutableList
- .of(column("product_id", "int", "product id"), column("product_name", "string", "product name"),
+ .of(column("product_id", "int", "product id"),
+ column("product_name", "string", "product name"),
column("brand_name", "int", "brand name"));
Id productDim =
@@ -226,7 +230,8 @@ public class BaseHiveRepositoryTest {
view("product_dim_view", reportingDB, ImmutableList.of(productDim), "Dimension", "JdbcAccess");
- List<Referenceable> customerDimColumns = ImmutableList.of(column("customer_id", "int", "customer id", "PII"),
+ List<Referenceable> customerDimColumns = ImmutableList.of(
+ column("customer_id", "int", "customer id", "PII"),
column("name", "string", "customer name", "PII"),
column("address", "string", "customer address", "PII"));
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/TestUtils.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/TestUtils.java b/repository/src/test/java/org/apache/atlas/TestUtils.java
index 01a8158..12c47d4 100755
--- a/repository/src/test/java/org/apache/atlas/TestUtils.java
+++ b/repository/src/test/java/org/apache/atlas/TestUtils.java
@@ -187,6 +187,8 @@ public final class TestUtils {
public static final String DATABASE_TYPE = "hive_database";
public static final String DATABASE_NAME = "foo";
public static final String TABLE_TYPE = "hive_table";
+ public static final String PARTITION_TYPE = "partition_type";
+ public static final String SERDE_TYPE = "serdeType";
public static final String TABLE_NAME = "bar";
public static final String CLASSIFICATION = "classification";
public static final String PII = "PII";
@@ -208,7 +210,8 @@ public final class TestUtils {
StructTypeDefinition structTypeDefinition = new StructTypeDefinition("serdeType",
new AttributeDefinition[]{createRequiredAttrDef("name", DataTypes.STRING_TYPE),
- createRequiredAttrDef("serde", DataTypes.STRING_TYPE)});
+ createRequiredAttrDef("serde", DataTypes.STRING_TYPE),
+ createOptionalAttrDef("description", DataTypes.STRING_TYPE)});
EnumValue values[] = {new EnumValue("MANAGED", 1), new EnumValue("EXTERNAL", 2),};
@@ -244,21 +247,23 @@ public final class TestUtils {
new AttributeDefinition("parametersMap",
DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(), DataTypes.STRING_TYPE.getName()),
Multiplicity.OPTIONAL, true, null),
- // map of classes - todo - enable this
- // new AttributeDefinition("columnsMap",
- // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- // "column_type"),
- // Multiplicity.COLLECTION, true, null),
- // map of structs todo - enable this
- // new AttributeDefinition("partitionsMap",
- // DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
- // "partition_type"),
- // Multiplicity.COLLECTION, true, null),
+ //map of classes -
+ new AttributeDefinition("columnsMap",
+ DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+ "column_type"),
+ Multiplicity.COLLECTION, true, null),
+ //map of structs
+ new AttributeDefinition("partitionsMap",
+ DataTypes.mapTypeName(DataTypes.STRING_TYPE.getName(),
+ "partition_type"),
+ Multiplicity.COLLECTION, true, null),
// struct reference
new AttributeDefinition("serde1", "serdeType", Multiplicity.OPTIONAL, false, null),
new AttributeDefinition("serde2", "serdeType", Multiplicity.OPTIONAL, false, null),
// class reference
- new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, true, null));
+ new AttributeDefinition("database", DATABASE_TYPE, Multiplicity.REQUIRED, false, null),
+ //class reference as composite
+ new AttributeDefinition("databaseComposite", DATABASE_TYPE, Multiplicity.OPTIONAL, true, null));
HierarchicalTypeDefinition<TraitType> piiTypeDefinition =
createTraitTypeDef(PII, ImmutableList.<String>of());
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
index 36b207a..ea8718d 100755
--- a/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/GraphBackedDiscoveryServiceTest.java
@@ -24,7 +24,7 @@ import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
import org.apache.atlas.TestUtils;
import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.repository.graph.GraphBackedMetadataRepository;
+import org.apache.atlas.repository.MetadataRepository;
import org.apache.atlas.repository.graph.GraphProvider;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
@@ -55,7 +55,7 @@ public class GraphBackedDiscoveryServiceTest extends BaseHiveRepositoryTest {
private GraphProvider<TitanGraph> graphProvider;
@Inject
- private GraphBackedMetadataRepository repositoryService;
+ private MetadataRepository repositoryService;
@Inject
private GraphBackedDiscoveryService discoveryService;
http://git-wip-us.apache.org/repos/asf/incubator-atlas/blob/51656991/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
----------------------------------------------------------------------
diff --git a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
index 0e6913d..6d5a15a 100644
--- a/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/discovery/HiveLineageServiceTest.java
@@ -20,8 +20,7 @@ package org.apache.atlas.discovery;
import org.apache.atlas.BaseHiveRepositoryTest;
import org.apache.atlas.RepositoryMetadataModule;
-import org.apache.atlas.discovery.graph.GraphBackedDiscoveryService;
-import org.apache.atlas.repository.EntityNotFoundException;
+import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.codehaus.jettison.json.JSONArray;
import org.codehaus.jettison.json.JSONObject;
import org.testng.Assert;
@@ -32,7 +31,6 @@ import org.testng.annotations.Guice;
import org.testng.annotations.Test;
import javax.inject.Inject;
-import java.util.List;
/**
* Unit tests for Hive LineageService.
@@ -41,7 +39,7 @@ import java.util.List;
public class HiveLineageServiceTest extends BaseHiveRepositoryTest {
@Inject
- private GraphBackedDiscoveryService discoveryService;
+ private DiscoveryService discoveryService;
@Inject
private HiveLineageService hiveLineageService;