You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by ma...@apache.org on 2017/11/12 18:14:40 UTC
[31/42] atlas git commit: ATLAS-2251: Remove TypeSystem and related
implementation, to avoid unncessary duplicate of type details in cache
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java b/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
deleted file mode 100644
index f0fef1f..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/graph/DeleteHandler.java
+++ /dev/null
@@ -1,468 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * <p/>
- * http://www.apache.org/licenses/LICENSE-2.0
- * <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graph;
-
-import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
-import static org.apache.atlas.repository.graph.GraphHelper.string;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.graph.GraphHelper.VertexInfo;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.typesystem.exception.NullRequiredAttributeException;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.FieldMapping;
-import org.apache.atlas.typesystem.types.HierarchicalType;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public abstract class DeleteHandler {
- public static final Logger LOG = LoggerFactory.getLogger(DeleteHandler.class);
-
- protected static final GraphHelper graphHelper = GraphHelper.getInstance();
-
- protected TypeSystem typeSystem;
- private boolean shouldUpdateReverseAttribute;
- private boolean softDelete;
-
- public DeleteHandler(TypeSystem typeSystem, boolean shouldUpdateReverseAttribute, boolean softDelete) {
- this.typeSystem = typeSystem;
- this.shouldUpdateReverseAttribute = shouldUpdateReverseAttribute;
- this.softDelete = softDelete;
- }
-
- /**
- * Deletes the specified entity vertices.
- * Deletes any traits, composite entities, and structs owned by each entity.
- * Also deletes all the references from/to the entity.
- *
- * @param instanceVertices
- * @throws AtlasException
- */
- public void deleteEntities(Collection<AtlasVertex> instanceVertices) throws AtlasException {
- RequestContext requestContext = RequestContext.get();
-
- Set<AtlasVertex> deletionCandidateVertices = new HashSet<>();
-
- for (AtlasVertex instanceVertex : instanceVertices) {
- String guid = GraphHelper.getGuid(instanceVertex);
- Id.EntityState state = GraphHelper.getState(instanceVertex);
- if (requestContext.getDeletedEntityIds().contains(guid) || state == Id.EntityState.DELETED) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skipping deletion of {} as it is already deleted", guid);
- }
-
- continue;
- }
-
- // Get GUIDs and vertices for all deletion candidates.
- Set<VertexInfo> compositeVertices = graphHelper.getCompositeVertices(instanceVertex);
-
- // Record all deletion candidate GUIDs in RequestContext
- // and gather deletion candidate vertices.
- for (VertexInfo vertexInfo : compositeVertices) {
- requestContext.recordEntityDelete(vertexInfo.getGuid(), vertexInfo.getTypeName());
- deletionCandidateVertices.add(vertexInfo.getVertex());
- }
- }
-
- // Delete traits and vertices.
- for (AtlasVertex deletionCandidateVertex : deletionCandidateVertices) {
- deleteAllTraits(deletionCandidateVertex);
- deleteTypeVertex(deletionCandidateVertex, false);
- }
- }
-
- protected abstract void deleteEdge(AtlasEdge edge, boolean force) throws AtlasException;
-
- /**
- * Deletes a type vertex - can be entity(class type) or just vertex(struct/trait type)
- * @param instanceVertex
- * @param typeCategory
- * @throws AtlasException
- */
- protected void deleteTypeVertex(AtlasVertex instanceVertex, DataTypes.TypeCategory typeCategory, boolean force) throws AtlasException {
- switch (typeCategory) {
- case STRUCT:
- case TRAIT:
- deleteTypeVertex(instanceVertex, force);
- break;
-
- case CLASS:
- deleteEntities(Collections.singletonList(instanceVertex));
- break;
-
- default:
- throw new IllegalStateException("Type category " + typeCategory + " not handled");
- }
- }
-
- /**
- * Deleting any type vertex. Goes over the complex attributes and removes the references
- * @param instanceVertex
- * @throws AtlasException
- */
- protected void deleteTypeVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting {}", string(instanceVertex));
- }
-
- String typeName = GraphHelper.getTypeName(instanceVertex);
- IDataType type = typeSystem.getDataType(IDataType.class, typeName);
- FieldMapping fieldMapping = getFieldMapping(type);
-
- for (AttributeInfo attributeInfo : fieldMapping.fields.values()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting attribute {} for {}", attributeInfo.name, string(instanceVertex));
- }
-
- String edgeLabel = GraphHelper.getEdgeLabel(type, attributeInfo);
-
- switch (attributeInfo.dataType().getTypeCategory()) {
- case CLASS:
- //If its class attribute, delete the reference
- deleteEdgeReference(instanceVertex, edgeLabel, DataTypes.TypeCategory.CLASS, attributeInfo.isComposite);
- break;
-
- case STRUCT:
- //If its struct attribute, delete the reference
- deleteEdgeReference(instanceVertex, edgeLabel, DataTypes.TypeCategory.STRUCT, false);
- break;
-
- case ARRAY:
- //For array attribute, if the element is struct/class, delete all the references
- IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
- DataTypes.TypeCategory elementTypeCategory = elementType.getTypeCategory();
- if (elementTypeCategory == DataTypes.TypeCategory.STRUCT ||
- elementTypeCategory == DataTypes.TypeCategory.CLASS) {
- Iterator<AtlasEdge> edges = graphHelper.getOutGoingEdgesByLabel(instanceVertex, edgeLabel);
- if (edges != null) {
- while (edges.hasNext()) {
- AtlasEdge edge = edges.next();
- deleteEdgeReference(edge, elementType.getTypeCategory(), attributeInfo.isComposite, false);
- }
- }
- }
- break;
-
- case MAP:
- //For map attribute, if the value type is struct/class, delete all the references
- DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
- DataTypes.TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
- String propertyName = GraphHelper.getQualifiedFieldName(type, attributeInfo.name);
-
- if (valueTypeCategory == DataTypes.TypeCategory.STRUCT ||
- valueTypeCategory == DataTypes.TypeCategory.CLASS) {
- List<String> keys = GraphHelper.getListProperty(instanceVertex, propertyName);
- if (keys != null) {
- for (String key : keys) {
- String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
- deleteEdgeReference(instanceVertex, mapEdgeLabel, valueTypeCategory, attributeInfo.isComposite);
- }
- }
- }
- }
- }
-
- deleteVertex(instanceVertex, force);
- }
-
- /**
- * Force delete is used to remove struct/trait in case of entity updates
- * @param edge
- * @param typeCategory
- * @param isComposite
- * @param forceDeleteStructTrait
- * @return returns true if the edge reference is hard deleted
- * @throws AtlasException
- */
- public boolean deleteEdgeReference(AtlasEdge edge, DataTypes.TypeCategory typeCategory, boolean isComposite,
- boolean forceDeleteStructTrait) throws AtlasException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting {}", string(edge));
- }
-
- boolean forceDelete =
- (typeCategory == DataTypes.TypeCategory.STRUCT || typeCategory == DataTypes.TypeCategory.TRAIT) && forceDeleteStructTrait;
- if (typeCategory == DataTypes.TypeCategory.STRUCT || typeCategory == DataTypes.TypeCategory.TRAIT
- || (typeCategory == DataTypes.TypeCategory.CLASS && isComposite)) {
- //If the vertex is of type struct/trait, delete the edge and then the reference vertex as the vertex is not shared by any other entities.
- //If the vertex is of type class, and its composite attribute, this reference vertex' lifecycle is controlled
- //through this delete, hence delete the edge and the reference vertex.
- AtlasVertex vertexForDelete = edge.getInVertex();
-
- //If deleting the edge and then the in vertex, reverse attribute shouldn't be updated
- deleteEdge(edge, false, forceDelete);
- deleteTypeVertex(vertexForDelete, typeCategory, forceDelete);
- } else {
- //If the vertex is of type class, and its not a composite attributes, the reference AtlasVertex' lifecycle is not controlled
- //through this delete. Hence just remove the reference edge. Leave the reference AtlasVertex as is
-
- //If deleting just the edge, reverse attribute should be updated for any references
- //For example, for the department type system, if the person's manager edge is deleted, subordinates of manager should be updated
- deleteEdge(edge, true, false);
- }
- return !softDelete || forceDelete;
- }
-
- public void deleteEdgeReference(AtlasVertex outVertex, String edgeLabel, DataTypes.TypeCategory typeCategory,
- boolean isComposite) throws AtlasException {
- AtlasEdge edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
- if (edge != null) {
- deleteEdgeReference(edge, typeCategory, isComposite, false);
- }
- }
-
- protected void deleteEdge(AtlasEdge edge, boolean updateReverseAttribute, boolean force) throws AtlasException {
- //update reverse attribute
- if (updateReverseAttribute) {
- AttributeInfo attributeInfo = getAttributeForEdge(edge.getLabel());
- if (attributeInfo.reverseAttributeName != null) {
- deleteEdgeBetweenVertices(edge.getInVertex(), edge.getOutVertex(),
- attributeInfo.reverseAttributeName);
- }
- }
-
- deleteEdge(edge, force);
- }
-
- protected void deleteVertex(AtlasVertex instanceVertex, boolean force) throws AtlasException {
- //Update external references(incoming edges) to this vertex
- if (LOG.isDebugEnabled()) {
- LOG.debug("Setting the external references to {} to null(removing edges)", string(instanceVertex));
- }
-
- for (AtlasEdge edge : (Iterable<AtlasEdge>) instanceVertex.getEdges(AtlasEdgeDirection.IN)) {
- Id.EntityState edgeState = GraphHelper.getState(edge);
- if (edgeState == Id.EntityState.ACTIVE) {
- //Delete only the active edge references
- AttributeInfo attribute = getAttributeForEdge(edge.getLabel());
- //TODO use delete edge instead??
- deleteEdgeBetweenVertices(edge.getOutVertex(), edge.getInVertex(), attribute.name);
- }
- }
- _deleteVertex(instanceVertex, force);
- }
-
- protected abstract void _deleteVertex(AtlasVertex instanceVertex, boolean force);
-
- /**
- * Deletes the edge between outvertex and inVertex. The edge is for attribute attributeName of outVertex
- * @param outVertex
- * @param inVertex
- * @param attributeName
- * @throws AtlasException
- */
- protected void deleteEdgeBetweenVertices(AtlasVertex outVertex, AtlasVertex inVertex, String attributeName) throws AtlasException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing edge from {} to {} with attribute name {}", string(outVertex), string(inVertex),
- attributeName);
- }
-
- String typeName = GraphHelper.getTypeName(outVertex);
- String outId = GraphHelper.getGuid(outVertex);
- Id.EntityState state = GraphHelper.getState(outVertex);
- if ((outId != null && RequestContext.get().isDeletedEntity(outId)) || state == Id.EntityState.DELETED) {
- //If the reference vertex is marked for deletion, skip updating the reference
- return;
- }
-
- IDataType type = typeSystem.getDataType(IDataType.class, typeName);
- AttributeInfo attributeInfo = getFieldMapping(type).fields.get(attributeName);
- String propertyName = GraphHelper.getQualifiedFieldName(type, attributeName);
- String edgeLabel = EDGE_LABEL_PREFIX + propertyName;
- AtlasEdge edge = null;
-
- switch (attributeInfo.dataType().getTypeCategory()) {
- case CLASS:
- //If its class attribute, its the only edge between two vertices
- if (attributeInfo.multiplicity.nullAllowed()) {
- edge = graphHelper.getEdgeForLabel(outVertex, edgeLabel);
- if (shouldUpdateReverseAttribute) {
- GraphHelper.setProperty(outVertex, propertyName, null);
- }
- } else {
- // Cannot unset a required attribute.
- throw new NullRequiredAttributeException("Cannot unset required attribute " + GraphHelper.getQualifiedFieldName(type, attributeName) +
- " on " + GraphHelper.getVertexDetails(outVertex) + " edge = " + edgeLabel);
- }
- break;
-
- case ARRAY:
- //If its array attribute, find the right edge between the two vertices and update array property
- List<String> elements = GraphHelper.getListProperty(outVertex, propertyName);
- if (elements != null) {
- elements = new ArrayList<>(elements); //Make a copy, else list.remove reflects on titan.getProperty()
- for (String elementEdgeId : elements) {
- AtlasEdge elementEdge = graphHelper.getEdgeByEdgeId(outVertex, edgeLabel, elementEdgeId);
- if (elementEdge == null) {
- continue;
- }
-
- AtlasVertex elementVertex = elementEdge.getInVertex();
- if (elementVertex.equals(inVertex)) {
- edge = elementEdge;
-
- //TODO element.size includes deleted items as well. should exclude
- if (!attributeInfo.multiplicity.nullAllowed()
- && elements.size() <= attributeInfo.multiplicity.lower) {
- // Deleting this edge would violate the attribute's lower bound.
- throw new NullRequiredAttributeException(
- "Cannot remove array element from required attribute " +
- GraphHelper.getQualifiedFieldName(type, attributeName) + " on "
- + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(elementEdge));
- }
-
- if (shouldUpdateReverseAttribute) {
- //if composite attribute, remove the reference as well. else, just remove the edge
- //for example, when table is deleted, process still references the table
- //but when column is deleted, table will not reference the deleted column
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing edge {} from the array attribute {}", string(elementEdge),
- attributeName);
- }
-
- // Remove all occurrences of the edge ID from the list.
- // This prevents dangling edge IDs (i.e. edge IDs for deleted edges)
- // from the remaining in the list if there are duplicates.
- elements.removeAll(Collections.singletonList(elementEdge.getId().toString()));
- GraphHelper.setProperty(outVertex, propertyName, elements);
- break;
-
- }
- }
- }
- }
- break;
-
- case MAP:
- //If its map attribute, find the right edge between two vertices and update map property
- List<String> keys = GraphHelper.getListProperty(outVertex, propertyName);
- if (keys != null) {
- keys = new ArrayList<>(keys); //Make a copy, else list.remove reflects on titan.getProperty()
- for (String key : keys) {
- String keyPropertyName = GraphHelper.getQualifiedNameForMapKey(propertyName, key);
- String mapEdgeId = GraphHelper.getSingleValuedProperty(outVertex, keyPropertyName, String.class);
- AtlasEdge mapEdge = graphHelper.getEdgeByEdgeId(outVertex, keyPropertyName, mapEdgeId);
- if(mapEdge != null) {
- AtlasVertex mapVertex = mapEdge.getInVertex();
- if (mapVertex.getId().toString().equals(inVertex.getId().toString())) {
- //TODO keys.size includes deleted items as well. should exclude
- if (attributeInfo.multiplicity.nullAllowed() || keys.size() > attributeInfo.multiplicity.lower) {
- edge = mapEdge;
- } else {
- // Deleting this entry would violate the attribute's lower bound.
- throw new NullRequiredAttributeException(
- "Cannot remove map entry " + keyPropertyName + " from required attribute " +
- GraphHelper.getQualifiedFieldName(type, attributeName) + " on " + GraphHelper.getVertexDetails(outVertex) + " " + GraphHelper.getEdgeDetails(mapEdge));
- }
-
- if (shouldUpdateReverseAttribute) {
- //remove this key
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removing edge {}, key {} from the map attribute {}", string(mapEdge), key,
- attributeName);
- }
-
- keys.remove(key);
- GraphHelper.setProperty(outVertex, propertyName, keys);
- GraphHelper.setProperty(outVertex, keyPropertyName, null);
- }
- break;
- }
- }
- }
- }
- break;
-
- case STRUCT:
- case TRAIT:
- break;
-
- default:
- throw new IllegalStateException("There can't be an edge from " + GraphHelper.getVertexDetails(outVertex) + " to "
- + GraphHelper.getVertexDetails(inVertex) + " with attribute name " + attributeName + " which is not class/array/map attribute");
- }
-
- if (edge != null) {
- deleteEdge(edge, false);
- RequestContext requestContext = RequestContext.get();
- GraphHelper.setProperty(outVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
- requestContext.getRequestTime());
- GraphHelper.setProperty(outVertex, Constants.MODIFIED_BY_KEY, requestContext.getUser());
- requestContext.recordEntityUpdate(outId);
- }
- }
-
- protected AttributeInfo getAttributeForEdge(String edgLabel) throws AtlasException {
- AtlasEdgeLabel atlasEdgeLabel = new AtlasEdgeLabel(edgLabel);
- IDataType referenceType = typeSystem.getDataType(IDataType.class, atlasEdgeLabel.getTypeName());
- return getFieldMapping(referenceType).fields.get(atlasEdgeLabel.getAttributeName());
- }
-
- protected FieldMapping getFieldMapping(IDataType type) {
- switch (type.getTypeCategory()) {
- case CLASS:
- case TRAIT:
- return ((HierarchicalType)type).fieldMapping();
-
- case STRUCT:
- return ((StructType)type).fieldMapping();
-
- default:
- throw new IllegalStateException("Type " + type + " doesn't have any fields!");
- }
- }
-
- /**
- * Delete all traits from the specified vertex.
- * @param instanceVertex
- * @throws AtlasException
- */
- private void deleteAllTraits(AtlasVertex instanceVertex) throws AtlasException {
- List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Deleting traits {} for {}", traitNames, string(instanceVertex));
- }
-
- String typeName = GraphHelper.getTypeName(instanceVertex);
-
- for (String traitNameToBeDeleted : traitNames) {
- String relationshipLabel = GraphHelper.getTraitLabel(typeName, traitNameToBeDeleted);
- deleteEdgeReference(instanceVertex, relationshipLabel, DataTypes.TypeCategory.TRAIT, false);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java b/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
deleted file mode 100644
index 892b36d..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/graph/EntityProcessor.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.graph;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.typesystem.IReferenceableInstance;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.ObjectGraphWalker;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.LinkedHashMap;
-import java.util.Map;
-
-@Deprecated
-public final class EntityProcessor implements ObjectGraphWalker.NodeProcessor {
-
- private final Map<Id, IReferenceableInstance> idToInstanceMap;
-
- public EntityProcessor() {
- idToInstanceMap = new LinkedHashMap<>();
- }
-
- public Collection<IReferenceableInstance> getInstances() {
- ArrayList<IReferenceableInstance> instances = new ArrayList<>(idToInstanceMap.values());
- Collections.reverse(instances);
- return instances;
- }
-
- @Override
- public void processNode(ObjectGraphWalker.Node nd) throws AtlasException {
- IReferenceableInstance ref = null;
- Id id = null;
-
- if (nd.attributeName == null) {
- ref = (IReferenceableInstance) nd.instance;
- id = ref.getId();
- } else if (nd.aInfo.dataType().getTypeCategory() == DataTypes.TypeCategory.CLASS) {
- if (nd.value != null && (nd.value instanceof Id)) {
- id = (Id) nd.value;
- }
- }
-
- if (id != null) {
- if (id.isUnassigned()) {
- if (ref != null) {
- if (idToInstanceMap.containsKey(id)) { // Oops
- throw new RepositoryException(
- String.format("Unexpected internal error: Id %s processed again", id));
- }
-
- idToInstanceMap.put(id, ref);
- }
- }
- }
- }
-
- public void addInstanceIfNotExists(ITypedReferenceableInstance ref) {
- if(!idToInstanceMap.containsKey(ref.getId())) {
- idToInstanceMap.put(ref.getId(), ref);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
deleted file mode 100644
index 2e8ae0c..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapper.java
+++ /dev/null
@@ -1,174 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.atlas.repository.graph;
-
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.typesystem.ITypedInstance;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.EnumValue;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.commons.lang.StringUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.util.List;
-import java.util.Map;
-
-@Deprecated
-public class FullTextMapper {
-
- private static final Logger LOG = LoggerFactory.getLogger(FullTextMapper.class);
-
- private final GraphToTypedInstanceMapper graphToTypedInstanceMapper;
- private final TypedInstanceToGraphMapper typedInstanceToGraphMapper;
-
- private static final GraphHelper graphHelper = GraphHelper.getInstance();
-
- private static final String FULL_TEXT_DELIMITER = " ";
-
- public FullTextMapper(TypedInstanceToGraphMapper typedInstanceToGraphMapper,
- GraphToTypedInstanceMapper graphToTypedInstanceMapper) {
- this.graphToTypedInstanceMapper = graphToTypedInstanceMapper;
- this.typedInstanceToGraphMapper = typedInstanceToGraphMapper;
- }
-
- public String mapRecursive(AtlasVertex instanceVertex, boolean followReferences) throws AtlasException {
- String guid = GraphHelper.getGuid(instanceVertex);
- ITypedReferenceableInstance typedReference;
- RequestContext context = RequestContext.get();
- typedReference = context.getInstanceV1(guid);
- if (typedReference != null) {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cache hit: guid = {}, entityId = {}", guid, typedReference.getId()._getId());
- }
- } else {
- typedReference =
- graphToTypedInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- context.cache(typedReference);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Cache miss: guid = {}, entityId = {}", guid, typedReference.getId().getId());
- }
- }
- String fullText = forInstance(typedReference, followReferences);
- StringBuilder fullTextBuilder =
- new StringBuilder(typedReference.getTypeName()).append(FULL_TEXT_DELIMITER).append(fullText);
-
- List<String> traits = typedReference.getTraits();
- for (String traitName : traits) {
- String traitText = forInstance((ITypedInstance) typedReference.getTrait(traitName), false);
- fullTextBuilder.append(FULL_TEXT_DELIMITER).append(traitName).append(FULL_TEXT_DELIMITER)
- .append(traitText);
- }
- return fullTextBuilder.toString();
- }
-
- private String forAttribute(IDataType type, Object value, boolean followReferences)
- throws AtlasException {
- if (value == null) {
- return null;
- }
- switch (type.getTypeCategory()) {
- case PRIMITIVE:
- return String.valueOf(value);
- case ENUM:
-
- return ((EnumValue) value).value;
-
- case ARRAY:
- StringBuilder fullText = new StringBuilder();
- IDataType elemType = ((DataTypes.ArrayType) type).getElemType();
- List list = (List) value;
-
- for (Object element : list) {
- String elemFullText = forAttribute(elemType, element, false);
- if (StringUtils.isNotEmpty(elemFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(elemFullText);
- }
- }
- return fullText.toString();
-
- case MAP:
- fullText = new StringBuilder();
- IDataType keyType = ((DataTypes.MapType) type).getKeyType();
- IDataType valueType = ((DataTypes.MapType) type).getValueType();
- Map map = (Map) value;
-
- for (Object entryObj : map.entrySet()) {
- Map.Entry entry = (Map.Entry) entryObj;
- String keyFullText = forAttribute(keyType, entry.getKey(), false);
- if (StringUtils.isNotEmpty(keyFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(keyFullText);
- }
- String valueFullText = forAttribute(valueType, entry.getValue(), false);
- if (StringUtils.isNotEmpty(valueFullText)) {
- fullText = fullText.append(FULL_TEXT_DELIMITER).append(valueFullText);
- }
- }
- return fullText.toString();
-
- case CLASS:
- if (followReferences) {
- Id refId = ((ITypedReferenceableInstance) value).getId();
- String refGuid = refId._getId();
- AtlasVertex refVertex = typedInstanceToGraphMapper.lookupVertex(refId);
- if(refVertex == null) {
- refVertex = graphHelper.getVertexForGUID(refGuid);
- }
- return mapRecursive(refVertex, false);
- }
- break;
-
- case STRUCT:
- if (followReferences) {
- return forInstance((ITypedInstance) value, true);
- }
- break;
-
- default:
- throw new IllegalStateException("Unhandled type category " + type.getTypeCategory());
-
- }
- return null;
- }
-
- private String forInstance(ITypedInstance typedInstance, boolean followReferences)
- throws AtlasException {
- StringBuilder fullText = new StringBuilder();
- for (AttributeInfo attributeInfo : typedInstance.fieldMapping().fields.values()) {
- Object attrValue = typedInstance.get(attributeInfo.name);
- if (attrValue == null) {
- continue;
- }
-
- String attrFullText = forAttribute(attributeInfo.dataType(), attrValue, followReferences);
- if (StringUtils.isNotEmpty(attrFullText)) {
- fullText =
- fullText.append(FULL_TEXT_DELIMITER).append(attributeInfo.name).append(FULL_TEXT_DELIMITER)
- .append(attrFullText);
- }
- }
- return fullText.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
index 76acf8c..c42aa15 100644
--- a/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/FullTextMapperV2.java
@@ -17,7 +17,7 @@
*/
package org.apache.atlas.repository.graph;
-import org.apache.atlas.RequestContext;
+import org.apache.atlas.RequestContextV1;
import org.apache.atlas.exception.AtlasBaseException;
import org.apache.atlas.model.instance.AtlasClassification;
import org.apache.atlas.model.instance.AtlasEntity;
@@ -203,7 +203,7 @@ public class FullTextMapperV2 {
}
private AtlasEntityWithExtInfo getAndCacheEntity(String guid) throws AtlasBaseException {
- RequestContext context = RequestContext.get();
+ RequestContextV1 context = RequestContextV1.get();
AtlasEntityWithExtInfo entityWithExtInfo = context.getInstanceV2(guid);
if (entityWithExtInfo == null) {
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
deleted file mode 100755
index 74886b5..0000000
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedMetadataRepository.java
+++ /dev/null
@@ -1,505 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.atlas.repository.graph;
-
-import com.google.common.base.Preconditions;
-import org.apache.atlas.AtlasException;
-import org.apache.atlas.CreateUpdateEntitiesResult;
-import org.apache.atlas.GraphTransactionInterceptor;
-import org.apache.atlas.RequestContext;
-import org.apache.atlas.annotation.GraphTransaction;
-import org.apache.atlas.model.instance.GuidMapping;
-import org.apache.atlas.model.legacy.EntityResult;
-import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.MetadataRepository;
-import org.apache.atlas.repository.RepositoryException;
-import org.apache.atlas.repository.graphdb.AtlasEdge;
-import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
-import org.apache.atlas.repository.graphdb.AtlasVertex;
-import org.apache.atlas.typesystem.ITypedReferenceableInstance;
-import org.apache.atlas.typesystem.ITypedStruct;
-import org.apache.atlas.typesystem.exception.EntityExistsException;
-import org.apache.atlas.typesystem.exception.EntityNotFoundException;
-import org.apache.atlas.typesystem.exception.TraitNotFoundException;
-import org.apache.atlas.typesystem.persistence.Id;
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.TypeSystem;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.stereotype.Component;
-
-import javax.inject.Inject;
-import javax.inject.Singleton;
-import java.util.*;
-
-/**
- * An implementation backed by a Graph database provided
- * as a Graph Service.
- */
-@Singleton
-@Component
-@Deprecated
-public class GraphBackedMetadataRepository implements MetadataRepository {
-
- private static final Logger LOG = LoggerFactory.getLogger(GraphBackedMetadataRepository.class);
-
- private static TypeSystem typeSystem = TypeSystem.getInstance();
-
- private static final GraphHelper graphHelper = GraphHelper.getInstance();
-
- private DeleteHandler deleteHandler;
-
- private final AtlasGraph atlasGraph;
- private final GraphToTypedInstanceMapper graphToInstanceMapper;
-
- @Inject
- public GraphBackedMetadataRepository(DeleteHandler deleteHandler, AtlasGraph atlasGraph) {
- this.atlasGraph = atlasGraph;
- this.graphToInstanceMapper = new GraphToTypedInstanceMapper(atlasGraph);
- this.deleteHandler = deleteHandler;
- }
-
- public GraphToTypedInstanceMapper getGraphToInstanceMapper() {
- return graphToInstanceMapper;
- }
-
- @Override
- public String getTypeAttributeName() {
- return Constants.ENTITY_TYPE_PROPERTY_KEY;
- }
-
- @Override
- public String getStateAttributeName() {
- return Constants.STATE_PROPERTY_KEY;
- }
-
- /**
- * Returns the property key used to store super type names.
- *
- * @return property key used to store super type names.
- */
- @Override
- public String getSuperTypeAttributeName() {
- return Constants.SUPER_TYPES_PROPERTY_KEY;
- }
-
- public String getIdAttributeName() {
- return Constants.GUID_PROPERTY_KEY;
- }
-
- @Override
- public String getVersionAttributeName() {
- return Constants.VERSION_PROPERTY_KEY;
- }
-
- @Override
- public String getTraitLabel(IDataType<?> dataType, String traitName) {
- return GraphHelper.getTraitLabel(dataType.getName(), traitName);
- }
-
- @Override
- public String getFieldNameInVertex(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
- if (aInfo.name.startsWith(Constants.INTERNAL_PROPERTY_KEY_PREFIX)) {
- return aInfo.name;
- }
- return GraphHelper.encodePropertyKey(GraphHelper.getQualifiedFieldName(dataType, aInfo.name));
- }
-
- public String getFieldNameInVertex(IDataType<?> dataType, String attrName) throws AtlasException {
- return GraphHelper.getQualifiedFieldName(dataType, attrName);
- }
-
- @Override
- public String getEdgeLabel(IDataType<?> dataType, AttributeInfo aInfo) throws AtlasException {
- return GraphHelper.getEdgeLabel(dataType, aInfo);
- }
-
- @Override
- @GraphTransaction
- public CreateUpdateEntitiesResult createEntities(ITypedReferenceableInstance... entities) throws RepositoryException,
- EntityExistsException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("adding entities={}", entities);
- }
-
- try {
- TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
- instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.CREATE, entities);
- List<String> createdGuids = RequestContext.get().getCreatedEntityIds();
- CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
- EntityResult entityResult = new EntityResult(createdGuids, null, null);
- GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
- result.setEntityResult(entityResult);
- result.setGuidMapping(mapping);
- return result;
- } catch (EntityExistsException e) {
- throw e;
- } catch (AtlasException e) {
- throw new RepositoryException(e);
- }
- }
-
- @Override
- @GraphTransaction
- public ITypedReferenceableInstance getEntityDefinition(String guid) throws RepositoryException, EntityNotFoundException {
- return getEntityDefinitions(guid).get(0);
- }
-
- @Override
- @GraphTransaction
- public List<ITypedReferenceableInstance> getEntityDefinitions(String... guids) throws RepositoryException, EntityNotFoundException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving entities with guids={}", Arrays.toString(guids));
- }
-
- RequestContext context = RequestContext.get();
- ITypedReferenceableInstance[] result = new ITypedReferenceableInstance[guids.length];
-
- // Map of the guids of instances not in the cache to their index(es) in the result.
- // This is used to put the loaded instances into the location(s) corresponding
- // to their guid in the result. Note that a set is needed since guids can
- // appear more than once in the list.
- Map<String, Set<Integer>> uncachedGuids = new HashMap<>();
-
- for (int i = 0; i < guids.length; i++) {
- String guid = guids[i];
-
- // First, check the cache.
- ITypedReferenceableInstance cached = context.getInstanceV1(guid);
- if (cached != null) {
- result[i] = cached;
- } else {
- Set<Integer> indices = uncachedGuids.get(guid);
- if (indices == null) {
- indices = new HashSet<>(1);
- uncachedGuids.put(guid, indices);
- }
- indices.add(i);
- }
- }
-
- List<String> guidsToFetch = new ArrayList<>(uncachedGuids.keySet());
- Map<String, AtlasVertex> instanceVertices = graphHelper.getVerticesForGUIDs(guidsToFetch);
-
- // search for missing entities
- if (instanceVertices.size() != guidsToFetch.size()) {
- Set<String> missingGuids = new HashSet<String>(guidsToFetch);
- missingGuids.removeAll(instanceVertices.keySet());
- if (!missingGuids.isEmpty()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Failed to find guids={}", missingGuids);
- }
- throw new EntityNotFoundException(
- "Could not find entities in the repository with guids: " + missingGuids.toString());
- }
- }
-
- for (String guid : guidsToFetch) {
- try {
- ITypedReferenceableInstance entity = graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertices.get(guid));
- for(int index : uncachedGuids.get(guid)) {
- result[index] = entity;
- }
- } catch (AtlasException e) {
- throw new RepositoryException(e);
- }
- }
- return Arrays.asList(result);
- }
-
- @Override
- @GraphTransaction
- public ITypedReferenceableInstance getEntityDefinition(String entityType, String attribute, Object value)
- throws AtlasException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving entity with type={} and {}={}", entityType, attribute, value);
- }
-
- IDataType type = typeSystem.getDataType(IDataType.class, entityType);
- String propertyKey = getFieldNameInVertex(type, attribute);
- AtlasVertex instanceVertex = graphHelper.findVertex(propertyKey, value,
- Constants.ENTITY_TYPE_PROPERTY_KEY, entityType,
- Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
-
- String guid = GraphHelper.getGuid(instanceVertex);
- ITypedReferenceableInstance cached = RequestContext.get().getInstanceV1(guid);
- if(cached != null) {
- return cached;
- }
- return graphToInstanceMapper.mapGraphToTypedInstance(guid, instanceVertex);
- }
-
- @Override
- @GraphTransaction
- public List<String> getEntityList(String entityType) throws RepositoryException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving entity list for type={}", entityType);
- }
-
- AtlasGraphQuery query = getGraph().query().has(Constants.ENTITY_TYPE_PROPERTY_KEY, entityType);
- Iterator<AtlasVertex> results = query.vertices().iterator();
- if (!results.hasNext()) {
- return Collections.emptyList();
- }
-
- ArrayList<String> entityList = new ArrayList<>();
- while (results.hasNext()) {
- AtlasVertex vertex = results.next();
- entityList.add(GraphHelper.getGuid(vertex));
- }
-
- return entityList;
- }
-
- /**
- * Gets the list of trait names for a given entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @return a list of trait names for the given entity guid
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public List<String> getTraitNames(String guid) throws AtlasException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Retrieving trait names for entity={}", guid);
- }
-
- AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
- return GraphHelper.getTraitNames(instanceVertex);
- }
-
- /**
- * Adds a new trait to the list of entities represented by their respective guids
- * @param entityGuids list of globally unique identifier for the entities
- * @param traitInstance trait instance that needs to be added to entities
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public void addTrait(List<String> entityGuids, ITypedStruct traitInstance) throws RepositoryException {
- Preconditions.checkNotNull(entityGuids, "entityGuids list cannot be null");
- Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding a new trait={} for entities={}", traitInstance.getTypeName(), entityGuids);
- }
-
- GraphTransactionInterceptor.lockObjectAndReleasePostCommit(entityGuids);
- for (String entityGuid : entityGuids) {
- addTraitImpl(entityGuid, traitInstance);
- }
- }
-
- /**
- * Adds a new trait to an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitInstance trait instance that needs to be added to entity
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public void addTrait(String guid, ITypedStruct traitInstance) throws RepositoryException {
- Preconditions.checkNotNull(guid, "guid cannot be null");
- Preconditions.checkNotNull(traitInstance, "Trait instance cannot be null");
-
- GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid);
- addTraitImpl(guid, traitInstance);
- }
-
- private void addTraitImpl(String guid, ITypedStruct traitInstance) throws RepositoryException {
- final String traitName = traitInstance.getTypeName();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Adding a new trait={} for entity={}", traitName, guid);
- }
-
- try {
- AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
-
- // add the trait instance as a new vertex
- final String typeName = GraphHelper.getTypeName(instanceVertex);
-
- TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
- instanceToGraphMapper.mapTraitInstanceToVertex(traitInstance,
- typeSystem.getDataType(ClassType.class, typeName), instanceVertex);
-
-
- // update the traits in entity once adding trait instance is successful
- GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
- GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
- RequestContext.get().getRequestTime());
- GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
-
- } catch (RepositoryException e) {
- throw e;
- } catch (Exception e) {
- throw new RepositoryException(e);
- }
- }
-
- /**
- * Deletes a given trait from an existing entity represented by a guid.
- *
- * @param guid globally unique identifier for the entity
- * @param traitNameToBeDeleted name of the trait
- * @throws RepositoryException
- */
- @Override
- @GraphTransaction
- public void deleteTrait(String guid, String traitNameToBeDeleted) throws TraitNotFoundException, EntityNotFoundException, RepositoryException {
- LOG.debug("Deleting trait={} from entity={}", traitNameToBeDeleted, guid);
- GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid);
-
- AtlasVertex instanceVertex = graphHelper.getVertexForGUID(guid);
-
- List<String> traitNames = GraphHelper.getTraitNames(instanceVertex);
- if (!traitNames.contains(traitNameToBeDeleted)) {
- throw new TraitNotFoundException(
- "Could not find trait=" + traitNameToBeDeleted + " in the repository for entity: " + guid);
- }
-
- try {
- final String entityTypeName = GraphHelper.getTypeName(instanceVertex);
- String relationshipLabel = GraphHelper.getTraitLabel(entityTypeName, traitNameToBeDeleted);
- AtlasEdge edge = graphHelper.getEdgeForLabel(instanceVertex, relationshipLabel);
- if(edge != null) {
- deleteHandler.deleteEdgeReference(edge, DataTypes.TypeCategory.TRAIT, false, true);
- }
-
- // update the traits in entity once trait removal is successful
- traitNames.remove(traitNameToBeDeleted);
- updateTraits(instanceVertex, traitNames);
- } catch (Exception e) {
- throw new RepositoryException(e);
- }
- }
-
-
- private void updateTraits(AtlasVertex instanceVertex, List<String> traitNames) {
- // remove the key
- instanceVertex.removeProperty(Constants.TRAIT_NAMES_PROPERTY_KEY);
-
- // add it back again
- for (String traitName : traitNames) {
- GraphHelper.addProperty(instanceVertex, Constants.TRAIT_NAMES_PROPERTY_KEY, traitName);
- }
- GraphHelper.setProperty(instanceVertex, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
- RequestContext.get().getRequestTime());
- GraphHelper.setProperty(instanceVertex, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
- }
-
- @Override
- @GraphTransaction
- public CreateUpdateEntitiesResult updateEntities(ITypedReferenceableInstance... entitiesUpdated) throws RepositoryException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("updating entity {}", entitiesUpdated);
- }
-
- try {
- TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
- instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_FULL,
- entitiesUpdated);
- CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
- RequestContext requestContext = RequestContext.get();
- result.setEntityResult(createEntityResultFromContext(requestContext));
- GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
- result.setGuidMapping(mapping);
- return result;
- } catch (AtlasException e) {
- throw new RepositoryException(e);
- }
- }
-
- @Override
- @GraphTransaction
- public CreateUpdateEntitiesResult updatePartial(ITypedReferenceableInstance entity) throws RepositoryException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("updating entity {}", entity);
- }
-
- try {
- TypedInstanceToGraphMapper instanceToGraphMapper = new TypedInstanceToGraphMapper(graphToInstanceMapper, deleteHandler);
- instanceToGraphMapper.mapTypedInstanceToGraph(TypedInstanceToGraphMapper.Operation.UPDATE_PARTIAL, entity);
- RequestContext requestContext = RequestContext.get();
- CreateUpdateEntitiesResult result = new CreateUpdateEntitiesResult();
- GuidMapping mapping = instanceToGraphMapper.createGuidMapping();
- result.setEntityResult(createEntityResultFromContext(requestContext));
- result.setGuidMapping(mapping);
- return result;
- } catch (AtlasException e) {
- throw new RepositoryException(e);
- }
- }
-
-
-
- @Override
- @GraphTransaction
- public EntityResult deleteEntities(List<String> guids) throws RepositoryException {
-
- if (guids == null || guids.size() == 0) {
- throw new IllegalArgumentException("guids must be non-null and non-empty");
- }
-
- // Retrieve vertices for requested guids.
- Map<String, AtlasVertex> vertices = graphHelper.getVerticesForGUIDs(guids);
- Collection<AtlasVertex> deletionCandidates = vertices.values();
-
- if(LOG.isDebugEnabled()) {
- for(String guid : guids) {
- if(! vertices.containsKey(guid)) {
- // Entity does not exist - treat as non-error, since the caller
- // wanted to delete the entity and it's already gone.
- LOG.debug("Deletion request ignored for non-existent entity with guid " + guid);
- }
- }
- }
-
- if (deletionCandidates.isEmpty()) {
- LOG.info("No deletion candidate entities were found for guids %s", guids);
- return new EntityResult(Collections.<String>emptyList(), Collections.<String>emptyList(), Collections.<String>emptyList());
- }
-
- try {
- deleteHandler.deleteEntities(deletionCandidates);
- }
- catch (AtlasException e) {
- throw new RepositoryException(e);
- }
-
- RequestContext requestContext = RequestContext.get();
- return createEntityResultFromContext(requestContext);
- }
-
- private EntityResult createEntityResultFromContext(RequestContext requestContext) {
- return new EntityResult(
- requestContext.getCreatedEntityIds(),
- requestContext.getUpdatedEntityIds(),
- requestContext.getDeletedEntityIds());
- }
-
- public AtlasGraph getGraph() throws RepositoryException {
- return atlasGraph;
- }
-}
http://git-wip-us.apache.org/repos/asf/atlas/blob/435fe3fb/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
index 6eee24b..9f1206c 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphBackedSearchIndexer.java
@@ -34,7 +34,6 @@ import org.apache.atlas.model.typedef.AtlasEnumDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.model.typedef.AtlasStructDef.AtlasAttributeDef;
import org.apache.atlas.repository.Constants;
-import org.apache.atlas.repository.IndexCreationException;
import org.apache.atlas.repository.IndexException;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasCardinality;
@@ -49,13 +48,6 @@ import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.type.AtlasTypeUtil;
-import org.apache.atlas.typesystem.types.AttributeInfo;
-import org.apache.atlas.typesystem.types.ClassType;
-import org.apache.atlas.typesystem.types.DataTypes;
-import org.apache.atlas.typesystem.types.IDataType;
-import org.apache.atlas.typesystem.types.Multiplicity;
-import org.apache.atlas.typesystem.types.StructType;
-import org.apache.atlas.typesystem.types.TraitType;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.configuration.Configuration;
import org.slf4j.Logger;
@@ -66,11 +58,9 @@ import javax.inject.Inject;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
@@ -80,8 +70,7 @@ import static org.apache.atlas.model.typedef.AtlasBaseTypeDef.*;
* Adds index for properties of a given type when its added before any instances are added.
*/
@Component
-public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler,
- TypeDefChangeListener {
+public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChangeHandler, TypeDefChangeListener {
private static final Logger LOG = LoggerFactory.getLogger(GraphBackedSearchIndexer.class);
@@ -118,6 +107,110 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
/**
+ * Initialize global indices for Titan graph on server activation.
+ *
+ * Since the indices are shared state, we need to do this only from an active instance.
+ */
+ @Override
+ public void instanceIsActive() throws AtlasException {
+ LOG.info("Reacting to active: initializing index");
+ try {
+ initialize();
+ } catch (RepositoryException | IndexException e) {
+ throw new AtlasException("Error in reacting to active on initialization", e);
+ }
+ }
+
+ @Override
+ public void instanceIsPassive() {
+ LOG.info("Reacting to passive state: No action right now.");
+ }
+
+ @Override
+ public int getHandlerOrder() {
+ return HandlerOrder.GRAPH_BACKED_SEARCH_INDEXER.getOrder();
+ }
+
+ @Override
+ public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Processing changed typedefs {}", changedTypeDefs);
+ }
+ AtlasGraphManagement management = null;
+ try {
+ management = provider.get().getManagementSystem();
+
+ // Update index for newly created types
+ if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreateTypeDefs())) {
+ for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreateTypeDefs()) {
+ updateIndexForTypeDef(management, typeDef);
+ }
+ }
+
+ // Update index for updated types
+ if (CollectionUtils.isNotEmpty(changedTypeDefs.getUpdatedTypeDefs())) {
+ for (AtlasBaseTypeDef typeDef : changedTypeDefs.getUpdatedTypeDefs()) {
+ updateIndexForTypeDef(management, typeDef);
+ }
+ }
+
+ // Invalidate the property key for deleted types
+ if (CollectionUtils.isNotEmpty(changedTypeDefs.getDeletedTypeDefs())) {
+ for (AtlasBaseTypeDef typeDef : changedTypeDefs.getDeletedTypeDefs()) {
+ cleanupIndices(management, typeDef);
+ }
+ }
+
+ //Commit indexes
+ commit(management);
+ } catch (RepositoryException | IndexException e) {
+ LOG.error("Failed to update indexes for changed typedefs", e);
+ attemptRollback(changedTypeDefs, management);
+ }
+
+ }
+
+ public Set<String> getVertexIndexKeys() {
+ if (recomputeIndexedKeys) {
+ AtlasGraphManagement management = null;
+
+ try {
+ management = provider.get().getManagementSystem();
+
+ if (management != null) {
+ AtlasGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
+
+ if (vertexIndex != null) {
+ recomputeIndexedKeys = false;
+
+ Set<String> indexKeys = new HashSet<>();
+
+ for (AtlasPropertyKey fieldKey : vertexIndex.getFieldKeys()) {
+ indexKeys.add(fieldKey.getName());
+ }
+
+ vertexIndexKeys = indexKeys;
+ }
+
+ management.commit();
+ }
+ } catch (Exception excp) {
+ LOG.error("getVertexIndexKeys(): failed to get indexedKeys from graph", excp);
+
+ if (management != null) {
+ try {
+ management.rollback();
+ } catch (Exception e) {
+ LOG.error("getVertexIndexKeys(): rollback failed", e);
+ }
+ }
+ }
+ }
+
+ return vertexIndexKeys;
+ }
+
+ /**
* Initializes the indices for the graph - create indices for Global AtlasVertex Keys
*/
private void initialize() throws RepositoryException, IndexException {
@@ -220,81 +313,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
true, true);
}
- /**
- * This is upon adding a new type to Store.
- *
- * @param dataTypes data type
- * @throws AtlasException
- */
- @Override
- public void onAdd(Collection<? extends IDataType> dataTypes) throws AtlasException {
- AtlasGraphManagement management = provider.get().getManagementSystem();
-
- for (IDataType dataType : dataTypes) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Creating indexes for type name={}, definition={}", dataType.getName(), dataType.getClass());
- }
-
- try {
- addIndexForType(management, dataType);
- LOG.info("Index creation for type {} complete", dataType.getName());
- } catch (Throwable throwable) {
- LOG.error("Error creating index for type {}", dataType, throwable);
- //Rollback indexes if any failure
- rollback(management);
- throw new IndexCreationException("Error while creating index for type " + dataType, throwable);
- }
- }
-
- //Commit indexes
- commit(management);
- }
-
- @Override
- public void onChange(Collection<? extends IDataType> dataTypes) throws AtlasException {
- onAdd(dataTypes);
- }
-
- public Set<String> getVertexIndexKeys() {
- if (recomputeIndexedKeys) {
- AtlasGraphManagement management = null;
-
- try {
- management = provider.get().getManagementSystem();
-
- if (management != null) {
- AtlasGraphIndex vertexIndex = management.getGraphIndex(Constants.VERTEX_INDEX);
-
- if (vertexIndex != null) {
- recomputeIndexedKeys = false;
-
- Set<String> indexKeys = new HashSet<>();
-
- for (AtlasPropertyKey fieldKey : vertexIndex.getFieldKeys()) {
- indexKeys.add(fieldKey.getName());
- }
-
- vertexIndexKeys = indexKeys;
- }
-
- management.commit();
- }
- } catch (Exception excp) {
- LOG.error("getVertexIndexKeys(): failed to get indexedKeys from graph", excp);
-
- if (management != null) {
- try {
- management.rollback();
- } catch (Exception e) {
- LOG.error("getVertexIndexKeys(): rollback failed", e);
- }
- }
- }
- }
-
- return vertexIndexKeys;
- }
-
private void addIndexForType(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) {
if (typeDef instanceof AtlasEnumDef) {
// Only handle complex types like Struct, Classification and Entity
@@ -414,82 +432,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
throw new IllegalArgumentException(String.format("Bad cardinality %s", cardinality));
}
- private void addIndexForType(AtlasGraphManagement management, IDataType dataType) {
- switch (dataType.getTypeCategory()) {
- case PRIMITIVE:
- case ENUM:
- case ARRAY:
- case MAP:
- // do nothing since these are only attributes
- // and not types like structs, traits or classes
- break;
-
- case STRUCT:
- StructType structType = (StructType) dataType;
- createIndexForFields(management, structType, structType.fieldMapping().fields);
- break;
-
- case TRAIT:
- TraitType traitType = (TraitType) dataType;
- createIndexForFields(management, traitType, traitType.fieldMapping().fields);
- break;
-
- case CLASS:
- ClassType classType = (ClassType) dataType;
- createIndexForFields(management, classType, classType.fieldMapping().fields);
- break;
-
- default:
- throw new IllegalArgumentException("bad data type" + dataType);
- }
- }
-
- private void createIndexForFields(AtlasGraphManagement management, IDataType dataType, Map<String, AttributeInfo> fields) {
- for (AttributeInfo field : fields.values()) {
- createIndexForAttribute(management, dataType.getName(), field);
- }
- }
-
- private void createIndexForAttribute(AtlasGraphManagement management, String typeName, AttributeInfo field) {
- final String propertyName = GraphHelper.encodePropertyKey(typeName + "." + field.name);
- switch (field.dataType().getTypeCategory()) {
- case PRIMITIVE:
- AtlasCardinality cardinality = getCardinality(field.multiplicity);
- createIndexes(management, propertyName, getPrimitiveClass(field.dataType()), field.isUnique,
- cardinality, false, field.isIndexable);
- break;
-
- case ENUM:
- cardinality = getCardinality(field.multiplicity);
- createIndexes(management, propertyName, String.class, field.isUnique, cardinality, false, field.isIndexable);
- break;
-
- case ARRAY:
- createLabelIfNeeded(management, propertyName, field.dataType().getName());
- break;
- case MAP:
- // todo - how do we overcome this limitation?
- // IGNORE: Can only index single-valued property keys on vertices in Mixed Index
- break;
-
- case STRUCT:
- StructType structType = (StructType) field.dataType();
- createIndexForFields(management, structType, structType.fieldMapping().fields);
- break;
-
- case TRAIT:
- // do nothing since this is NOT contained in other types
- break;
-
- case CLASS:
- createEdgeLabel(management, propertyName);
- break;
-
- default:
- throw new IllegalArgumentException("bad data type" + field.dataType().getName());
- }
- }
-
private void createEdgeLabel(final AtlasGraphManagement management, final String propertyName) {
// Create the edge label upfront to avoid running into concurrent call issue (ATLAS-2092)
// ATLAS-2092 addresses this problem by creating the edge label upfront while type creation
@@ -506,50 +448,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
- private Class getPrimitiveClass(IDataType dataType) {
- if (dataType == DataTypes.STRING_TYPE) {
- return String.class;
- } else if (dataType == DataTypes.SHORT_TYPE) {
- return Short.class;
- } else if (dataType == DataTypes.INT_TYPE) {
- return Integer.class;
- } else if (dataType == DataTypes.BIGINTEGER_TYPE) {
- return BigInteger.class;
- } else if (dataType == DataTypes.BOOLEAN_TYPE) {
- return Boolean.class;
- } else if (dataType == DataTypes.BYTE_TYPE) {
- return Byte.class;
- } else if (dataType == DataTypes.LONG_TYPE) {
- return Long.class;
- } else if (dataType == DataTypes.FLOAT_TYPE) {
- return Float.class;
- } else if (dataType == DataTypes.DOUBLE_TYPE) {
- return Double.class;
- } else if (dataType == DataTypes.BIGDECIMAL_TYPE) {
- return BigDecimal.class;
- } else if (dataType == DataTypes.DATE_TYPE) {
- //Indexing with date converted to long as of now since Titan is yet to add support for Date type with mixed indexes
- return Long.class;
- }
-
-
- throw new IllegalArgumentException("unknown data type " + dataType);
- }
-
-
- private AtlasCardinality getCardinality(Multiplicity multiplicity) {
- if (multiplicity == Multiplicity.OPTIONAL || multiplicity == Multiplicity.REQUIRED) {
- return AtlasCardinality.SINGLE;
- } else if (multiplicity == Multiplicity.COLLECTION) {
- return AtlasCardinality.LIST;
- } else if (multiplicity == Multiplicity.SET) {
- return AtlasCardinality.SET;
- }
-
- // todo - default to LIST as this is the most forgiving
- return AtlasCardinality.LIST;
- }
-
private AtlasPropertyKey createIndexes(AtlasGraphManagement management, String propertyName, Class propertyClass,
boolean isUnique, AtlasCardinality cardinality, boolean createCompositeForAttribute,
boolean createCompositeWithTypeandSuperTypes) {
@@ -677,70 +575,6 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
}
}
- /**
- * Initialize global indices for Titan graph on server activation.
- *
- * Since the indices are shared state, we need to do this only from an active instance.
- */
- @Override
- public void instanceIsActive() throws AtlasException {
- LOG.info("Reacting to active: initializing index");
- try {
- initialize();
- } catch (RepositoryException | IndexException e) {
- throw new AtlasException("Error in reacting to active on initialization", e);
- }
- }
-
- @Override
- public void instanceIsPassive() {
- LOG.info("Reacting to passive state: No action right now.");
- }
-
- @Override
- public int getHandlerOrder() {
- return HandlerOrder.GRAPH_BACKED_SEARCH_INDEXER.getOrder();
- }
-
- @Override
- public void onChange(ChangedTypeDefs changedTypeDefs) throws AtlasBaseException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Processing changed typedefs {}", changedTypeDefs);
- }
- AtlasGraphManagement management = null;
- try {
- management = provider.get().getManagementSystem();
-
- // Update index for newly created types
- if (CollectionUtils.isNotEmpty(changedTypeDefs.getCreateTypeDefs())) {
- for (AtlasBaseTypeDef typeDef : changedTypeDefs.getCreateTypeDefs()) {
- updateIndexForTypeDef(management, typeDef);
- }
- }
-
- // Update index for updated types
- if (CollectionUtils.isNotEmpty(changedTypeDefs.getUpdatedTypeDefs())) {
- for (AtlasBaseTypeDef typeDef : changedTypeDefs.getUpdatedTypeDefs()) {
- updateIndexForTypeDef(management, typeDef);
- }
- }
-
- // Invalidate the property key for deleted types
- if (CollectionUtils.isNotEmpty(changedTypeDefs.getDeletedTypeDefs())) {
- for (AtlasBaseTypeDef typeDef : changedTypeDefs.getDeletedTypeDefs()) {
- cleanupIndices(management, typeDef);
- }
- }
-
- //Commit indexes
- commit(management);
- } catch (RepositoryException | IndexException e) {
- LOG.error("Failed to update indexes for changed typedefs", e);
- attemptRollback(changedTypeDefs, management);
- }
-
- }
-
private void cleanupIndices(AtlasGraphManagement management, AtlasBaseTypeDef typeDef) {
Preconditions.checkNotNull(typeDef, "Cannot process null typedef");
if (LOG.isDebugEnabled()) {
@@ -816,14 +650,4 @@ public class GraphBackedSearchIndexer implements SearchIndexer, ActiveStateChang
addIndexForType(management, typeDef);
LOG.info("Index creation for type {} complete", typeDef.getName());
}
-
- /* Commenting this out since we do not need an index for edge label here
- private void createEdgeMixedIndex(String propertyName) {
- EdgeLabel edgeLabel = management.getEdgeLabel(propertyName);
- if (edgeLabel == null) {
- edgeLabel = management.makeEdgeLabel(propertyName).make();
- management.buildEdgeIndex(edgeLabel, propertyName, Direction.BOTH, Order.DEFAULT);
- LOG.info("Created index for edge label {}", propertyName);
- }
- }*/
}