You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by am...@apache.org on 2020/05/05 16:39:32 UTC
[atlas] branch master updated (de69653 -> 23aea76)
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git.
from de69653 ATLAS-3777:- Classic UI , Regression : On clicking tag filters , the dialog window says "No Attributes are available !"
new f467229 ATLAS-3775: Migration Import: Status display fix.
new 23aea76 ATLAS-3762: Improve edge creation using genuine iterator. Part 2
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../atlas/repository/graphdb/AtlasGraph.java | 13 +++++-
.../repository/graphdb/janus/AtlasJanusGraph.java | 11 ++++++
.../apache/atlas/repository/graph/GraphHelper.java | 20 +++++-----
.../migration/DataMigrationStatusService.java | 46 ++++++++--------------
.../migration/ZipFileMigrationImporter.java | 2 +
.../store/graph/v2/AtlasRelationshipStoreV2.java | 25 +-----------
.../impexp/DataMigrationStatusServiceTest.java | 1 -
7 files changed, 52 insertions(+), 66 deletions(-)
[atlas] 01/02: ATLAS-3775: Migration Import: Status display fix.
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit f467229294baa5550f4101d1e8ea59e82fdf3e3b
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon May 4 15:59:46 2020 -0700
ATLAS-3775: Migration Import: Status display fix.
---
.../migration/DataMigrationStatusService.java | 46 ++++++++--------------
.../migration/ZipFileMigrationImporter.java | 2 +
.../impexp/DataMigrationStatusServiceTest.java | 1 -
3 files changed, 18 insertions(+), 31 deletions(-)
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
index e4c4e36..a22c687 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/DataMigrationStatusService.java
@@ -22,15 +22,12 @@ import org.apache.atlas.model.migration.MigrationImportStatus;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graph.AtlasGraphProvider;
import org.apache.atlas.repository.graphdb.AtlasGraph;
-import org.apache.atlas.repository.graphdb.AtlasIndexQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2;
-import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Date;
-import java.util.Iterator;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.getEncodedProperty;
import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.setEncodedProperty;
@@ -47,8 +44,8 @@ public class DataMigrationStatusService {
this.migrationStatusVertexManagement = new MigrationStatusVertexManagement(AtlasGraphProvider.getGraphInstance());
}
- public DataMigrationStatusService(AtlasGraph atlasGraph) {
- this.migrationStatusVertexManagement = new MigrationStatusVertexManagement(atlasGraph);
+ public DataMigrationStatusService(AtlasGraph graph) {
+ this.migrationStatusVertexManagement = new MigrationStatusVertexManagement(graph);
}
@@ -76,13 +73,11 @@ public class DataMigrationStatusService {
}
public MigrationImportStatus getStatus() {
- if (this.status != null &&
- StringUtils.isEmpty(this.status.getOperationStatus()) &&
- this.migrationStatusVertexManagement.exists(this.status.getName())) {
+ if (this.status != null && this.migrationStatusVertexManagement.exists(this.status.getName())) {
return getCreate(this.status);
- } else {
- return this.status;
}
+
+ return this.status;
}
public MigrationImportStatus getByName(String name) {
@@ -115,18 +110,18 @@ public class DataMigrationStatusService {
public static final String PROPERTY_KEY_POSITION = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.position");
public static final String PROPERTY_KEY_STATUS = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "migration.status");
- private AtlasGraph atlasGraph;
+ private AtlasGraph graph;
private AtlasVertex vertex;
- public MigrationStatusVertexManagement(AtlasGraph atlasGraph) {
- this.atlasGraph = atlasGraph;
+ public MigrationStatusVertexManagement(AtlasGraph graph) {
+ this.graph = graph;
}
public MigrationImportStatus createOrUpdate(MigrationImportStatus status) {
this.vertex = findByNameInternal(status.getName());
if (this.vertex == null) {
- this.vertex = atlasGraph.addVertex();
+ this.vertex = graph.addVertex();
LOG.info("MigrationStatusVertexManagement: Vertex created!");
updateVertex(this.vertex, status);
}
@@ -156,29 +151,20 @@ public class DataMigrationStatusService {
public void delete(String name) {
try {
AtlasVertex vertex = findByNameInternal(name);
- atlasGraph.removeVertex(vertex);
+ graph.removeVertex(vertex);
this.vertex = null;
} finally {
- atlasGraph.commit();
+ graph.commit();
}
}
private AtlasVertex findByNameInternal(String name) {
try {
- String idxQueryString = String.format("%s\"%s\":\"%s\"", AtlasGraphUtilsV2.getIndexSearchPrefix(), Constants.GUID_PROPERTY_KEY, name);
- AtlasIndexQuery idxQuery = atlasGraph.indexQuery(Constants.VERTEX_INDEX, idxQueryString);
- Iterator<AtlasIndexQuery.Result<Object, Object>> results = idxQuery.vertices();
-
- AtlasIndexQuery.Result<?, ?> qryResult = results.hasNext() ? results.next() : null;
- if (qryResult != null) {
- return qryResult.getVertex();
- } else {
- return null;
- }
+ return AtlasGraphUtilsV2.findByGuid(graph, name);
} catch (Exception e) {
LOG.error("MigrationStatusVertexManagement.findByNameInternal: Failed!", e);
} finally {
- atlasGraph.commit();
+ graph.commit();
}
return null;
@@ -190,7 +176,7 @@ public class DataMigrationStatusService {
} catch (Exception e) {
LOG.warn("Error updating status. Please rely on log messages.", e);
} finally {
- atlasGraph.commit();
+ graph.commit();
}
}
@@ -200,7 +186,7 @@ public class DataMigrationStatusService {
} catch (Exception e) {
LOG.warn("Error updating status. Please rely on log messages.", e);
} finally {
- atlasGraph.commit();
+ graph.commit();
}
}
@@ -219,7 +205,7 @@ public class DataMigrationStatusService {
} catch (Exception ex) {
LOG.error("Error updating MigrationImportStatus vertex. Status may not be persisted correctly.", ex);
} finally {
- atlasGraph.commit();
+ graph.commit();
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
index f44f2a8..d56261f 100644
--- a/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
+++ b/repository/src/main/java/org/apache/atlas/repository/migration/ZipFileMigrationImporter.java
@@ -73,8 +73,10 @@ public class ZipFileMigrationImporter implements Runnable {
dataMigrationStatusService.setStatus("DONE");
} catch (IOException e) {
LOG.error("Migration Import: IO Error!", e);
+ dataMigrationStatusService.setStatus("FAIL");
} catch (AtlasBaseException e) {
LOG.error("Migration Import: Error!", e);
+ dataMigrationStatusService.setStatus("FAIL");
}
}
diff --git a/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
index 742b4ac..f1dc990 100644
--- a/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
+++ b/repository/src/test/java/org/apache/atlas/repository/impexp/DataMigrationStatusServiceTest.java
@@ -68,6 +68,5 @@ public class DataMigrationStatusServiceTest {
dataMigrationStatusService.delete();
assertNull(dataMigrationStatusService.getStatus());
- assertNull(dataMigrationStatusService.getByName(ret.getName()));
}
}
[atlas] 02/02: ATLAS-3762: Improve edge creation using genuine
iterator. Part 2
Posted by am...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
amestry pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/atlas.git
commit 23aea76fff6c8f530816319649218583f4dfd091
Author: Ashutosh Mestry <am...@cloudera.com>
AuthorDate: Mon May 4 18:46:17 2020 -0700
ATLAS-3762: Improve edge creation using genuine iterator. Part 2
---
.../atlas/repository/graphdb/AtlasGraph.java | 13 +++++++++--
.../repository/graphdb/janus/AtlasJanusGraph.java | 11 ++++++++++
.../apache/atlas/repository/graph/GraphHelper.java | 20 +++++++++--------
.../store/graph/v2/AtlasRelationshipStoreV2.java | 25 +---------------------
4 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
index 7bd5f2d..c016f63 100644
--- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
+++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasGraph.java
@@ -48,10 +48,19 @@ public interface AtlasGraph<V, E> {
AtlasEdge<V, E> addEdge(AtlasVertex<V, E> outVertex, AtlasVertex<V, E> inVertex, String label);
/**
- * Adds a vertex to the graph.
- *
+ * Fetch edges between two vertices using relationshipLabel
+ * @param fromVertex
+ * @param toVertex
+ * @param relationshipLabel
* @return
*/
+ AtlasEdge<V, E> getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVertex, String relationshipLabel);
+
+ /**
+ * Adds a vertex to the graph.
+ *
+ * @return
+ */
AtlasVertex<V, E> addVertex();
/**
diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
index a30dbc7..35274c9 100644
--- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
+++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusGraph.java
@@ -172,6 +172,17 @@ public class AtlasJanusGraph implements AtlasGraph<AtlasJanusVertex, AtlasJanusE
}
@Override
+ public AtlasEdge getEdgeBetweenVertices(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
+ GraphTraversal gt = V(fromVertex.getId()).outE(edgeLabel).where(__.otherV().hasId(toVertex.getId()));
+ Object o = gt.hasNext() ? gt.next() : null;
+ if (o == null) {
+ return null;
+ }
+
+ return GraphDbObjectFactory.createEdge(this, (Edge) o);
+ }
+
+ @Override
public AtlasEdge<AtlasJanusVertex, AtlasJanusEdge> getEdge(String edgeId) {
Iterator<Edge> it = getGraph().edges(edgeId);
Edge e = getSingleElement(it, edgeId);
diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
index b9e3a5e..b1368e5 100755
--- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
+++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java
@@ -739,19 +739,21 @@ public final class GraphHelper {
public static List<AtlasVertex> getPropagationEnabledClassificationVertices(AtlasVertex entityVertex) {
List<AtlasVertex> ret = new ArrayList<>();
- Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges();
+ if (entityVertex.hasEdges(AtlasEdgeDirection.OUT, CLASSIFICATION_LABEL)) {
+ Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL).edges();
- if (edges != null) {
- Iterator<AtlasEdge> iterator = edges.iterator();
+ if (edges != null) {
+ Iterator<AtlasEdge> iterator = edges.iterator();
- while (iterator.hasNext()) {
- AtlasEdge edge = iterator.next();
+ while (iterator.hasNext()) {
+ AtlasEdge edge = iterator.next();
- if (edge != null) {
- AtlasVertex classificationVertex = edge.getInVertex();
+ if (edge != null) {
+ AtlasVertex classificationVertex = edge.getInVertex();
- if (isPropagationEnabled(classificationVertex)) {
- ret.add(classificationVertex);
+ if (isPropagationEnabled(classificationVertex)) {
+ ret.add(classificationVertex);
+ }
}
}
}
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
index ab431bc..8d74489 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java
@@ -784,36 +784,13 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore {
AtlasEdge ret = null;
if (toVertex.hasEdges(AtlasEdgeDirection.IN, relationshipLabel) && fromVertex.hasEdges(AtlasEdgeDirection.OUT, relationshipLabel)) {
- long fromVertexOutgoingEdgeCount = graphHelper.getOutGoingEdgesCountByLabel(fromVertex, relationshipLabel);
- long toVertexIncomingEdgeCount = graphHelper.getInComingEdgesCountByLabel(toVertex, relationshipLabel);
- if (toVertexIncomingEdgeCount < fromVertexOutgoingEdgeCount) {
- Iterator<AtlasEdge> edgesIteratorIn = graphHelper.getIncomingEdgesByLabel(toVertex, relationshipLabel);
- ret = getActiveEdgeFromList(edgesIteratorIn, fromVertex.getId(), e -> e.getOutVertex().getId());
- } else {
- Iterator<AtlasEdge> edgesIteratorOut = graphHelper.getOutGoingEdgesByLabel(fromVertex, relationshipLabel);
- ret = getActiveEdgeFromList(edgesIteratorOut, toVertex.getId(), e -> e.getInVertex().getId());
- }
+ ret = graph.getEdgeBetweenVertices(fromVertex, toVertex, relationshipLabel);
}
RequestContext.get().endMetricRecord(metric);
return ret;
}
- private AtlasEdge getActiveEdgeFromList(Iterator<AtlasEdge> edgesIterator, Object vertexIdToCompare, Function<AtlasEdge, Object> edgeIdFn) {
- while (edgesIterator != null && edgesIterator.hasNext()) {
- AtlasEdge edge = edgesIterator.next();
- if (edge != null) {
- Status status = graphHelper.getStatus(edge);
-
- if ((status == null || status == ACTIVE) && edgeIdFn.apply(edge).equals(vertexIdToCompare)) {
- return edge;
- }
- }
- }
-
- return null;
- }
-
private Long getRelationshipVersion(AtlasRelationship relationship) {
Long ret = relationship != null ? relationship.getVersion() : null;