You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@atlas.apache.org by sa...@apache.org on 2017/08/17 21:39:24 UTC
[2/2] atlas git commit: ATLAS-1988: Implement REST API to search for
related entities - Add excludeDeletedEntities flag
ATLAS-1988: Implement REST API to search for related entities - Add excludeDeletedEntities flag
(cherry picked from commit 0183beca085b95de4e8a24da85bae843011ce732)
Project: http://git-wip-us.apache.org/repos/asf/atlas/repo
Commit: http://git-wip-us.apache.org/repos/asf/atlas/commit/bb9384ca
Tree: http://git-wip-us.apache.org/repos/asf/atlas/tree/bb9384ca
Diff: http://git-wip-us.apache.org/repos/asf/atlas/diff/bb9384ca
Branch: refs/heads/master
Commit: bb9384ca32b2d4773fa3985851f3d5edf861dc13
Parents: f2a49be
Author: Sarath Subramanian <ss...@hortonworks.com>
Authored: Thu Aug 17 13:51:28 2017 -0700
Committer: Sarath Subramanian <ss...@hortonworks.com>
Committed: Thu Aug 17 14:33:16 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/atlas/AtlasClient.java | 1 +
.../atlas/discovery/AtlasDiscoveryService.java | 3 +-
.../atlas/discovery/EntityDiscoveryService.java | 61 ++++++++++++--------
.../store/graph/v1/EntityGraphRetriever.java | 2 +
.../atlas/util/AtlasGremlin2QueryProvider.java | 8 +--
.../apache/atlas/web/rest/DiscoveryREST.java | 17 +++---
6 files changed, 54 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/client/src/main/java/org/apache/atlas/AtlasClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/atlas/AtlasClient.java b/client/src/main/java/org/apache/atlas/AtlasClient.java
index c50f3f5..5fa5df7 100755
--- a/client/src/main/java/org/apache/atlas/AtlasClient.java
+++ b/client/src/main/java/org/apache/atlas/AtlasClient.java
@@ -101,6 +101,7 @@ public class AtlasClient extends AtlasBaseClient {
public static final String NAME = "name";
public static final String DESCRIPTION = "description";
public static final String OWNER = "owner";
+ public static final String CREATE_TIME = "createTime";
public static final String INFRASTRUCTURE_SUPER_TYPE = "Infrastructure";
public static final String DATA_SET_SUPER_TYPE = "DataSet";
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
index ead5d3c..8196a67 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java
@@ -73,9 +73,10 @@ public interface AtlasDiscoveryService {
* @param relation relation name.
* @param sortByAttribute sort the result using this attribute name, default value is 'name'
* @param sortOrder sorting order
+ * @param excludeDeletedEntities exclude deleted entities in search result.
* @param limit number of resultant rows (for pagination). [ limit > 0 ] and [ limit < maxlimit ]. -1 maps to atlas.search.defaultlimit property.
* @param offset offset to the results returned (for pagination). [ offset >= 0 ]. -1 maps to offset 0.
* @return AtlasSearchResult
*/
- AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, int limit, int offset) throws AtlasBaseException;
+ AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttribute, SortOrder sortOrder, boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException;
}
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
index d5062a7..1e68835 100644
--- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
+++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java
@@ -76,14 +76,19 @@ import java.util.*;
import static org.apache.atlas.AtlasErrorCode.CLASSIFICATION_NOT_FOUND;
import static org.apache.atlas.AtlasErrorCode.DISCOVERY_QUERY_FAILED;
import static org.apache.atlas.AtlasErrorCode.UNKNOWN_TYPENAME;
+import static org.apache.atlas.SortOrder.ASCENDING;
import static org.apache.atlas.SortOrder.DESCENDING;
import static org.apache.atlas.model.TypeCategory.ARRAY;
import static org.apache.atlas.model.TypeCategory.MAP;
import static org.apache.atlas.model.TypeCategory.OBJECT_ID_TYPE;
+import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE;
+import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED;
import static org.apache.atlas.repository.graph.GraphHelper.EDGE_LABEL_PREFIX;
+import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_DESCENDING_SORT;
import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.RELATIONSHIP_SEARCH_ASCENDING_SORT;
+import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST;
@Component
public class EntityDiscoveryService implements AtlasDiscoveryService {
@@ -364,9 +369,9 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
if (excludeDeletedEntities) {
- bindings.put("state", Status.ACTIVE.toString());
+ bindings.put("state", ACTIVE.toString());
- basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER);
+ basicQuery += gremlinQueryProvider.getQuery(BASIC_SEARCH_STATE_FILTER);
}
if (isGuidPrefixSearch) {
@@ -378,7 +383,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
bindings.put("startIdx", params.offset());
bindings.put("endIdx", params.offset() + params.limit());
- basicQuery += gremlinQueryProvider.getQuery(AtlasGremlinQuery.TO_RANGE_LIST);
+ basicQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
@@ -504,8 +509,8 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
@Override
@GraphTransaction
- public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName,
- SortOrder sortOrder, int limit, int offset) throws AtlasBaseException {
+ public AtlasSearchResult searchRelatedEntities(String guid, String relation, String sortByAttributeName, SortOrder sortOrder,
+ boolean excludeDeletedEntities, int limit, int offset) throws AtlasBaseException {
AtlasSearchResult ret = new AtlasSearchResult(AtlasQueryType.RELATIONSHIP);
if (StringUtils.isEmpty(guid) || StringUtils.isEmpty(relation)) {
@@ -543,21 +548,37 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
sortByAttributeName = sortByAttribute.getQualifiedName();
if (sortOrder == null) {
- sortOrder = SortOrder.ASCENDING;
+ sortOrder = ASCENDING;
}
}
- String relatedEntitiesQuery = getRelatedEntitiesQuery(sortOrder);
+ QueryParams params = validateSearchParams(limit, offset);
ScriptEngine scriptEngine = graph.getGremlinScriptEngine();
Bindings bindings = scriptEngine.createBindings();
- QueryParams params = validateSearchParams(limit, offset);
+ Set<String> states = getEntityStates();
+ String relatedEntitiesQuery = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
+
+ if (excludeDeletedEntities) {
+ states.remove(DELETED.toString());
+ }
+
+ if (sortOrder == ASCENDING) {
+ relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
+ bindings.put("sortAttributeName", sortByAttributeName);
+
+ } else if (sortOrder == DESCENDING) {
+ relatedEntitiesQuery += gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
+ bindings.put("sortAttributeName", sortByAttributeName);
+ }
+
+ relatedEntitiesQuery += gremlinQueryProvider.getQuery(TO_RANGE_LIST);
bindings.put("g", graph);
bindings.put("guid", guid);
bindings.put("relation", relation);
- bindings.put("sortAttributeName", sortByAttributeName);
- bindings.put("offset", params.offset());
- bindings.put("limit", params.offset() + params.limit());
+ bindings.put("states", Collections.unmodifiableSet(states));
+ bindings.put("startIdx", params.offset());
+ bindings.put("endIdx", params.offset() + params.limit());
try {
Object result = graph.executeGremlinScript(scriptEngine, bindings, relatedEntitiesQuery, false);
@@ -725,7 +746,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
}
private boolean skipDeletedEntities(boolean excludeDeletedEntities, AtlasVertex<?, ?> vertex) {
- return excludeDeletedEntities && GraphHelper.getStatus(vertex) == Status.DELETED;
+ return excludeDeletedEntities && GraphHelper.getStatus(vertex) == DELETED;
}
private static String getClassificationFilter(AtlasTypeRegistry typeRegistry, String classificationName, int maxTypesLengthInIdxQuery) {
@@ -767,17 +788,7 @@ public class EntityDiscoveryService implements AtlasDiscoveryService {
return ret;
}
- private String getRelatedEntitiesQuery(SortOrder sortOrder) {
- final String ret;
-
- if (sortOrder == null) {
- ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH);
- } else if (sortOrder == DESCENDING) {
- ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_DESCENDING_SORT);
- } else {
- ret = gremlinQueryProvider.getQuery(RELATIONSHIP_SEARCH_ASCENDING_SORT);
- }
-
- return ret;
+ private Set<String> getEntityStates() {
+ return new HashSet<>(Arrays.asList(ACTIVE.toString(), DELETED.toString()));
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
index 3c2ddb7..4e7aa47 100644
--- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
+++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/EntityGraphRetriever.java
@@ -232,11 +232,13 @@ public final class EntityGraphRetriever {
Object name = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.NAME));
Object description = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.DESCRIPTION));
Object owner = getVertexAttribute(entityVertex, entityType.getAttribute(AtlasClient.OWNER));
+ Object createTime = entityVertex.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
Object displayText = name != null ? name : ret.getAttribute(AtlasClient.QUALIFIED_NAME);
ret.setAttribute(AtlasClient.NAME, name);
ret.setAttribute(AtlasClient.DESCRIPTION, description);
ret.setAttribute(AtlasClient.OWNER, owner);
+ ret.setAttribute(AtlasClient.CREATE_TIME, createTime);
if (displayText != null) {
ret.setDisplayText(displayText.toString());
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
----------------------------------------------------------------------
diff --git a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
index 58303bf..33f9b7a 100644
--- a/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
+++ b/repository/src/main/java/org/apache/atlas/util/AtlasGremlin2QueryProvider.java
@@ -96,11 +96,11 @@ public class AtlasGremlin2QueryProvider extends AtlasGremlinQueryProvider {
case COMPARE_CONTAINS:
return ".filter({it.getProperty('%s').contains(%s)})";
case RELATIONSHIP_SEARCH:
- return "g.V('__guid', guid).both(relation)[offset..<limit].toList()";
- case RELATIONSHIP_SEARCH_DESCENDING_SORT:
- return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}.toList()";
+ return "g.V('__guid', guid).both(relation).has('__state', T.in, states)";
case RELATIONSHIP_SEARCH_ASCENDING_SORT:
- return "g.V('__guid', guid).both(relation)[offset..<limit].order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}.toList()";
+ return ".order{it.a.getProperty(sortAttributeName) <=> it.b.getProperty(sortAttributeName)}";
+ case RELATIONSHIP_SEARCH_DESCENDING_SORT:
+ return ".order{it.b.getProperty(sortAttributeName) <=> it.a.getProperty(sortAttributeName)}";
}
// Should never reach this point
return null;
http://git-wip-us.apache.org/repos/asf/atlas/blob/bb9384ca/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
index 032bde8..52258e3 100644
--- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
+++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -285,21 +285,22 @@ public class DiscoveryREST {
@Path("relationship")
@Consumes(Servlets.JSON_MEDIA_TYPE)
@Produces(Servlets.JSON_MEDIA_TYPE)
- public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid,
- @QueryParam("relation") String relation,
- @QueryParam("sortBy") String sortByAttribute,
- @QueryParam("sortOrder") SortOrder sortOrder,
- @QueryParam("limit") int limit,
- @QueryParam("offset") int offset) throws AtlasBaseException {
+ public AtlasSearchResult searchRelatedEntities(@QueryParam("guid") String guid,
+ @QueryParam("relation") String relation,
+ @QueryParam("sortBy") String sortByAttribute,
+ @QueryParam("sortOrder") SortOrder sortOrder,
+ @QueryParam("excludeDeletedEntities") boolean excludeDeletedEntities,
+ @QueryParam("limit") int limit,
+ @QueryParam("offset") int offset) throws AtlasBaseException {
AtlasPerfTracer perf = null;
try {
if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.relatedEntitiesSearchUsingGremlin(" + guid +
- ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + limit + ", " + offset + ")");
+ ", " + relation + ", " + sortByAttribute + ", " + sortOrder + ", " + excludeDeletedEntities + ", " + ", " + limit + ", " + offset + ")");
}
- return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, limit, offset);
+ return atlasDiscoveryService.searchRelatedEntities(guid, relation, sortByAttribute, sortOrder, excludeDeletedEntities, limit, offset);
} finally {
AtlasPerfTracer.log(perf);
}