You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/09/12 22:29:33 UTC
[5/5] git commit: FALCON-662 Fetch relationships for a given type
API. Contributed by Balu Vellanki
FALCON-662 Fetch relationships for a given type API. Contributed by Balu Vellanki
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/fa0f6126
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/fa0f6126
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/fa0f6126
Branch: refs/heads/master
Commit: fa0f612695fa6538a02ea1c15eb4e96c78b4641a
Parents: 7f98570
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Fri Sep 12 13:29:04 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Fri Sep 12 13:29:04 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../apache/falcon/cli/FalconMetadataCLI.java | 16 ++
.../org/apache/falcon/client/FalconClient.java | 41 +++-
docs/src/site/twiki/FalconCLI.twiki | 13 +-
.../site/twiki/restapi/MetadataRelations.twiki | 45 +++++
docs/src/site/twiki/restapi/ResourceList.twiki | 3 +-
.../metadata/MetadataDiscoveryResource.java | 97 ++++++++-
.../metadata/MetadataDiscoveryResourceTest.java | 195 +++++++++++++++++--
.../java/org/apache/falcon/cli/FalconCLIIT.java | 57 +++++-
.../resource/MetadataResourceJerseyIT.java | 28 ++-
10 files changed, 445 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f1132d9..c65a378 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -33,6 +33,9 @@ Trunk (Unreleased)
FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
IMPROVEMENTS
+ FALCON-662 Fetch relationships for a given type API (Balu Vellanki via
+ Venkatesh Seetharam)
+
FALCON-661 Add list types to Lineage API (Balu Vellanki via
Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/client/src/main/java/org/apache/falcon/cli/FalconMetadataCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconMetadataCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconMetadataCLI.java
index d3956b0..b0342f0 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconMetadataCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconMetadataCLI.java
@@ -40,10 +40,12 @@ public class FalconMetadataCLI {
public static final AtomicReference<PrintStream> OUT = new AtomicReference<PrintStream>(System.out);
public static final String LIST_OPT = "list";
+ public static final String RELATIONS_OPT = "relations";
public static final String URL_OPTION = "url";
public static final String TYPE_OPT = "type";
public static final String CLUSTER_OPT = "cluster";
+ public static final String NAME_OPT = "name";
public FalconMetadataCLI() {}
@@ -56,11 +58,15 @@ public class FalconMetadataCLI {
String result = null;
String dimensionType = commandLine.getOptionValue(TYPE_OPT);
String cluster = commandLine.getOptionValue(CLUSTER_OPT);
+ String dimensionName = commandLine.getOptionValue(NAME_OPT);
validateDimensionType(dimensionType.toUpperCase());
if (optionsList.contains(LIST_OPT)) {
result = client.getDimensionList(dimensionType, cluster);
+ } else if (optionsList.contains(RELATIONS_OPT)) {
+ validateDimensionName(dimensionName, RELATIONS_OPT);
+ result = client.getDimensionRelations(dimensionType, dimensionName);
}
OUT.get().println(result);
@@ -78,21 +84,31 @@ public class FalconMetadataCLI {
}
}
+ private void validateDimensionName(String dimensionName, String action) throws FalconCLIException {
+ if (StringUtils.isEmpty(dimensionName)) {
+ throw new FalconCLIException("Dimension ID cannot be empty or null for action " + action);
+ }
+ }
+
public Options createMetadataOptions() {
Options metadataOptions = new Options();
OptionGroup group = new OptionGroup();
Option list = new Option(LIST_OPT, false, "List all dimensions");
+ Option relations = new Option(RELATIONS_OPT, false, "List all relations for a dimension");
group.addOption(list);
+ group.addOption(relations);
Option url = new Option(URL_OPTION, true, "Falcon URL");
Option type = new Option(TYPE_OPT, true, "Dimension type");
+ Option name = new Option(NAME_OPT, true, "Dimension name");
Option cluster = new Option(CLUSTER_OPT, true, "Cluster name");
metadataOptions.addOptionGroup(group);
metadataOptions.addOption(url);
metadataOptions.addOption(type);
metadataOptions.addOption(cluster);
+ metadataOptions.addOption(name);
return metadataOptions;
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 205ff31..d73560d 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -208,7 +208,8 @@ public class FalconClient {
*/
protected static enum MetadataOperations {
- LIST("api/metadata/", HttpMethod.GET, MediaType.APPLICATION_JSON);
+ LIST("api/metadata/", HttpMethod.GET, MediaType.APPLICATION_JSON),
+ RELATIONS("api/metadata/", HttpMethod.GET, MediaType.APPLICATION_JSON);
private String path;
private String method;
@@ -504,7 +505,11 @@ public class FalconClient {
}
public String getDimensionList(String dimensionType, String cluster) throws FalconCLIException {
- return sendMetadataListRequest(MetadataOperations.LIST, dimensionType, cluster);
+ return sendMetadataRequest(MetadataOperations.LIST, dimensionType, null, cluster);
+ }
+
+ public String getDimensionRelations(String dimensionType, String dimensionName) throws FalconCLIException {
+ return sendMetadataRequest(MetadataOperations.RELATIONS, dimensionType, dimensionName, null);
}
/**
@@ -788,12 +793,28 @@ public class FalconClient {
return parseStringResult(clientResponse);
}
- private String sendMetadataListRequest(final MetadataOperations op, final String dimensionType,
- final String cluster) throws FalconCLIException {
- WebResource resource = service
- .path(op.path)
- .path(dimensionType)
- .path(FalconMetadataCLI.LIST_OPT);
+ private String sendMetadataRequest(final MetadataOperations operation,
+ final String dimensionType,
+ final String dimensionName,
+ final String cluster) throws FalconCLIException {
+ WebResource resource;
+ switch (operation) {
+ case LIST:
+ resource = service.path(operation.path)
+ .path(dimensionType)
+ .path(FalconMetadataCLI.LIST_OPT);
+ break;
+
+ case RELATIONS:
+ resource = service.path(operation.path)
+ .path(dimensionType)
+ .path(dimensionName)
+ .path(FalconMetadataCLI.RELATIONS_OPT);
+ break;
+
+ default:
+ throw new FalconCLIException("Invalid Metadata client Operation " + operation.toString());
+ }
if (!StringUtils.isEmpty(cluster)) {
resource = resource.queryParam(FalconMetadataCLI.CLUSTER_OPT, cluster);
@@ -801,8 +822,8 @@ public class FalconClient {
ClientResponse clientResponse = resource
.header("Cookie", AUTH_COOKIE_EQ + authenticationToken)
- .accept(op.mimeType).type(op.mimeType)
- .method(op.method, ClientResponse.class);
+ .accept(operation.mimeType).type(operation.mimeType)
+ .method(operation.method, ClientResponse.class);
checkIfSuccessful(clientResponse);
return parseStringResult(clientResponse);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/docs/src/site/twiki/FalconCLI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconCLI.twiki b/docs/src/site/twiki/FalconCLI.twiki
index d4ce366..b61feeb 100644
--- a/docs/src/site/twiki/FalconCLI.twiki
+++ b/docs/src/site/twiki/FalconCLI.twiki
@@ -305,18 +305,25 @@ $FALCON_HOME/bin/falcon admin -status
---+++ List
-List all dimensions of type.
-
+Lists of all dimensions of given type. If the user provides optional param cluster, only the dimensions related to the cluster are listed.
Usage:
$FALCON_HOME/bin/falcon metadata -list -type [cluster_entity|feed_entity|process_entity|user|colo|tags|groups|pipelines]
Optional Args : -cluster <<cluster name>>
-Lists of all dimensions of given type. If the user provides optional param cluster, only the dimensions related to the cluster are listed.
Example:
$FALCON_HOME/bin/falcon metadata -list -type process_entity -cluster primary-cluster
$FALCON_HOME/bin/falcon metadata -list -type tags
+---+++ Relations
+
+List all dimensions related to specified Dimension identified by dimension-type and dimension-name.
+Usage:
+$FALCON_HOME/bin/falcon metadata -relations -type [cluster_entity|feed_entity|process_entity|user|colo|tags|groups|pipelines] -name <<Dimension Name>>
+
+Example:
+$FALCON_HOME/bin/falcon metadata -relations -type process_entity -name sample-process
+
---++ Recipe Options
---+++ Submit Recipe
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/docs/src/site/twiki/restapi/MetadataRelations.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/MetadataRelations.twiki b/docs/src/site/twiki/restapi/MetadataRelations.twiki
new file mode 100644
index 0000000..015a57b
--- /dev/null
+++ b/docs/src/site/twiki/restapi/MetadataRelations.twiki
@@ -0,0 +1,45 @@
+---++ GET api/metadata/:dimension-type/:dimension-name/relations
+ * <a href="#Description">Description</a>
+ * <a href="#Parameters">Parameters</a>
+ * <a href="#Results">Results</a>
+ * <a href="#Examples">Examples</a>
+
+---++ Description
+Get all relations of a specific dimension.
+
+---++ Parameters
+ * :type Valid dimension types are cluster_entity,feed_entity, process_entity, user, colo, tags, groups, pipelines
+ * :name Name of the dimension.
+
+---++ Results
+Get all relations of a specific dimension.
+
+---++ Examples
+---+++ Rest Call
+<verbatim>
+GET http://localhost:15000/api/metadata/process_entity/sample-process/relations
+</verbatim>
+---+++ Result
+<verbatim>
+{
+ "timestamp":"2014-09-09T01:31Z",
+ "userWorkflowEngine":"pig",
+ "name":"sample-process",
+ "type":"PROCESS_ENTITY",
+ "userWorkflowName":"imp-click-join-workflow",
+ "version":"1.0.9",
+ "inVertices":[
+ {"name":"clicks-feed","type":"FEED_ENTITY","label":"input"},
+ {"name":"impression-feed","type":"FEED_ENTITY","label":"input"},
+ {"name":"sample-process\/2014-01-01T01:00Z","type":"PROCESS_INSTANCE","label":"instance-of"}
+ ],
+ "outVertices":[
+ {"name":"Critical","type":"TAGS","label":"classified-as"},
+ {"name":"testPipeline","type":"PIPELINES","label":"pipeline"},
+ {"name":"primary-cluster","type":"CLUSTER_ENTITY","label":"runs-on"},
+ {"name":"imp-click-join2","type":"FEED_ENTITY","label":"output"},
+ {"name":"imp-click-join1","type":"FEED_ENTITY","label":"output"},
+ {"name":"falcon-user","type":"USER","label":"owned-by"}
+ ]
+}
+</verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/docs/src/site/twiki/restapi/ResourceList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/ResourceList.twiki b/docs/src/site/twiki/restapi/ResourceList.twiki
index 34a07d5..d6a73f2 100644
--- a/docs/src/site/twiki/restapi/ResourceList.twiki
+++ b/docs/src/site/twiki/restapi/ResourceList.twiki
@@ -81,4 +81,5 @@ See also: [[../Security.twiki][Security in Falcon]]
---++ REST Call on Metadata Resource
| *Call Type* | *Resource* | *Description* |
-| GET | [[MetadataList][api/metadata/:dimension-type/list]] | list of dimensions |
\ No newline at end of file
+| GET | [[MetadataList][api/metadata/:dimension-type/list]] | list of dimensions |
+| GET | [MetadataRelations][api/metadata/:dimension-type/:dimension-name/relations]] | Return all relations of a dimension |
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/prism/src/main/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResource.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResource.java b/prism/src/main/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResource.java
index d15d7cc..63ed3b6 100644
--- a/prism/src/main/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResource.java
+++ b/prism/src/main/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResource.java
@@ -22,6 +22,8 @@ import com.tinkerpop.blueprints.Direction;
import com.tinkerpop.blueprints.Edge;
import com.tinkerpop.blueprints.GraphQuery;
import com.tinkerpop.blueprints.Vertex;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONMode;
+import com.tinkerpop.blueprints.util.io.graphson.GraphSONUtility;
import org.apache.commons.lang.StringUtils;
import org.apache.falcon.metadata.RelationshipLabel;
import org.apache.falcon.metadata.RelationshipProperty;
@@ -54,9 +56,9 @@ public class MetadataDiscoveryResource extends AbstractMetadataResource {
@GET
@Path("/{type}/list")
@Produces({MediaType.APPLICATION_JSON})
- public Response listDimensionValues(@PathParam("type") String type,
+ public Response listDimensionValues(@PathParam("type") String dimensionType,
@QueryParam("cluster") final String clusterName) {
- RelationshipType relationshipType = validateAndParseDimensionType(type.toUpperCase());
+ RelationshipType relationshipType = validateAndParseDimensionType(dimensionType.toUpperCase());
GraphQuery query = getGraph().query();
JSONArray dimensionValues = new JSONArray();
@@ -66,13 +68,10 @@ public class MetadataDiscoveryResource extends AbstractMetadataResource {
query.vertices().iterator());
} else {
// Get clusterVertex, get adjacent vertices of clusterVertex that match dimension type.
- query = query
- .has(RelationshipProperty.TYPE.getName(), RelationshipType.CLUSTER_ENTITY.getName())
- .has(RelationshipProperty.NAME.getName(), clusterName);
- Iterator<Vertex> clusterIterator = query.vertices().iterator();
- if (clusterIterator.hasNext()) {
- dimensionValues = getDimensionsFromClusterVertex(
- dimensionValues, clusterIterator.next(), relationshipType);
+ Vertex clusterVertex = getVertex(clusterName, RelationshipType.CLUSTER_ENTITY.getName());
+ if (clusterVertex != null) {
+ dimensionValues = getDimensionsFromClusterVertex(dimensionValues,
+ clusterVertex, relationshipType);
} // else, no cluster found. Return empty results
}
@@ -87,6 +86,76 @@ public class MetadataDiscoveryResource extends AbstractMetadataResource {
}
}
+ /**
+ * Get relations of a dimension identified by type and name.
+ *
+ * GET http://host/metadata/dimension-type/dimension-name/relations
+ */
+ @GET
+ @Path("/{type}/{name}/relations")
+ @Produces({MediaType.APPLICATION_JSON})
+ public Response getDimensionRelations(@PathParam("type") String dimensionType,
+ @PathParam("name") String dimensionName) {
+ RelationshipType relationshipType = validateAndParseDimensionType(dimensionType.toUpperCase());
+ validateDimensionName(dimensionName);
+ Vertex dimensionVertex = getVertex(dimensionName, relationshipType.getName());
+ if (dimensionVertex == null) {
+ return Response.ok(new JSONObject()).build();
+ }
+
+ JSONObject vertexProperties;
+ try {
+ vertexProperties = GraphSONUtility.jsonFromElement(
+ dimensionVertex, getVertexIndexedKeys(), GraphSONMode.NORMAL);
+ // over-write the type - fix this kludge
+ vertexProperties.put(RelationshipProperty.TYPE.getName(), relationshipType.toString());
+
+ Iterator<Edge> inEdges = dimensionVertex.query().direction(Direction.IN).edges().iterator();
+ vertexProperties.put("inVertices", getAdjacentVerticesJson(inEdges, Direction.OUT));
+
+ Iterator<Edge> outEdges = dimensionVertex.query().direction(Direction.OUT).edges().iterator();
+ vertexProperties.put("outVertices", getAdjacentVerticesJson(outEdges, Direction.IN));
+
+ } catch (JSONException e) {
+ throw new WebApplicationException(Response.status(Response.Status.INTERNAL_SERVER_ERROR)
+ .entity(JSONObject.quote("An error occurred: " + e.getMessage())).build());
+ }
+
+ return Response.ok(vertexProperties).build();
+ }
+
+ private JSONArray getAdjacentVerticesJson(Iterator<Edge> edges,
+ Direction direction) throws JSONException {
+ JSONArray adjVertices = new JSONArray();
+ while (edges.hasNext()) {
+ Edge edge = edges.next();
+ Vertex vertex = edge.getVertex(direction);
+ JSONObject vertexObject = new JSONObject();
+ vertexObject.put(RelationshipProperty.NAME.getName(),
+ vertex.getProperty(RelationshipProperty.NAME.getName()));
+ vertexObject.put(RelationshipProperty.TYPE.getName(),
+ getVertexRelationshipType(vertex));
+ vertexObject.put("label", edge.getLabel());
+ adjVertices.put(vertexObject);
+ }
+
+ return adjVertices;
+ }
+
+ private String getVertexRelationshipType(Vertex vertex) {
+ String type = vertex.getProperty(RelationshipProperty.TYPE.getName());
+ return RelationshipType.fromString(type).toString();
+ }
+
+ private Vertex getVertex(String name, String type) {
+ Iterator<Vertex> vertexIterator = getGraph().query()
+ .has(RelationshipProperty.TYPE.getName(), type)
+ .has(RelationshipProperty.NAME.getName(), name)
+ .vertices().iterator();
+
+ return vertexIterator.hasNext() ? vertexIterator.next() : null;
+ }
+
private JSONArray getDimensionsFromClusterVertex(JSONArray dimensionValues, Vertex clusterVertex,
RelationshipType relationshipType) {
switch (relationshipType) {
@@ -153,7 +222,15 @@ public class MetadataDiscoveryResource extends AbstractMetadataResource {
return RelationshipType.valueOf(type);
} catch (IllegalArgumentException iae) {
throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
- .entity("Invalid Dimension type : " + type).type("text/plain").build());
+ .entity("Invalid Dimension type : " + type).type("text/plain").build());
+ }
+ }
+
+ private void validateDimensionName(String name) {
+ if (StringUtils.isEmpty(name)) {
+ throw new WebApplicationException(Response.status(Response.Status.BAD_REQUEST)
+ .entity("Dimension name cannot be empty for Relations API").type("text/plain")
+ .build());
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResourceTest.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResourceTest.java b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResourceTest.java
index 0198172..7095785 100644
--- a/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResourceTest.java
+++ b/prism/src/test/java/org/apache/falcon/resource/metadata/MetadataDiscoveryResourceTest.java
@@ -18,12 +18,14 @@
package org.apache.falcon.resource.metadata;
+import org.apache.falcon.metadata.RelationshipType;
import org.json.simple.JSONValue;
import org.testng.Assert;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Response;
import java.util.List;
import java.util.Map;
@@ -49,14 +51,15 @@ public class MetadataDiscoveryResourceTest {
@Test
public void testListDimensionsFeed() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("feed_entity", MetadataTestContext.CLUSTER_ENTITY_NAME);
+ Response response = resource.listDimensionValues(RelationshipType.FEED_ENTITY.toString(),
+ MetadataTestContext.CLUSTER_ENTITY_NAME);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 4);
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertTrue(dimensions.contains("impression-feed"));
- response = resource.listDimensionValues("feed_entity", null);
+ response = resource.listDimensionValues(RelationshipType.FEED_ENTITY.toString(), null);
results = (Map) JSONValue.parse(response.getEntity().toString());
dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 4);
@@ -66,7 +69,7 @@ public class MetadataDiscoveryResourceTest {
@Test
public void testListDimensionsProcess() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("process_entity",
+ Response response = resource.listDimensionValues(RelationshipType.PROCESS_ENTITY.toString(),
MetadataTestContext.CLUSTER_ENTITY_NAME);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
@@ -74,7 +77,7 @@ public class MetadataDiscoveryResourceTest {
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(dimensions.get(0), MetadataTestContext.PROCESS_ENTITY_NAME);
- response = resource.listDimensionValues("process_entity", null);
+ response = resource.listDimensionValues(RelationshipType.PROCESS_ENTITY.toString(), null);
results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
}
@@ -82,14 +85,15 @@ public class MetadataDiscoveryResourceTest {
@Test
public void testListDimensionsCluster() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("cluster_entity", MetadataTestContext.CLUSTER_ENTITY_NAME);
+ Response response = resource.listDimensionValues(RelationshipType.CLUSTER_ENTITY.toString(),
+ MetadataTestContext.CLUSTER_ENTITY_NAME);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(dimensions.get(0), MetadataTestContext.CLUSTER_ENTITY_NAME);
- response = resource.listDimensionValues("cluster_entity", null);
+ response = resource.listDimensionValues(RelationshipType.CLUSTER_ENTITY.toString(), null);
results = (Map) JSONValue.parse(response.getEntity().toString());
dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
@@ -99,38 +103,33 @@ public class MetadataDiscoveryResourceTest {
@Test
public void testListDimensionsColo() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("colo", MetadataTestContext.CLUSTER_ENTITY_NAME);
+ Response response = resource.listDimensionValues(RelationshipType.COLO.toString(),
+ MetadataTestContext.CLUSTER_ENTITY_NAME);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(dimensions.get(0), MetadataTestContext.COLO_NAME);
- response = resource.listDimensionValues("colo", null);
+ response = resource.listDimensionValues(RelationshipType.COLO.toString(), null);
results = (Map) JSONValue.parse(response.getEntity().toString());
dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
Assert.assertEquals(dimensions.get(0), MetadataTestContext.COLO_NAME);
-
- try {
- resource.listDimensionValues("INVALID", null);
- Assert.assertTrue(false);
- } catch (Exception e) {
- Assert.assertTrue(true);
- }
}
@Test
public void testListDimensionsPipelines() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("pipelines", null);
+ Response response = resource.listDimensionValues(RelationshipType.PIPELINES.toString(), null);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 1);
Assert.assertEquals(dimensions.get(0), "testPipeline");
- response = resource.listDimensionValues("pipelines", MetadataTestContext.CLUSTER_ENTITY_NAME);
+ response = resource.listDimensionValues(RelationshipType.PIPELINES.toString(),
+ MetadataTestContext.CLUSTER_ENTITY_NAME);
results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 0);
}
@@ -138,15 +137,169 @@ public class MetadataDiscoveryResourceTest {
@Test
public void testListDimensionsTags() throws Exception {
MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
- Response response = resource.listDimensionValues("tags", null);
+ Response response = resource.listDimensionValues(RelationshipType.TAGS.toString(), null);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
Map results = (Map) JSONValue.parse(response.getEntity().toString());
- Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 4);
+ Assert.assertEquals(
+ Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 4);
List dimensions = (List) results.get(MetadataDiscoveryResource.RESULTS);
Assert.assertTrue(dimensions.contains("production"));
-
- response = resource.listDimensionValues("tags", MetadataTestContext.CLUSTER_ENTITY_NAME);
+ response = resource.listDimensionValues(RelationshipType.TAGS.toString(),
+ MetadataTestContext.CLUSTER_ENTITY_NAME);
results = (Map) JSONValue.parse(response.getEntity().toString());
Assert.assertEquals(Integer.parseInt(results.get(MetadataDiscoveryResource.TOTAL_SIZE).toString()), 0);
}
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testListInvalidDimensionType() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.listDimensionValues("INVALID", null);
+ }
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testListFeedDimensionType() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.listDimensionValues("Feed", null);
+ }
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testListInstanceDimensionType() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.listDimensionValues("FEED_INSTANCE", null);
+ }
+
+ @Test
+ public void testProcessGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.PROCESS_ENTITY.toString(),
+ MetadataTestContext.PROCESS_ENTITY_NAME);
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.PROCESS_ENTITY.toString());
+ Assert.assertEquals(results.get("name"), MetadataTestContext.PROCESS_ENTITY_NAME);
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 3);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 6);
+ Assert.assertNotNull(((Map) outVertices.get(0)).get("name"));
+ }
+
+ @Test
+ public void testFeedGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.FEED_ENTITY.toString(), "clicks-feed");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.FEED_ENTITY.toString());
+ Assert.assertEquals(results.get("name"), "clicks-feed");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 1);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 3);
+ Assert.assertNotNull(((Map) outVertices.get(0)).get("name"));
+ }
+
+ @Test
+ public void testClusterGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.CLUSTER_ENTITY.toString(),
+ "primary-cluster");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.CLUSTER_ENTITY.toString());
+ Assert.assertEquals(results.get("name"), "primary-cluster");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 10);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 2);
+ Assert.assertNotNull(((Map) outVertices.get(0)).get("name"));
+ }
+
+ @Test
+ public void testTagsGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.TAGS.toString(), "Critical");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.TAGS.toString());
+ Assert.assertEquals(results.get("name"), "Critical");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 2);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 0);
+ }
+
+ @Test
+ public void testPipelinesGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.PIPELINES.toString(), "testPipeline");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.PIPELINES.toString());
+ Assert.assertEquals(results.get("name"), "testPipeline");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 2);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 0);
+ }
+
+ @Test
+ public void testUserGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.USER.toString(), "falcon-user");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.USER.toString());
+ Assert.assertEquals(results.get("name"), "falcon-user");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 10);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 0);
+ }
+
+ @Test
+ public void testColoGetDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.COLO.toString(), "west-coast");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.get("type"), RelationshipType.COLO.toString());
+ Assert.assertEquals(results.get("name"), "west-coast");
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertEquals(inVertices.size(), 1);
+ Assert.assertNotNull(((Map) inVertices.get(0)).get("name"));
+ List outVertices = (List) results.get("outVertices");
+ Assert.assertEquals(outVertices.size(), 0);
+ }
+
+ @Test
+ public void testGetNonExistingDimensionRelations() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ Response response = resource.getDimensionRelations(RelationshipType.GROUPS.toString(), "west-coast");
+ Map results = (Map) JSONValue.parse(response.getEntity().toString());
+ Assert.assertEquals(results.size(), 0);
+ }
+
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testEntityRelationsInvalidType() {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.getDimensionRelations("INVALID", "name");
+ }
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testEntityRelationsFeedType() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.getDimensionRelations("FEED", "name");
+ }
+
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testEntityRelationsInstanceType() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.getDimensionRelations("FEED_INSTANCE", "name");
+ }
+ @Test(expectedExceptions = WebApplicationException.class)
+ public void testEntityRelationsNoName() throws Exception {
+ MetadataDiscoveryResource resource = new MetadataDiscoveryResource();
+ resource.getDimensionRelations(RelationshipType.TAGS.toString(), null);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index 31855bc..0943103 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -732,20 +732,65 @@ public class FalconCLIIT {
String metadataListCommand = FalconCLI.METADATA_CMD + " -" + FalconMetadataCLI.LIST_OPT + " -"
+ FalconMetadataCLI.TYPE_OPT + " ";
String clusterString = " -" + FalconMetadataCLI.CLUSTER_OPT + " " + clusterName;
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.CLUSTER_ENTITY.toString()));
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.PROCESS_ENTITY.toString()));
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.FEED_ENTITY.toString()));
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.PROCESS_ENTITY.toString()
+
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.CLUSTER_ENTITY.name()));
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.PROCESS_ENTITY.name()));
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.FEED_ENTITY.name()));
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.PROCESS_ENTITY.name()
+ clusterString));
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.FEED_ENTITY.toString()
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.FEED_ENTITY.name()
+ clusterString));
- Assert.assertEquals(0, executeWithURL(metadataListCommand + RelationshipType.CLUSTER_ENTITY.toString()
+ Assert.assertEquals(0,
+ executeWithURL(metadataListCommand + RelationshipType.CLUSTER_ENTITY.name()
+ clusterString));
Assert.assertEquals(-1, executeWithURL(metadataListCommand + "feed"));
Assert.assertEquals(-1, executeWithURL(metadataListCommand + "invalid"));
}
+ @Test
+ public void testMetadataRelationsCommands() throws Exception {
+ TestContext context = new TestContext();
+ Map<String, String> overlay = context.getUniqueOverlay();
+ submitTestFiles(context, overlay);
+
+ String processName = overlay.get("processName");
+ String feedName = overlay.get("outputFeedName");
+ String clusterName = overlay.get("cluster");
+
+ Assert.assertEquals(0,
+ executeWithURL(FalconCLI.ENTITY_CMD + " -" + FalconCLI.SCHEDULE_OPT + " -"
+ + FalconCLI.ENTITY_TYPE_OPT + " process -" + FalconCLI.ENTITY_NAME_OPT + " " + processName));
+
+ Assert.assertEquals(0,
+ executeWithURL(FalconCLI.ENTITY_CMD + " -" + FalconCLI.SCHEDULE_OPT + " -"
+ + FalconCLI.ENTITY_TYPE_OPT + " feed -" + FalconCLI.ENTITY_NAME_OPT + " " + feedName));
+
+ OozieTestUtils.waitForProcessWFtoStart(context);
+
+ String metadataRelationsCommand = FalconCLI.METADATA_CMD + " -" + FalconMetadataCLI.RELATIONS_OPT + " -"
+ + FalconMetadataCLI.TYPE_OPT + " ";
+
+ Assert.assertEquals(0,
+ executeWithURL(metadataRelationsCommand + RelationshipType.CLUSTER_ENTITY.name()
+ + " -" + FalconMetadataCLI.NAME_OPT + " " + clusterName));
+ Assert.assertEquals(0,
+ executeWithURL(metadataRelationsCommand + RelationshipType.PROCESS_ENTITY.name()
+ + " -" + FalconMetadataCLI.NAME_OPT + " " + processName));
+
+ Assert.assertEquals(-1, executeWithURL(metadataRelationsCommand + "feed -"
+ + FalconMetadataCLI.NAME_OPT + " " + clusterName));
+ Assert.assertEquals(-1, executeWithURL(metadataRelationsCommand + "invalid -"
+ + FalconMetadataCLI.NAME_OPT + " " + clusterName));
+ Assert.assertEquals(-1,
+ executeWithURL(metadataRelationsCommand + RelationshipType.CLUSTER_ENTITY.name()));
+ }
+
public void testContinue() throws Exception {
TestContext context = new TestContext();
Map<String, String> overlay = context.getUniqueOverlay();
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/fa0f6126/webapp/src/test/java/org/apache/falcon/resource/MetadataResourceJerseyIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/MetadataResourceJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/MetadataResourceJerseyIT.java
index 462bcdc..5249888 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/MetadataResourceJerseyIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/MetadataResourceJerseyIT.java
@@ -21,6 +21,7 @@ package org.apache.falcon.resource;
import com.sun.jersey.api.client.ClientResponse;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.resource.metadata.AbstractMetadataResource;
+import org.apache.falcon.util.DeploymentUtil;
import org.json.simple.JSONValue;
import org.testng.Assert;
import org.testng.annotations.BeforeClass;
@@ -60,7 +61,7 @@ public class MetadataResourceJerseyIT {
}
@Test
- public void testMetadataList() throws Exception {
+ public void testMetadataDiscoveryResourceList() throws Exception {
ClientResponse response = context.service
.path("api/metadata/cluster_entity/list")
@@ -91,7 +92,30 @@ public class MetadataResourceJerseyIT {
.get(ClientResponse.class);
Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
results = (Map) JSONValue.parse(response.getEntity(String.class));
- Assert.assertEquals(((Long)results.get(AbstractMetadataResource.TOTAL_SIZE)).longValue(), 0);
+ Assert.assertEquals(Integer.parseInt(results.get(AbstractMetadataResource.TOTAL_SIZE).toString()), 0);
+ }
+
+ @Test
+ public void testMetadataDiscoveryResourceRelations() throws Exception {
+ ClientResponse response = context.service
+ .path("api/metadata/process_entity/" + context.processName + "/relations")
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
+ Map results = (Map) JSONValue.parse(response.getEntity(String.class));
+ Assert.assertEquals(results.get("name"), context.processName);
+
+ response = context.service
+ .path("api/metadata/colo/" + DeploymentUtil.getCurrentColo() + "/relations")
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+ Assert.assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
+ results = (Map) JSONValue.parse(response.getEntity(String.class));
+ Assert.assertEquals(results.get("name"), DeploymentUtil.getCurrentColo());
+ List inVertices = (List) results.get("inVertices");
+ Assert.assertTrue(inVertices.size() >= 1);
}
private ThreadLocal<TestContext> contexts = new ThreadLocal<TestContext>();