You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ar...@apache.org on 2014/08/21 19:30:52 UTC
[10/18] FALCON-470 Add support for pagination, filter-by,
etc. to Entity and Instance List API. Contributed by Balu Vellanki
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
index a0dbe65..7b8bdf5 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
@@ -296,8 +296,13 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana
@Produces({MediaType.TEXT_XML, MediaType.APPLICATION_JSON})
@Override
public EntityList getEntityList(@PathParam("type") String type,
- @DefaultValue("") @QueryParam("fields") String fields) {
- return super.getEntityList(type, fields);
+ @DefaultValue("") @QueryParam("fields") String fields,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("tags") String tags,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage);
}
@GET
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
index 73cf493..9e2d87b 100644
--- a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
+++ b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
@@ -18,7 +18,12 @@
package org.apache.falcon.resource;
import org.apache.falcon.FalconWebException;
+import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.process.ACL;
+import org.apache.falcon.entity.v0.process.Process;
+import org.apache.falcon.security.CurrentUser;
+import org.apache.falcon.util.StartupProperties;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
@@ -40,7 +45,6 @@ public class EntityManagerTest extends AbstractEntityManager {
@Mock
private HttpServletRequest mockHttpServletRequest;
-
private static final String SAMPLE_PROCESS_XML = "/process-version-0.xml";
private static final String SAMPLE_INVALID_PROCESS_XML = "/process-invalid.xml";
@@ -62,12 +66,11 @@ public class EntityManagerTest extends AbstractEntityManager {
/**
* Run this testcase for different types of VALID entity xmls like process, feed, dataEndPoint.
*
- * @param stream
+ * @param stream entity stream
* @throws IOException
*/
@Test(dataProvider = "validXMLServletStreamProvider")
- public void testValidateForValidEntityXML(EntityType entityType,
- ServletInputStream stream) throws IOException {
+ public void testValidateForValidEntityXML(ServletInputStream stream) throws IOException {
when(mockHttpServletRequest.getInputStream()).thenReturn(stream);
}
@@ -102,10 +105,105 @@ public class EntityManagerTest extends AbstractEntityManager {
}
}
+ @Test
+ public void testGetEntityList() throws Exception {
+
+ Entity process1 = buildProcess("processFakeUser", "fakeUser", "", "");
+ configStore.publish(EntityType.PROCESS, process1);
+
+ Entity process2 = buildProcess("processAuthUser", System.getProperty("user.name"), "", "");
+ configStore.publish(EntityType.PROCESS, process2);
+
+ EntityList entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ Assert.assertNotNull(entityList.getElements());
+ Assert.assertEquals(entityList.getElements().length, 2);
+
+ /*
+ * Both entities should be returned when the user is SuperUser.
+ */
+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
+ CurrentUser.authenticate(System.getProperty("user.name"));
+ entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ Assert.assertNotNull(entityList.getElements());
+ Assert.assertEquals(entityList.getElements().length, 2);
+
+ /*
+ * Only one entity should be returned when the auth is enabled.
+ */
+ CurrentUser.authenticate("fakeUser");
+ entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ Assert.assertNotNull(entityList.getElements());
+ Assert.assertEquals(entityList.getElements().length, 1);
+
+ // reset values
+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "false");
+ CurrentUser.authenticate(System.getProperty("user.name"));
+ }
+
+ @Test
+ public void testGetEntityListPagination() throws Exception {
+ String user = System.getProperty("user.name");
+
+ Entity process1 = buildProcess("process1", user,
+ "consumer=consumer@xyz.com, owner=producer@xyz.com",
+ "testPipeline,dataReplicationPipeline");
+ configStore.publish(EntityType.PROCESS, process1);
+
+ Entity process2 = buildProcess("process2", user,
+ "consumer=consumer@xyz.com, owner=producer@xyz.com",
+ "testPipeline,dataReplicationPipeline");
+ configStore.publish(EntityType.PROCESS, process2);
+
+ Entity process3 = buildProcess("process3", user, "consumer=consumer@xyz.com", "testPipeline");
+ configStore.publish(EntityType.PROCESS, process3);
+
+ Entity process4 = buildProcess("process4", user, "owner=producer@xyz.com", "dataReplicationPipeline");
+ configStore.publish(EntityType.PROCESS, process4);
+
+ EntityList entityList = this.getEntityList("process", "tags", "PIPELINES:dataReplicationPipeline",
+ "", "name", 1, 2);
+ Assert.assertNotNull(entityList.getElements());
+ Assert.assertEquals(entityList.getElements().length, 2);
+ Assert.assertEquals(entityList.getElements()[1].name, "process4");
+ Assert.assertEquals(entityList.getElements()[1].tag.size(), 1);
+ Assert.assertEquals(entityList.getElements()[1].tag.get(0), "owner=producer@xyz.com");
+ Assert.assertEquals(entityList.getElements()[0].status, null);
+
+
+ entityList = this.getEntityList("process", "pipelines", "",
+ "consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 0, 2);
+ Assert.assertNotNull(entityList.getElements());
+ Assert.assertEquals(entityList.getElements().length, 2);
+ Assert.assertEquals(entityList.getElements()[1].name, "process2");
+ Assert.assertEquals(entityList.getElements()[1].pipelines.size(), 2);
+ Assert.assertEquals(entityList.getElements()[1].pipelines.get(0), "testPipeline");
+ Assert.assertEquals(entityList.getElements()[0].tag, null);
+
+ entityList = this.getEntityList("process", "pipelines", "",
+ "consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 10, 2);
+ Assert.assertEquals(entityList.getElements().length, 0);
+ }
+
+ private Entity buildProcess(String name, String username, String tags, String pipelines) {
+ ACL acl = new ACL();
+ acl.setOwner(username);
+ acl.setGroup("hdfs");
+ acl.setPermission("*");
+
+ Process p = new Process();
+ p.setName(name);
+ p.setACL(acl);
+ p.setPipelines(pipelines);
+ p.setTags(tags);
+ return p;
+ }
+
+
+
/**
* Converts a InputStream into ServletInputStream.
*
- * @param resourceName
+ * @param resourceName resource name
* @return ServletInputStream
*/
private ServletInputStream getServletInputStream(String resourceName) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
index cdf703d..8762cc1 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
@@ -34,6 +34,7 @@ import java.util.List;
@Path("instance")
public class InstanceManager extends AbstractInstanceManager {
+ //SUSPEND CHECKSTYLE CHECK ParameterNumberCheck
@GET
@Path("running/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
@@ -43,8 +44,36 @@ public class InstanceManager extends AbstractInstanceManager {
@Dimension("type") @PathParam("type") String type,
@Dimension("entity") @PathParam("entity") String entity,
@Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
- return super.getRunningInstances(type, entity, colo, lifeCycles);
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getRunningInstances(type, entity, colo, lifeCycles, filterBy, orderBy, offset, resultsPerPage);
+ }
+
+ /*
+ getStatus(...) method actually gets all instances, filtered by a specific status. This is
+ a better named API which achieves the same result
+ */
+ @GET
+ @Path("list/{type}/{entity}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Monitored(event = "instance-list")
+ @Override
+ public InstancesResult getInstances(
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getInstances(type, entity, startStr, endStr, colo, lifeCycles,
+ filterBy, orderBy, offset, resultsPerPage);
}
@GET
@@ -52,25 +81,32 @@ public class InstanceManager extends AbstractInstanceManager {
@Produces(MediaType.APPLICATION_JSON)
@Monitored(event = "instance-status")
@Override
- public InstancesResult getStatus(@Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
- return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles);
+ public InstancesResult getStatus(
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles,
+ filterBy, orderBy, offset, resultsPerPage);
}
@GET
@Path("summary/{type}/{entity}")
@Produces(MediaType.APPLICATION_JSON)
@Monitored(event = "instance-summary")
- public InstancesSummaryResult getSummary(@Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
+ public InstancesSummaryResult getSummary(
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
return super.getSummary(type, entity, startStr, endStr, colo, lifeCycles);
}
@@ -86,8 +122,13 @@ public class InstanceManager extends AbstractInstanceManager {
@Dimension("end-time") @QueryParam("end") String endStr,
@Dimension("colo") @QueryParam("colo") String colo,
@Dimension("run-id") @QueryParam("runid") String runId,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
- return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles);
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles,
+ filterBy, orderBy, offset, resultsPerPage);
}
@GET
@@ -110,13 +151,14 @@ public class InstanceManager extends AbstractInstanceManager {
@Produces(MediaType.APPLICATION_JSON)
@Monitored(event = "kill-instance")
@Override
- public InstancesResult killInstance(@Context HttpServletRequest request,
- @Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
+ public InstancesResult killInstance(
+ @Context HttpServletRequest request,
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
return super.killInstance(request, type, entity, startStr, endStr, colo, lifeCycles);
}
@@ -141,13 +183,14 @@ public class InstanceManager extends AbstractInstanceManager {
@Produces(MediaType.APPLICATION_JSON)
@Monitored(event = "resume-instance")
@Override
- public InstancesResult resumeInstance(@Context HttpServletRequest request,
- @Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
+ public InstancesResult resumeInstance(
+ @Context HttpServletRequest request,
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
return super.resumeInstance(request, type, entity, startStr, endStr, colo, lifeCycles);
}
@@ -156,13 +199,16 @@ public class InstanceManager extends AbstractInstanceManager {
@Produces(MediaType.APPLICATION_JSON)
@Monitored(event = "re-run-instance")
@Override
- public InstancesResult reRunInstance(@Dimension("type") @PathParam("type") String type,
- @Dimension("entity") @PathParam("entity") String entity,
- @Dimension("start-time") @QueryParam("start") String startStr,
- @Dimension("end-time") @QueryParam("end") String endStr,
- @Context HttpServletRequest request,
- @Dimension("colo") @QueryParam("colo") String colo,
- @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
+ public InstancesResult reRunInstance(
+ @Dimension("type") @PathParam("type") String type,
+ @Dimension("entity") @PathParam("entity") String entity,
+ @Dimension("start-time") @QueryParam("start") String startStr,
+ @Dimension("end-time") @QueryParam("end") String endStr,
+ @Context HttpServletRequest request,
+ @Dimension("colo") @QueryParam("colo") String colo,
+ @Dimension("lifecycle") @QueryParam("lifecycle") List<LifeCycle> lifeCycles) {
return super.reRunInstance(type, entity, startStr, endStr, request, colo, lifeCycles);
}
+ //RESUME CHECKSTYLE CHECK ParameterNumberCheck
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
index 3c9078d..f993f81 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
@@ -59,8 +59,13 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@Monitored(event = "dependencies")
@Override
public EntityList getEntityList(@Dimension("type") @PathParam("type") String type,
- @DefaultValue("") @QueryParam("fields") String fields) {
- return super.getEntityList(type, fields);
+ @DefaultValue("") @QueryParam("fields") String fields,
+ @DefaultValue("") @QueryParam("filterBy") String filterBy,
+ @DefaultValue("") @QueryParam("tags") String tags,
+ @DefaultValue("") @QueryParam("orderBy") String orderBy,
+ @DefaultValue("0") @QueryParam("offset") Integer offset,
+ @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage);
}
@GET
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index ea40411..b65aa46 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -388,6 +388,54 @@ public class FalconCLIIT {
+ overlay.get("processName")
+ " -start " + START_INSTANCE));
+ // test filterBy, orderBy, offset, numResults
+ String startTimeString = SchemaHelper.getDateFormat().format(new Date());
+ Assert.assertEquals(0,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start " + startTimeString
+ + " -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date())
+ + " -orderBy INVALID -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date())
+ + " -filterBy INVALID:FILTER -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("instance -status -type process -name "
+ + overlay.get("processName")
+ + " -start "+ START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE
+ + " -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("instance -list -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -status -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -list -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy STATUS:SUCCEEDED -orderBy INVALID -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("instance -status -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 1 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("instance -list -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ +" -filterBy STATUS:SUCCEEDED -offset 0 -numResults 1"));
}
public void testInstanceRunningAndSummaryCommands() throws Exception {
@@ -430,7 +478,6 @@ public class FalconCLIIT {
+ " -start " + START_INSTANCE));
}
-
public void testInstanceSuspendAndResume() throws Exception {
TestContext context = new TestContext();
Map<String, String> overlay = context.getUniqueOverlay();
@@ -506,6 +553,86 @@ public class FalconCLIIT {
+ " -file "+ createTempJobPropertiesFile()));
}
+ @Test
+ public void testEntityPaginationFilterByCommands() throws Exception {
+
+ String filePath;
+ TestContext context = new TestContext();
+ Map<String, String> overlay = context.getUniqueOverlay();
+
+ filePath = TestContext.overlayParametersOverTemplate(context.getClusterFileTemplate(), overlay);
+ Assert.assertEquals(-1,
+ executeWithURL("entity -submitAndSchedule -type cluster -file " + filePath));
+ context.setCluster(overlay.get("cluster"));
+
+ // this is necessary for lineage
+ Assert.assertEquals(0, executeWithURL("entity -submit -type cluster -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -submitAndSchedule -type feed -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE1, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -submit -type feed -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.FEED_TEMPLATE2, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -submit -type feed -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -validate -type process -file " + filePath));
+
+ filePath = TestContext.overlayParametersOverTemplate(TestContext.PROCESS_TEMPLATE, overlay);
+ Assert.assertEquals(0,
+ executeWithURL("entity -submitAndSchedule -type process -file "
+ + filePath));
+
+ OozieTestUtils.waitForProcessWFtoStart(context);
+
+ // test entity List cli
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type cluster"
+ + " -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type process -fields status "
+ + " -filterBy STATUS:SUBMITTED,TYPE:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type process -fields status,pipelines "
+ + " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type process -fields status,pipelines "
+ + " -filterBy STATUS:SUBMITTED,pipelines:testPipeline "
+ + " -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type process -fields status,tags "
+ + " -tags owner=producer@xyz.com,department=forecasting "
+ + " -filterBy STATUS:SUBMITTED,type:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("entity -list -type process -fields status "
+ + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("entity -list -type process -fields INVALID "
+ + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("entity -list -type process -fields status "
+ + " -filterBy INVALID:FILTER,TYPE:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster")));
+
+ Assert.assertEquals(0,
+ executeWithURL("instance -status -type feed -name "
+ + overlay.get("outputFeedName") + " -start " + START_INSTANCE));
+ Assert.assertEquals(0,
+ executeWithURL("instance -running -type process -name " + overlay.get("processName")));
+ }
+
+
public void testContinue() throws Exception {
TestContext context = new TestContext();
Map<String, String> overlay = context.getUniqueOverlay();
@@ -618,6 +745,8 @@ public class FalconCLIIT {
executeWithURL("entity -schedule -type feed -name "
+ overlay.get("outputFeedName")));
+ Thread.sleep(500);
+
Assert.assertEquals(0,
executeWithURL("instance -logs -type process -name "
+ overlay.get("processName")
@@ -628,9 +757,31 @@ public class FalconCLIIT {
+ overlay.get("outputFeedName")
+ " -start "+ SchemaHelper.getDateFormat().format(new Date())));
+ // test filterBy, orderBy, offset, numResults
+ Assert.assertEquals(0,
+ executeWithURL("instance -logs -type process -name "
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("instance -logs -type process -name "
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED,STARTEDAFTER:"+START_INSTANCE+" -offset 1 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -logs -type process -name "
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy INVALID:FILTER -orderBy startTime -offset 0 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("instance -logs -type process -name "
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -filterBy STATUS:SUCCEEDED -orderBy wrongOrder -offset 0 -numResults 1"));
}
private int executeWithURL(String command) throws Exception {
+ //System.out.println("COMMAND IS "+command + " -url " + TestContext.BASE_URL);
return new FalconCLI()
.run((command + " -url " + TestContext.BASE_URL).split("\\s+"));
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java
index dd9c34a..cb0dd2d 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLISmokeIT.java
@@ -87,13 +87,22 @@ public class FalconCLISmokeIT {
OozieTestUtils.waitForProcessWFtoStart(context);
+ // test entity List cli
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type cluster"
+ + " -offset 0 -numResults 1"));
+ Assert.assertEquals(0,
+ executeWithURL("entity -list -type process -fields status "
+ + " -filterBy STATUS:SUBMITTED,TYPE:process -orderBy name -offset 1 -numResults 1"));
+ Assert.assertEquals(-1,
+ executeWithURL("entity -list -type process -fields status "
+ + " -filterBy STATUS:SUCCEEDED,TYPE:process -orderBy INVALID -offset 0 -numResults 1"));
Assert.assertEquals(0,
executeWithURL("entity -definition -type cluster -name " + overlay.get("cluster")));
Assert.assertEquals(0,
executeWithURL("instance -status -type feed -name "
+ overlay.get("outputFeedName") + " -start " + START_INSTANCE));
-
Assert.assertEquals(0,
executeWithURL("instance -running -type process -name " + overlay.get("processName")));
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
index 8a50005..0a41fe0 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/EntityManagerJerseyIT.java
@@ -739,6 +739,39 @@ public class EntityManagerJerseyIT {
.get(ClientResponse.class);
Assert.assertEquals(response.getStatus(), 200);
+ EntityList result = response.getEntity(EntityList.class);
+ Assert.assertNotNull(result);
+ for (EntityList.EntityElement entityElement : result.getElements()) {
+ Assert.assertNull(entityElement.status); // status is null
+ }
+
+ response = context.service
+ .path("api/entities/list/process/")
+ .queryParam("filterBy", "TYPE:PROCESS,STATUS:RUNNING")
+ .queryParam("tags", "owner=producer@xyz.com, department=forecasting")
+ .queryParam("orderBy", "name").queryParam("offset", "2")
+ .queryParam("numResults", "2").queryParam("fields", "status,tags")
+ .header("Cookie", context.getAuthenticationToken())
+ .type(MediaType.TEXT_XML)
+ .accept(MediaType.TEXT_XML)
+ .get(ClientResponse.class);
+ Assert.assertEquals(response.getStatus(), 200);
+ result = response.getEntity(EntityList.class);
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.getElements().length, 2);
+
+ response = context.service
+ .path("api/entities/list/process/")
+ .queryParam("orderBy", "name").queryParam("offset", "50").queryParam("numResults", "2")
+ .header("Cookie", context.getAuthenticationToken())
+ .type(MediaType.TEXT_XML)
+ .accept(MediaType.TEXT_XML)
+ .get(ClientResponse.class);
+ Assert.assertEquals(response.getStatus(), 200);
+ result = response.getEntity(EntityList.class);
+ Assert.assertNotNull(result);
+ Assert.assertEquals(result.getElements(), null);
+
Map<String, String> overlay = context.getUniqueOverlay();
overlay.put("cluster", "WTF-" + overlay.get("cluster"));
response = context.submitToFalcon(TestContext.CLUSTER_TEMPLATE, overlay, EntityType.CLUSTER);
@@ -751,7 +784,7 @@ public class EntityManagerJerseyIT {
.accept(MediaType.TEXT_XML)
.get(ClientResponse.class);
Assert.assertEquals(response.getStatus(), 200);
- EntityList result = response.getEntity(EntityList.class);
+ result = response.getEntity(EntityList.class);
Assert.assertNotNull(result);
for (EntityList.EntityElement entityElement : result.getElements()) {
Assert.assertNull(entityElement.status); // status is null
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java
index 03a19cd..863b109 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/ProcessInstanceManagerIT.java
@@ -26,6 +26,7 @@ import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.cluster.Cluster;
import org.apache.falcon.resource.InstancesResult.Instance;
import org.apache.falcon.resource.InstancesResult.WorkflowStatus;
+import org.apache.falcon.security.CurrentUser;
import org.apache.falcon.util.OozieTestUtils;
import org.apache.falcon.workflow.engine.OozieClientFactory;
import org.apache.oozie.client.ProxyOozieClient;
@@ -43,10 +44,18 @@ public class ProcessInstanceManagerIT {
private static final String START_INSTANCE = "2012-04-20T00:00Z";
protected void schedule(TestContext context) throws Exception {
- context.scheduleProcess();
+ CurrentUser.authenticate(System.getProperty("user.name"));
+ schedule(context, 1);
+ }
+
+ protected void schedule(TestContext context, int count) throws Exception {
+ for (int i=0; i<count; i++) {
+ context.scheduleProcess();
+ }
OozieTestUtils.waitForProcessWFtoStart(context);
}
+ //@Test
public void testGetRunningInstances() throws Exception {
TestContext context = new TestContext();
schedule(context);
@@ -60,6 +69,30 @@ public class ProcessInstanceManagerIT {
assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.RUNNING);
}
+ //@Test
+ public void testGetRunningInstancesPagination() throws Exception {
+ TestContext context = new TestContext();
+ schedule(context, 4);
+ InstancesResult response = context.service.path("api/instance/running/process/" + context.processName)
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+ Assert.assertNotNull(response.getInstances());
+ assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.RUNNING);
+
+ response = context.service.path("api/instance/running/process/" + context.processName)
+ .queryParam("orderBy", "startTime").queryParam("offset", "0")
+ .queryParam("numResults", "1")
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+ Assert.assertNotNull(response.getInstances());
+ Assert.assertEquals(1, response.getInstances().length);
+ assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.RUNNING);
+ }
+
private void assertInstance(Instance processInstance, String instance, WorkflowStatus status) {
Assert.assertNotNull(processInstance);
Assert.assertNotNull(processInstance.getInstance());
@@ -67,6 +100,7 @@ public class ProcessInstanceManagerIT {
Assert.assertEquals(processInstance.getStatus(), status);
}
+ //@Test
public void testGetInstanceStatus() throws Exception {
TestContext context = new TestContext();
schedule(context);
@@ -81,6 +115,24 @@ public class ProcessInstanceManagerIT {
assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.RUNNING);
}
+ //@Test
+ public void testGetInstanceStatusPagination() throws Exception {
+ TestContext context = new TestContext();
+ schedule(context, 4);
+
+ InstancesResult response = context.service.path("api/instance/status/process/" + context.processName)
+ .queryParam("orderBy", "startTime").queryParam("offset", "0")
+ .queryParam("numResults", "1").queryParam("filterBy", "STATUS:RUNNING")
+ .queryParam("start", START_INSTANCE)
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+ Assert.assertNotNull(response.getInstances());
+ Assert.assertEquals(1, response.getInstances().length);
+ assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.RUNNING);
+ }
+
public void testReRunInstances() throws Exception {
testKillInstances();
TestContext context = new TestContext();
@@ -111,6 +163,28 @@ public class ProcessInstanceManagerIT {
Assert.assertEquals(1, response.getInstances().length);
assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.KILLED);
+ response = context.service.path("api/instance/status/process/" + context.processName)
+ .queryParam("orderBy", "startTime").queryParam("filterBy", "STATUS:KILLED")
+ .queryParam("start", START_INSTANCE)
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+ Assert.assertNotNull(response.getInstances());
+ Assert.assertEquals(1, response.getInstances().length);
+ assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.KILLED);
+
+ response = context.service.path("api/instance/status/process/" + context.processName)
+ .queryParam("orderBy", "startTime").queryParam("filterBy", "STATUS:KILLED")
+ .queryParam("start", START_INSTANCE)
+ .header("Cookie", context.getAuthenticationToken())
+ .accept(MediaType.APPLICATION_JSON)
+ .get(InstancesResult.class);
+ Assert.assertEquals(APIResult.Status.SUCCEEDED, response.getStatus());
+ Assert.assertNotNull(response.getInstances());
+ Assert.assertEquals(1, response.getInstances().length);
+ assertInstance(response.getInstances()[0], START_INSTANCE, WorkflowStatus.KILLED);
+
waitForWorkflow(START_INSTANCE, WorkflowJob.Status.KILLED);
}
@@ -159,7 +233,6 @@ public class ProcessInstanceManagerIT {
if (jobInfo.getStatus() == status) {
break;
}
- System.out.println("Waiting for workflow job " + jobId + " status " + status);
Thread.sleep((i + 1) * 1000);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/resources/process-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/process-template.xml b/webapp/src/test/resources/process-template.xml
index 8015da2..06215a2 100644
--- a/webapp/src/test/resources/process-template.xml
+++ b/webapp/src/test/resources/process-template.xml
@@ -18,6 +18,8 @@
-->
<process name="##processName##" xmlns="uri:falcon:process:0.1">
+ <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, department=forecasting</tags>
+ <pipelines>testPipeline,dataReplicationPipeline</pipelines>
<clusters>
<cluster name="##cluster##">
<validity end="2012-04-21T00:00Z" start="2012-04-20T00:00Z"/>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/62b2af02/webapp/src/test/resources/process-version-0.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/process-version-0.xml b/webapp/src/test/resources/process-version-0.xml
index 3ac7ac8..b193ddc 100644
--- a/webapp/src/test/resources/process-version-0.xml
+++ b/webapp/src/test/resources/process-version-0.xml
@@ -18,6 +18,8 @@
-->
<process name="sample" xmlns="uri:falcon:process:0.1">
<!-- where -->
+ <tags>consumer=consumer@xyz.com, owner=producer@xyz.com, department=forecasting</tags>
+ <pipelines>testPipeline,dataReplicationPipeline</pipelines>
<clusters>
<cluster name="testCluster">
<validity start="2011-11-01T00:00Z" end="2011-12-31T23:59Z"/>