You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2016/01/04 13:47:31 UTC
falcon git commit: FALCON-1697 Stabilization of scenarios which are
based on instances lifecycle. Contributed by Paul Isaychuk.
Repository: falcon
Updated Branches:
refs/heads/master 6d0119b7b -> 1e3dcb73f
FALCON-1697 Stabilization of scenarios which are based on instances lifecycle. Contributed by Paul Isaychuk.
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/1e3dcb73
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/1e3dcb73
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/1e3dcb73
Branch: refs/heads/master
Commit: 1e3dcb73fd30b93332ae81a075a7cecbc3dd3508
Parents: 6d0119b
Author: Ajay Yadava <aj...@gmail.com>
Authored: Mon Jan 4 18:16:47 2016 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Mon Jan 4 18:16:47 2016 +0530
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 2 +
.../regression/core/util/InstanceUtil.java | 8 +-
.../regression/ProcessInstanceStatusTest.java | 2 +
.../regression/ProcessInstanceSuspendTest.java | 7 +-
.../lineage/ListFeedInstancesTest.java | 159 ++++++++++---------
.../lineage/ListProcessInstancesTest.java | 40 +++--
.../regression/searchUI/EntityPageTest.java | 8 +-
7 files changed, 124 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 19dc351..c083c2c 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -5,6 +5,8 @@ Trunk (Unreleased)
INCOMPATIBLE CHANGES
NEW FEATURES
+ FALCON-1697 Stabilization of scenarios which are based on instances lifecycle(Paul Isaychuk via Ajay Yadava)
+
FALCON-1700 Add new test cases to HiveDRTest(Paul Isaychuk & Murali Ramasami via Ajay Yadava)
FALCON-1689 NoOutputProcessTest fails due to scheme missing in workflow.xml(Pragya Mittal)
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
index 3d05ae9..b07e275 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/InstanceUtil.java
@@ -73,8 +73,8 @@ public final class InstanceUtil {
public static final int INSTANCES_CREATED_TIMEOUT = OSUtil.IS_WINDOWS ? 20 : 10;
private static final Logger LOGGER = Logger.getLogger(InstanceUtil.class);
- private static final EnumSet<Status> RUNNING_PREP_SUCCEEDED = EnumSet.of(Status.RUNNING,
- Status.PREP, Status.SUCCEEDED);
+ private static final EnumSet<Status> LIVE_STATUS = EnumSet.of(Status.RUNNING,
+ Status.PREP, Status.SUCCEEDED, Status.SUSPENDED);
private InstanceUtil() {
throw new AssertionError("Instantiating utility class...");
@@ -573,7 +573,7 @@ public final class InstanceUtil {
for (String bundleId : bundleIds) {
LOGGER.info(String.format("Using bundle %s", bundleId));
final Status status = client.getBundleJobInfo(bundleId).getStatus();
- Assert.assertTrue(RUNNING_PREP_SUCCEEDED.contains(status),
+ Assert.assertTrue(LIVE_STATUS.contains(status),
String.format("Bundle job %s is should be prep/running but is %s", bundleId, status));
OozieUtil.waitForCoordinatorJobCreation(client, bundleId);
List<CoordinatorJob> coords = client.getBundleJobInfo(bundleId).getCoordinators();
@@ -607,7 +607,7 @@ public final class InstanceUtil {
CoordinatorJob coordinatorJob = client.getCoordJobInfo(coordId);
final Status coordinatorStatus = coordinatorJob.getStatus();
if (expectedStatus != CoordinatorAction.Status.TIMEDOUT){
- Assert.assertTrue(RUNNING_PREP_SUCCEEDED.contains(coordinatorStatus),
+ Assert.assertTrue(LIVE_STATUS.contains(coordinatorStatus),
String.format("Coordinator %s should be running/prep but is %s.", coordId, coordinatorStatus));
}
List<CoordinatorAction> coordinatorActions = coordinatorJob.getActions();
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
index 6493133..14ecfe4 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceStatusTest.java
@@ -339,6 +339,8 @@ public class ProcessInstanceStatusTest extends BaseTestClass {
Status.RUNNING, EntityType.PROCESS, 5);
AssertUtil.assertSucceeded(prism.getProcessHelper().suspend(process));
AssertUtil.checkStatus(clusterOC, EntityType.PROCESS, process, Job.Status.SUSPENDED);
+ InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 5, Status.SUSPENDED, EntityType.PROCESS, 3);
+ TimeUtil.sleepSeconds(TIMEOUT);
AssertUtil.assertSucceeded(prism.getProcessHelper().resume(process));
TimeUtil.sleepSeconds(TIMEOUT);
InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 5,
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
index f673314..4a27a0a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/ProcessInstanceSuspendTest.java
@@ -63,8 +63,9 @@ public class ProcessInstanceSuspendTest extends BaseTestClass {
bundles[0] = new Bundle(bundles[0], cluster);
bundles[0].generateUniqueBundle(this);
bundles[0].setInputFeedDataPath(feedInputPath);
- bundles[0].setProcessWorkflow(aggregateWorkflowDir);
+ bundles[0].setInputFeedPeriodicity(5, TimeUnit.minutes);
bundles[0].setProcessPeriodicity(5, TimeUnit.minutes);
+ bundles[0].setProcessWorkflow(aggregateWorkflowDir);
bundles[0].setOutputFeedLocationData(feedOutputPath);
processName = bundles[0].getProcessName();
}
@@ -171,7 +172,9 @@ public class ProcessInstanceSuspendTest extends BaseTestClass {
bundles[0].setProcessConcurrency(3);
bundles[0].submitFeedsScheduleProcess(prism);
InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 0);
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 0);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 1);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 2);
InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3,
CoordinatorAction.Status.RUNNING, EntityType.PROCESS, 5);
InstancesResult result = prism.getProcessHelper().getProcessInstanceStatus(processName,
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
index 7ad4c8e..17725ae 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListFeedInstancesTest.java
@@ -23,7 +23,6 @@ import org.apache.falcon.entity.v0.feed.ActionType;
import org.apache.falcon.entity.v0.feed.ClusterType;
import org.apache.falcon.regression.Entities.FeedMerlin;
import org.apache.falcon.regression.core.bundle.Bundle;
-import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
@@ -42,8 +41,9 @@ import org.apache.oozie.client.CoordinatorJob;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
import org.testng.Assert;
-import org.testng.annotations.AfterClass;
+import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import org.testng.asserts.SoftAssert;
@@ -55,12 +55,13 @@ import java.util.Date;
import java.util.List;
/**
- * Testing the list instances api for feed.
+ * Testing the list instances api for feed. Testing is based on initial scenario and sets of
+ * expected instance statuses which are being compared with actual result of -list request
+ * with different parameters in different order, variation, etc.
*/
@Test(groups = "embedded")
public class ListFeedInstancesTest extends BaseTestClass {
private static final Logger LOGGER = Logger.getLogger(ListFeedInstancesTest.class);
- private ColoHelper cluster2 = servers.get(1);
private OozieClient cluster2OC = serverOC.get(1);
private String baseTestHDFSDir = cleanAndGetTestDir();
private String aggregateWorkflowDir = baseTestHDFSDir + "/aggregator";
@@ -68,7 +69,8 @@ public class ListFeedInstancesTest extends BaseTestClass {
private String feedDataLocation = sourcePath + MINUTE_DATE_PATTERN;
private String targetPath = baseTestHDFSDir + "/target";
private String targetDataLocation = targetPath + MINUTE_DATE_PATTERN;
- private String startTime, endTime;
+ private final String startTime = "2010-01-02T00:00Z";
+ private final String endTime = "2010-01-02T00:57Z";
private String feedName;
@BeforeClass(alwaysRun = true)
@@ -76,23 +78,21 @@ public class ListFeedInstancesTest extends BaseTestClass {
throws IOException, OozieClientException, JAXBException, AuthenticationException,
URISyntaxException, InterruptedException {
uploadDirToClusters(aggregateWorkflowDir, OSUtil.RESOURCES_OOZIE);
- startTime = TimeUtil.getTimeWrtSystemTime(-55);
- endTime = TimeUtil.getTimeWrtSystemTime(3);
- LOGGER.info("Time range is between : " + startTime + " and " + endTime);
Bundle bundle = BundleUtil.readELBundle();
for (int i = 0; i < 2; i++) {
bundles[i] = new Bundle(bundle, servers.get(i));
bundles[i].generateUniqueBundle(this);
}
- prepareScenario();
}
/*
* Prepares running feed with instances ordered (desc): 1 waiting, 1 running, 1 suspended,
- * 3 waiting and 6 killed. Testing is based on expected instances statuses.
+ * 5 waiting, 2 killed, 2 waiting. Testing is based on expected sets of instance statuses.
+ * Variety of instance statuses increases accuracy of testing.
*/
+ @BeforeMethod(alwaysRun = true)
private void prepareScenario() throws AuthenticationException, IOException, URISyntaxException,
- JAXBException, OozieClientException, InterruptedException {
+ JAXBException, OozieClientException, InterruptedException {
bundles[0].setInputFeedPeriodicity(5, Frequency.TimeUnit.minutes);
bundles[0].setInputFeedDataPath(feedDataLocation);
String feed = bundles[0].getInputFeedFromBundle();
@@ -127,23 +127,23 @@ public class ListFeedInstancesTest extends BaseTestClass {
InstanceUtil.waitTillInstanceReachState(cluster2OC, feedName, 12,
CoordinatorAction.Status.WAITING, EntityType.FEED);
- //retrieve specific instances to rule them directly
+ //retrieve instances to rule them directly
List<CoordinatorAction> actions = getReplicationInstances(cluster2OC, feedName);
LOGGER.info(actions);
Assert.assertNotNull(actions, "Required coordinator not found.");
Assert.assertEquals(actions.size(), 12, "Unexpected number of actions.");
- //killing first 6 instances
+ //killing the 3d and the 4th instances
String range;
InstancesResult r;
- for (int i = 0; i < 6; i++) {
+ for (int i = 2; i <= 3; i++) {
HadoopUtil.createFolders(serverFS.get(0), "", Arrays.asList(actions.get(i)
.getMissingDependencies().split("#")));
//only running instance can be killed, so we should make it running and then kill it
InstanceUtil.waitTillInstanceReachState(cluster2OC, feedName, 1,
CoordinatorAction.Status.RUNNING, EntityType.FEED, 3);
- range = "?start=" + TimeUtil.addMinsToTime(startTime, i * 5 - 1)
- + "&end=" + TimeUtil.addMinsToTime(startTime, i * 5 + 1);
+ range = "?start=" + TimeUtil.addMinsToTime(startTime, i * 5 - 2)
+ + "&end=" + TimeUtil.addMinsToTime(startTime, i * 5 + 2);
r = prism.getFeedHelper().getProcessInstanceKill(feedName, range);
InstanceUtil.validateResponse(r, 1, 0, 0, 0, 1);
}
@@ -166,11 +166,17 @@ public class ListFeedInstancesTest extends BaseTestClass {
//check that the scenario works as expected.
r = prism.getFeedHelper().getProcessInstanceStatus(feedName,
"?start=" + startTime + "&numResults=12");
- InstanceUtil.validateResponse(r, 12, 1, 1, 4, 6);
+ InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
+ }
+
+ @AfterMethod(alwaysRun = true)
+ public void tearDown() throws IOException {
+ removeTestClassEntities();
+ HadoopUtil.deleteDirIfExists(sourcePath, serverFS.get(0));
}
/*
- * Retrieves replication coordinator instances.
+ * Retrieves replication coordinator actions (replication instances).
* @param client target oozie client
* @param fName feed name
*/
@@ -204,7 +210,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
throws URISyntaxException, OozieClientException, JAXBException, AuthenticationException,
IOException, InterruptedException {
SoftAssert softAssert = new SoftAssert();
- //orderBy start time
+ //orderBy start time, check an order
InstancesResult r = prism.getFeedHelper().listInstances(feedName,
"orderBy=startTime&sortOrder=desc", null);
InstancesResult.Instance[] instances = r.getInstances();
@@ -217,10 +223,10 @@ public class ListFeedInstancesTest extends BaseTestClass {
previousDate = (Date) current.clone();
}
}
- //orderBy status
+ //orderBy status, check an order
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc", null);
- InstanceUtil.validateResponse(r, 12, 1, 1, 4, 6);
+ InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
instances = r.getInstances();
InstancesResult.WorkflowStatus previousStatus = InstancesResult.WorkflowStatus.WAITING;
for (InstancesResult.Instance instance : instances) {
@@ -229,7 +235,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
"Wrong order. Compared " + current + " and " + previousStatus + " statuses.");
previousStatus = current;
}
- //sort by endTime
+ //sort by endTime, check an order
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12&orderBy=endTime&sortOrder=desc", null);
instances = r.getInstances();
@@ -246,7 +252,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
}
/**
- * Test the list feed instance api using start/end parameters. Check instances number.
+ * List instances through api using start/end parameters. Check a number of instances.
*/
@Test
public void testFeedStartEnd()
@@ -255,60 +261,64 @@ public class ListFeedInstancesTest extends BaseTestClass {
//actual start/end values.
InstancesResult r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&end=" + endTime, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
- //without params, the default start/end should be applied.
+ //without params, the default start/end should be applied. End is set to now,
+ // start is set to end - (10 * entityFrequency)
r = prism.getFeedHelper().listInstances(feedName, null, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
- //increasing a -start, the -end stays the same.
+ //increasing -start, -end stays the same.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(startTime, 6)
+ "&end=" + TimeUtil.addMinsToTime(endTime, -5), null);
- InstanceUtil.validateResponse(r, 9, 1, 1, 3, 4);
+ InstanceUtil.validateResponse(r, 9, 1, 1, 5, 2);
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(startTime, 11)
+ "&end=" + TimeUtil.addMinsToTime(endTime, -5), null);
- InstanceUtil.validateResponse(r, 8, 1, 1, 3, 3);
+ InstanceUtil.validateResponse(r, 8, 1, 1, 5, 1);
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(startTime, 16)
+ "&end=" + TimeUtil.addMinsToTime(endTime, -5), null);
- InstanceUtil.validateResponse(r, 7, 1, 1, 3, 2);
+ InstanceUtil.validateResponse(r, 7, 1, 1, 5, 0);
- //one instance between start/end, use instances with different statuses.
+ //one instance between start/end, killed instance
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(startTime, 12)
+ "&end=" + TimeUtil.addMinsToTime(startTime, 16), null);
InstanceUtil.validateResponse(r, 1, 0, 0, 0, 1);
+
+ //one instance between start/end, waiting instance
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(endTime, -5) + "&end=" + endTime, null);
InstanceUtil.validateResponse(r, 1, 0, 0, 1, 0);
- //only start, actual feed startTime, should get 1-10 instances(end is automatically set to freq*10).
+ //only start, actual feed startTime, should get 1-10 instances(end is automatically set to now).
r = prism.getFeedHelper().listInstances(feedName, "start=" + startTime, null);
- InstanceUtil.validateResponse(r, 10, 0, 1, 3, 6);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
//only start, greater then the actual startTime.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + TimeUtil.addMinsToTime(startTime, 16), null);
- InstanceUtil.validateResponse(r, 8, 1, 1, 4, 2);
+ InstanceUtil.validateResponse(r, 8, 1, 1, 6, 0);
//only end, 1 instance is expected
r = prism.getFeedHelper().listInstances(feedName,
"end=" + TimeUtil.addMinsToTime(startTime, 4), null);
- InstanceUtil.validateResponse(r, 1, 0, 0, 0, 1);
+ InstanceUtil.validateResponse(r, 1, 0, 0, 1, 0);
- //only the end, 10 the most recent instances are expected
+ //only end, actual value, 10 the most recent instances are expected
r = prism.getFeedHelper().listInstances(feedName, "end=" + endTime, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
- //only the end
+ //only end, first 6 instances
r = prism.getFeedHelper().listInstances(feedName,
"end=" + TimeUtil.addMinsToTime(endTime, -31), null);
- InstanceUtil.validateResponse(r, 6, 0, 0, 0, 6);
+ InstanceUtil.validateResponse(r, 6, 0, 0, 4, 2);
+ //only end, first 8 instances
r = prism.getFeedHelper().listInstances(feedName,
"end=" + TimeUtil.addMinsToTime(endTime, -21), null);
- InstanceUtil.validateResponse(r, 8, 0, 0, 2, 6);
+ InstanceUtil.validateResponse(r, 8, 0, 0, 6, 2);
}
/**
@@ -320,7 +330,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
throws URISyntaxException, IOException, AuthenticationException, InterruptedException {
//check the default value of the numResults param. Expecting 10 instances.
InstancesResult r = prism.getFeedHelper().listInstances(feedName, null, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
//changing a value to 6. 6 instances are expected
r = prism.getFeedHelper().listInstances(feedName, "numResults=6", null);
@@ -328,17 +338,17 @@ public class ListFeedInstancesTest extends BaseTestClass {
//use a start option without a numResults parameter. 10 instances are expected
r = prism.getFeedHelper().listInstances(feedName, "start=" + startTime, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
//use a start option with a numResults value which is smaller then the default.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=8", null);
- InstanceUtil.validateResponse(r, 8, 1, 1, 4, 2);
+ InstanceUtil.validateResponse(r, 8, 1, 1, 6, 0);
//use a start option with a numResults value greater then the default.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12", null);
- InstanceUtil.validateResponse(r, 12, 1, 1, 4, 6);
+ InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
//get all instances
InstancesResult.Instance[] allInstances = r.getInstances();
@@ -347,7 +357,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
int offset = 3;
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&offset=" + offset + "&numResults=12", null);
- InstanceUtil.validateResponse(r, 9, 0, 0, 3, 6);
+ InstanceUtil.validateResponse(r, 9, 0, 0, 7, 2);
//check that expected instances were retrieved
InstancesResult.Instance[] instances = r.getInstances();
@@ -359,7 +369,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
offset = 6;
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&offset=" + offset + "&numResults=6", null);
- InstanceUtil.validateResponse(r, 6, 0, 0, 0, 6);
+ InstanceUtil.validateResponse(r, 6, 0, 0, 4, 2);
//check that expected instances are present in response
instances = r.getInstances();
@@ -370,7 +380,7 @@ public class ListFeedInstancesTest extends BaseTestClass {
}
/**
- * Test the list feed instances api using filterBy parameter.
+ * List feed instances with filterBy parameter.
*/
@Test
public void testFeedFilterBy()
@@ -380,18 +390,20 @@ public class ListFeedInstancesTest extends BaseTestClass {
InstancesResult r = prism.getFeedHelper().listInstances(feedName,
"filterBy=STATUS:RUNNING", null);
InstanceUtil.validateResponse(r, 1, 1, 0, 0, 0);
+ //end is set to now (actual end), start is set to (end - (10 * entityFrequency))
+ //filtered range is from 3rd till 12th instance
r = prism.getFeedHelper().listInstances(feedName, "filterBy=STATUS:WAITING", null);
- InstanceUtil.validateResponse(r, 4, 0, 0, 4, 0);
+ InstanceUtil.validateResponse(r, 6, 0, 0, 6, 0);
//get all instances.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&numResults=12", null);
- InstanceUtil.validateResponse(r, 12, 1, 1, 4, 6);
+ InstanceUtil.validateResponse(r, 12, 1, 1, 8, 2);
//use different statuses, filterBy among all instances.
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&filterBy=STATUS:KILLED", null);
- InstanceUtil.validateResponse(r, 6, 0, 0, 0, 6);
+ InstanceUtil.validateResponse(r, 2, 0, 0, 0, 2);
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&filterBy=STATUS:SUSPENDED", null);
InstanceUtil.validateResponse(r, 1, 0, 1, 0, 0);
@@ -400,24 +412,17 @@ public class ListFeedInstancesTest extends BaseTestClass {
InstanceUtil.validateResponse(r, 1, 1, 0, 0, 0);
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&filterBy=STATUS:WAITING", null);
- InstanceUtil.validateResponse(r, 4, 0, 0, 4, 0);
+ InstanceUtil.validateResponse(r, 8, 0, 0, 8, 0);
//use additional filters.
- String sourceCluster = bundles[0].getClusterNames().get(0);
String clusterName = bundles[1].getClusterNames().get(0);
r = prism.getFeedHelper().listInstances(feedName,
"start=" + startTime + "&filterBy=CLUSTER:" + clusterName, null);
- InstanceUtil.validateResponse(r, 10, 1, 1, 4, 4);
- r = prism.getFeedHelper().listInstances(feedName, "start=" + startTime + "&numResults=12"
- + "&filterBy=SOURCECLUSTER:" + sourceCluster, null);
- InstanceUtil.validateResponse(r, 8, 1, 1, 0, 6);
- r = prism.getFeedHelper().listInstances(feedName,
- "filterBy=SOURCECLUSTER:" + sourceCluster, null);
- InstanceUtil.validateResponse(r, 6, 1, 1, 0, 4);
+ InstanceUtil.validateResponse(r, 10, 1, 1, 6, 2);
}
/**
- * Test list feed instances using custom filter. Expecting list of feed instances which
+ * List feed instances using custom filter. Expecting list of feed instances which
* satisfy custom filters.
*/
@Test
@@ -427,33 +432,39 @@ public class ListFeedInstancesTest extends BaseTestClass {
InstancesResult r = prism.getFeedHelper().listInstances(feedName, params, null);
InstanceUtil.validateResponse(r, 1, 1, 0, 0, 0);
+ //expecting 0 instances, because RUNNING instance is out of range start + 10 instances
params = "start=" + startTime + "&end=" + endTime + "&filterBy=status:RUNNING&offset=2";
r = prism.getFeedHelper().listInstances(feedName, params, null);
InstanceUtil.validateSuccessWOInstances(r);
+ //offset is absent that's why whole range is filtered
params = "start=" + startTime + "&end=" + endTime + "&filterBy=status:WAITING";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 4, 0, 0, 4, 0);
+ InstanceUtil.validateResponse(r, 8, 0, 0, 8, 0);
+ //filtered range is from 1st till 9th instances inclusively
params = "start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 41)
+ "&filterBy=status:WAITING";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 3, 0, 0, 3, 0);
+ InstanceUtil.validateResponse(r, 7, 0, 0, 7, 0);
- params = "start=" + startTime + "&offset=1&numResults=1&filterBy=status:WAITING";
+ //filtered range is within 3nd till 8th instance inclusively
+ params = "start=" + startTime + "&offset=4&numResults=6&filterBy=status:WAITING";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 1, 0, 0, 1, 0);
+ InstanceUtil.validateResponse(r, 4, 0, 0, 4, 0);
+ //filtered range is within 4th till 10th instances inclusively
params = "start=" + TimeUtil.addMinsToTime(startTime, 16) + "&offset=2&numResults=12";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 6, 0, 1, 3, 2);
+ InstanceUtil.validateResponse(r, 6, 0, 1, 5, 0);
+ //use mix of params
String sourceCluster = bundles[0].getClusterNames().get(0);
String clusterName = bundles[1].getClusterNames().get(0);
params = "start=" + startTime + "&filterBy=STATUS:KILLED,CLUSTER:"+ clusterName
+ "&numResults=5&orderBy=startTime&sortOrder=desc";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 5, 0, 0, 0, 5);
+ InstanceUtil.validateResponse(r, 2, 0, 0, 0, 2);
//should be ordered by a start time
SoftAssert softAssert = new SoftAssert();
@@ -468,21 +479,15 @@ public class ListFeedInstancesTest extends BaseTestClass {
}
softAssert.assertAll();
- //missing 1st, 11th, 12th instances, all other instances should be retrieved.
+ // filtered range is within 2nd and 10th instance inclusively
params = "start=" + TimeUtil.addMinsToTime(startTime, 2) + "&offset=2";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 9, 0, 1, 3, 5);
+ InstanceUtil.validateResponse(r, 9, 0, 1, 6, 2);
- //missing the 1st, 11th, 12th instance, all instances which have progressed should be present:
- //5 killed + 1 suspended, but numResults=5, so expecting 1 suspended and 4 killed instances.
+ //filtered range is within 7nd and 11th instances inclusively
params = "start=" + TimeUtil.addMinsToTime(startTime, 2) + "&filterBy=SOURCECLUSTER:"
- + sourceCluster + "&offset=1&numResults=5";
+ + sourceCluster + "&offset=1&numResults=5" + "&colo=*";
r = prism.getFeedHelper().listInstances(feedName, params, null);
- InstanceUtil.validateResponse(r, 5, 0, 1, 0, 4);
- }
-
- @AfterClass(alwaysRun = true)
- public void tearDown() throws IOException {
- removeTestClassEntities();
+ InstanceUtil.validateResponse(r, 5, 1, 1, 3, 0);
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java
index 43bdd87..a4022a8 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/lineage/ListProcessInstancesTest.java
@@ -19,9 +19,11 @@
package org.apache.falcon.regression.lineage;
import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.regression.core.bundle.Bundle;
import org.apache.falcon.regression.core.helpers.ColoHelper;
import org.apache.falcon.regression.core.util.BundleUtil;
+import org.apache.falcon.regression.core.util.HadoopUtil;
import org.apache.falcon.regression.core.util.InstanceUtil;
import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
@@ -78,11 +80,14 @@ public class ListProcessInstancesTest extends BaseTestClass {
bundles[0].setOutputFeedLocationData(baseTestHDFSDir + "/output/" + suffix + MINUTE_DATE_PATTERN);
bundles[0].setProcessValidity(startTime, endTime);
bundles[0].setProcessConcurrency(3);
+ bundles[0].setInputFeedPeriodicity(5, Frequency.TimeUnit.minutes);
bundles[0].submitAndScheduleProcess();
processName = bundles[0].getProcessName();
InstanceUtil.waitTillInstancesAreCreated(clusterOC, bundles[0].getProcessData(), 0);
//create data for processes to run and wait some time for instances to make progress
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 0);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 1);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 2);
InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3,
CoordinatorAction.Status.RUNNING, EntityType.PROCESS, 3);
}
@@ -90,6 +95,7 @@ public class ListProcessInstancesTest extends BaseTestClass {
@AfterMethod(alwaysRun = true)
public void tearDown() throws IOException {
removeTestClassEntities();
+ HadoopUtil.deleteDirIfExists(sourcePath, serverFS.get(0));
}
/**
@@ -97,9 +103,6 @@ public class ListProcessInstancesTest extends BaseTestClass {
*/
@Test
public void testProcessOrderBy() throws Exception {
- //provide data for 4th and 5th instances (fyi: indexing starts from 0th instance)
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 3);
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 4);
SoftAssert softAssert = new SoftAssert();
//orderBy startTime descending order
InstancesResult r = prism.getProcessHelper().listInstances(processName,
@@ -122,14 +125,23 @@ public class ListProcessInstancesTest extends BaseTestClass {
r = prism.getProcessHelper().getProcessInstanceKill(processName,
"?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 3));
InstanceUtil.validateResponse(r, 1, 0, 0, 0, 1);
- //wait till instances status be stable
+ //wait till another instances succeed
+ InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 1,
+ CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS);
+
+ //provide data for 4th, 5th and 6th instances (indexing starts from 0th instance)
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 3);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 4);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 5);
+
+ //wait for new 3 instances to run
InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3,
- CoordinatorAction.Status.RUNNING, EntityType.PROCESS, 3);
+ CoordinatorAction.Status.RUNNING, EntityType.PROCESS);
//orderBy status ascending order
r = prism.getProcessHelper().listInstances(processName,
"start=" + startTime + "&numResults=12&orderBy=status&sortOrder=desc", null);
- InstanceUtil.validateResponse(r, 12, 3, 1, 7, 1);
+ InstanceUtil.validateResponse(r, 12, 3, 1, 6, 1);
instances = r.getInstances();
InstancesResult.WorkflowStatus previousStatus = InstancesResult.WorkflowStatus.WAITING;
for (InstancesResult.Instance instance : instances) {
@@ -221,10 +233,6 @@ public class ListProcessInstancesTest extends BaseTestClass {
*/
@Test
public void testProcessFilterBy() throws Exception {
- //provide data for 4th and 5th instances (fyi: indexing starts from 0th instance)
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 3);
- OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, processName, 0, 4);
-
//test with simple filters
InstancesResult r = prism.getProcessHelper().listInstances(processName,
"filterBy=STATUS:RUNNING", null);
@@ -247,14 +255,14 @@ public class ListProcessInstancesTest extends BaseTestClass {
"?start=" + startTime + "&end=" + TimeUtil.addMinsToTime(startTime, 3));
InstanceUtil.validateResponse(r, 1, 0, 0, 0, 1);
- //wait till new instances be RUNNING and total status count be stable
- InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 3,
+ //check that running instance is still running
+ InstanceUtil.waitTillInstanceReachState(clusterOC, processName, 1,
CoordinatorAction.Status.RUNNING, EntityType.PROCESS, 3);
//get all instances
r = prism.getProcessHelper().listInstances(processName,
"start=" + startTime + "&numResults=12", null);
- InstanceUtil.validateResponse(r, 12, 3, 1, 7, 1);
+ InstanceUtil.validateResponse(r, 12, 1, 1, 9, 1);
//use different statuses, filterBy among all instances
r = prism.getProcessHelper().listInstances(processName,
@@ -265,10 +273,10 @@ public class ListProcessInstancesTest extends BaseTestClass {
InstanceUtil.validateResponse(r, 1, 0, 1, 0, 0);
r = prism.getProcessHelper().listInstances(processName,
"start=" + startTime + "&filterBy=STATUS:RUNNING", null);
- InstanceUtil.validateResponse(r, 3, 3, 0, 0, 0);
+ InstanceUtil.validateResponse(r, 1, 1, 0, 0, 0);
r = prism.getProcessHelper().listInstances(processName,
"start=" + startTime + "&filterBy=STATUS:WAITING", null);
- InstanceUtil.validateResponse(r, 7, 0, 0, 7, 0);
+ InstanceUtil.validateResponse(r, 9, 0, 0, 9, 0);
}
/**
http://git-wip-us.apache.org/repos/asf/falcon/blob/1e3dcb73/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/searchUI/EntityPageTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/searchUI/EntityPageTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/searchUI/EntityPageTest.java
index 4ad775e..6acec27 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/searchUI/EntityPageTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/searchUI/EntityPageTest.java
@@ -85,6 +85,7 @@ public class EntityPageTest extends BaseUITestClass {
bundles[0].setProcessPeriodicity(5, Frequency.TimeUnit.minutes);
bundles[0].setOutputFeedPeriodicity(5, Frequency.TimeUnit.minutes);
bundles[0].setOutputFeedLocationData(feedOutputPath);
+ bundles[0].setInputFeedPeriodicity(5, Frequency.TimeUnit.minutes);
openBrowser();
final LoginPage loginPage = LoginPage.open(getDriver());
@@ -295,14 +296,15 @@ public class EntityPageTest extends BaseUITestClass {
InstanceUtil.waitTillInstanceReachState(clusterOC, process.getName(), 1,
CoordinatorAction.Status.WAITING, EntityType.PROCESS, 1);
- final List<String> dataDates = TimeUtil.getMinuteDatesOnEitherSide(startTime,
- TimeUtil.addMinsToTime(endTime, -5), 5);
- HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.NORMAL_INPUT, prefix, dataDates);
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, process.getName(), 0, 0);
InstanceUtil.waitTillInstanceReachState(clusterOC, process.getName(), 1,
CoordinatorAction.Status.SUCCEEDED, EntityType.PROCESS, 5);
+
+ OozieUtil.createMissingDependencies(cluster, EntityType.PROCESS, process.getName(), 0, 1);
InstanceUtil.waitTillInstanceReachState(clusterOC, process.getName(), 1,
CoordinatorAction.Status.RUNNING, EntityType.PROCESS, 5);
+
//suspend the second instance
prism.getProcessHelper().getProcessInstanceSuspend(process.getName(),
"?start=" + TimeUtil.addMinsToTime(startTime, 5)