You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/08/28 02:14:58 UTC
git commit: FALCON-166 Instance status start and end dates are rigid
and inconvenient. Contributed by Balu Vellanki
Repository: incubator-falcon
Updated Branches:
refs/heads/master b7f5915f9 -> 1db8712d6
FALCON-166 Instance status start and end dates are rigid and inconvenient. Contributed by Balu Vellanki
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/1db8712d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/1db8712d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/1db8712d
Branch: refs/heads/master
Commit: 1db8712d61bbdf3f24f04a4a85b18ed91e171a21
Parents: b7f5915
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Wed Aug 27 17:14:50 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Wed Aug 27 17:14:50 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 4 +
.../java/org/apache/falcon/cli/FalconCLI.java | 12 +-
.../org/apache/falcon/client/FalconClient.java | 4 +-
.../org/apache/falcon/entity/EntityUtil.java | 16 ++
.../apache/falcon/entity/EntityUtilTest.java | 17 ++
docs/src/site/twiki/FalconCLI.twiki | 51 ++++-
docs/src/site/twiki/restapi/EntityList.twiki | 37 ++--
docs/src/site/twiki/restapi/InstanceList.twiki | 28 +--
docs/src/site/twiki/restapi/InstanceLogs.twiki | 31 ++--
.../site/twiki/restapi/InstanceRunning.twiki | 25 +--
.../src/site/twiki/restapi/InstanceStatus.twiki | 29 +--
.../site/twiki/restapi/InstanceSummary.twiki | 17 +-
docs/src/site/twiki/restapi/ResourceList.twiki | 12 +-
.../falcon/resource/AbstractEntityManager.java | 14 +-
.../resource/AbstractInstanceManager.java | 185 +++++++++----------
.../resource/proxy/InstanceManagerProxy.java | 10 +-
.../proxy/SchedulableEntityManagerProxy.java | 3 +-
.../falcon/resource/EntityManagerTest.java | 15 +-
.../apache/falcon/resource/InstanceManager.java | 8 +-
.../resource/SchedulableEntityManager.java | 3 +-
.../java/org/apache/falcon/cli/FalconCLIIT.java | 21 +++
21 files changed, 339 insertions(+), 203 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index b67310b..8bd1ff9 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -27,7 +27,11 @@ Trunk (Unreleased)
FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
IMPROVEMENTS
+ FALCON-166 Instance status start and end dates are rigid and inconvenient
+ (Balu Vellanki via Venkatesh Seetharam)
+
FALCON-612 Create CHANGES.txt for falcon-regression(Arpit Gupta via SamarthG)
+
FALCON-470 Add support for pagination, filter by, etc. to Entity and
Instance List API (Balu Vellanki via Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
index b0c133a..cb46d46 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -227,8 +227,9 @@ public class FalconCLI {
List<LifeCycle> lifeCycles = getLifeCycle(commandLine.getOptionValue(LIFECYCLE_OPT));
String filterBy = commandLine.getOptionValue(FILTER_BY_OPT);
String orderBy = commandLine.getOptionValue(ORDER_BY_OPT);
- Integer offset = validateIntInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
- Integer numResults = validateIntInput(commandLine.getOptionValue(NUM_RESULTS_OPT), -1, "numResults");
+ Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
+ Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT),
+ FalconClient.DEFAULT_NUM_RESULTS, "numResults");
colo = getColo(colo);
String instanceAction = "instance";
@@ -272,7 +273,7 @@ public class FalconCLI {
OUT.get().println(result);
}
- private Integer validateIntInput(String optionValue, int defaultVal, String optionName) throws FalconCLIException {
+ private Integer parseIntegerInput(String optionValue, int defaultVal, String optionName) throws FalconCLIException {
Integer integer = defaultVal;
if (optionValue != null) {
try {
@@ -343,8 +344,9 @@ public class FalconCLI {
String filterBy = commandLine.getOptionValue(FILTER_BY_OPT);
String filterTags = commandLine.getOptionValue(TAGS_OPT);
String fields = commandLine.getOptionValue(FIELDS_OPT);
- Integer offset = validateIntInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
- Integer numResults =validateIntInput(commandLine.getOptionValue(NUM_RESULTS_OPT), -1, "numResults");
+ Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
+ Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT),
+ FalconClient.DEFAULT_NUM_RESULTS, "numResults");
validateEntityType(entityType);
if (optionsList.contains(SUBMIT_OPT)) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 6697227..5e9543c 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -81,6 +81,8 @@ public class FalconClient {
private static final String AUTH_COOKIE_EQ = AUTH_COOKIE + "=";
private static final KerberosAuthenticator AUTHENTICATOR = new KerberosAuthenticator();
+ public static final int DEFAULT_NUM_RESULTS = 10;
+
public static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() {
@Override
public boolean verify(String hostname, SSLSession sslSession) {
@@ -571,7 +573,7 @@ public class FalconClient {
String runid, String colo,
List<LifeCycle> lifeCycles) throws FalconCLIException {
return sendInstanceRequest(instances, type, entity, start, end, props,
- runid, colo, lifeCycles, "", "", 0, -1);
+ runid, colo, lifeCycles, "", "", 0, DEFAULT_NUM_RESULTS);
}
private String sendInstanceRequest(Instances instances, String type, String entity,
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
index 73e19f5..e75f28e 100644
--- a/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
+++ b/common/src/main/java/org/apache/falcon/entity/EntityUtil.java
@@ -22,6 +22,7 @@ import org.apache.commons.beanutils.PropertyUtils;
import org.apache.commons.codec.binary.Hex;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.falcon.FalconException;
+import org.apache.falcon.Pair;
import org.apache.falcon.Tag;
import org.apache.falcon.entity.WorkflowNameBuilder.WorkflowName;
import org.apache.falcon.entity.store.ConfigurationStore;
@@ -687,4 +688,19 @@ public final class EntityUtil {
return Storage.TYPE.TABLE == storageType;
}
+ public static Pair<Date, Date> getEntityStartEndDates(Entity entityObject) {
+ Set<String> clusters = EntityUtil.getClustersDefined(entityObject);
+ Pair<Date, String> clusterMinStartDate = null;
+ Pair<Date, String> clusterMaxEndDate = null;
+ for (String cluster : clusters) {
+ if (clusterMinStartDate == null || clusterMinStartDate.first.after(getStartTime(entityObject, cluster))) {
+ clusterMinStartDate = Pair.of(getStartTime(entityObject, cluster), cluster);
+ }
+ if (clusterMaxEndDate == null || clusterMaxEndDate.first.before(getEndTime(entityObject, cluster))) {
+ clusterMaxEndDate = Pair.of(getEndTime(entityObject, cluster), cluster);
+ }
+ }
+ return new Pair<Date, Date>(clusterMinStartDate.first, clusterMaxEndDate.first);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
index d8a44ea..1613a66 100644
--- a/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
+++ b/common/src/test/java/org/apache/falcon/entity/EntityUtilTest.java
@@ -18,6 +18,7 @@
package org.apache.falcon.entity;
+import org.apache.falcon.Pair;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
@@ -212,4 +213,20 @@ public class EntityUtilTest extends AbstractTestBase {
frequency, tz, instance));
}
+ @Test
+ public void testGetEntityStartEndDates() throws Exception {
+ Process process = (Process) EntityType.PROCESS.getUnmarshaller().unmarshal(
+ getClass().getResourceAsStream(PROCESS_XML));
+
+ Cluster cluster = new Cluster();
+ cluster.setName("testCluster");
+ cluster.setValidity(process.getClusters().getClusters().get(0).getValidity());
+
+ process.getClusters().getClusters().add(cluster);
+
+ Pair<Date, Date> startEndDates = EntityUtil.getEntityStartEndDates(process);
+ Assert.assertEquals(startEndDates.second.toString(), "Sat Dec 29 16:00:00 PST 2091");
+ Assert.assertEquals(startEndDates.first.toString(), "Tue Nov 01 17:00:00 PDT 2011");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/FalconCLI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconCLI.twiki b/docs/src/site/twiki/FalconCLI.twiki
index 2c5d9c5..77a306e 100644
--- a/docs/src/site/twiki/FalconCLI.twiki
+++ b/docs/src/site/twiki/FalconCLI.twiki
@@ -51,6 +51,11 @@ Entities of a particular type can be listed with list sub-command.
Usage:
$FALCON_HOME/bin/falcon entity -type [cluster|feed|process] -list
+Optional Args : -fields <<field1,field2>> -filterBy <<field1:value1,field2:value2>> -tags <<tagkey=tagvalue,tagkey=tagvalue>>
+-orderBy <<field>> -offset 0 -numResults 10
+
+<a href="./Restapi/EntityList.html">Optional params described here.</a>
+
---+++Update
Update operation allows an already submitted/scheduled entity to be updated. Cluster update is currently
@@ -135,7 +140,31 @@ Example : Suppose a process has 3 instance, one has succeeded,one is in running
{"status":"SUCCEEDED","message":"getStatus is successful","instances":[{"instance":"2012-05-07T05:02Z","status":"SUCCEEDED","logFile":"http://oozie-dashboard-url"},{"instance":"2012-05-07T05:07Z","status":"RUNNING","logFile":"http://oozie-dashboard-url"}, {"instance":"2010-01-02T11:05Z","status":"WAITING"}]
Usage:
-$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -status -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'"
+$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -status
+
+Optional Args : -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'" -colo <<colo>>
+-filterBy <<field1:value1,field2:value2>> -lifecycle <<lifecycles>>
+-orderBy field -offset 0 -numResults 10
+
+<a href="./Restapi/InstanceStatus.html"> Optional params described here.</a>
+
+---+++List
+
+List option via CLI can be used to get single or multiple instances. If the instance is not yet materialized but is within the process validity range, WAITING is returned as the state. Instance time is also returned. Log location gives the oozie workflow url
+If the instance is in WAITING state, missing dependencies are listed
+
+Example : Suppose a process has 3 instance, one has succeeded,one is in running state and other one is waiting, the expected output is:
+
+{"status":"SUCCEEDED","message":"getStatus is successful","instances":[{"instance":"2012-05-07T05:02Z","status":"SUCCEEDED","logFile":"http://oozie-dashboard-url"},{"instance":"2012-05-07T05:07Z","status":"RUNNING","logFile":"http://oozie-dashboard-url"}, {"instance":"2010-01-02T11:05Z","status":"WAITING"}]
+
+Usage:
+$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -list
+
+Optional Args : -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'"
+-colo <<colo>> -lifecycle <<lifecycles>>
+-filterBy <<field1:value1,field2:value2>> -orderBy field -offset 0 -numResults 10
+
+<a href="./Restapi/InstanceList.html">Optional params described here.</a>
---+++Summary
@@ -148,7 +177,12 @@ Example : Suppose a process has 3 instance, one has succeeded,one is in running
{"status":"SUCCEEDED","message":"getSummary is successful", "cluster": <<name>> [{"SUCCEEDED":"1"}, {"WAITING":"1"}, {"RUNNING":"1"}]}
Usage:
-$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -summary -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'"
+$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -summary
+
+Optional Args : -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'"
+-colo <<colo>> -lifecycle <<lifecycles>>
+
+<a href="./Restapi/InstanceSummary.html">Optional params described here.</a>
---+++Running
@@ -157,12 +191,23 @@ Running option provides all the running instances of the mentioned process.
Usage:
$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -running
+Optional Args : -colo <<colo>> -lifecycle <<lifecycles>>
+-filterBy <<field1:value1,field2:value2>> -orderBy <<field>> -offset 0 -numResults 10
+
+<a href="./Restapi/InstanceRunning.html">Optional params described here.</a>
+
---+++Logs
Get logs for instance actions
Usage:
-$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -logs -start "yyyy-MM-dd'T'HH:mm'Z'" [-end "yyyy-MM-dd'T'HH:mm'Z'"] [-runid <<runid>>]
+$FALCON_HOME/bin/falcon instance -type <<feed/process>> -name <<name>> -logs
+
+Optional Args : -start "yyyy-MM-dd'T'HH:mm'Z'" -end "yyyy-MM-dd'T'HH:mm'Z'" -runid <<runid>>
+-colo <<colo>> -lifecycle <<lifecycles>>
+-filterBy <<field1:value1,field2:value2>> -orderBy field -offset 0 -numResults 10
+
+<a href="./Restapi/InstanceLogs.html">Optional params described here.</a>
---+++LifeCycle
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/EntityList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/EntityList.twiki b/docs/src/site/twiki/restapi/EntityList.twiki
index 678b907..0697561 100644
--- a/docs/src/site/twiki/restapi/EntityList.twiki
+++ b/docs/src/site/twiki/restapi/EntityList.twiki
@@ -8,17 +8,18 @@
Get list of the entities.
---++ Parameters
- * :entity-type can be cluster, feed or process.
- * :fields <optional param> additional fields that the client are interested in, separated by commas.
- * Currently falcon supports STATUS, TAGS, PIPELINES as valid fields.
- * :filterBy <optional param> Filter results by a given list of field,value pair. Example: filterBy=STATUS:RUNNING,PIPELINES:clickLogs
- * Supported filter fields are TYPE, NAME, STATUS, PIPELINES
- * Only the entities that match both filters are returned.
- * :tags <optional param> Filter results by a given list of tags, separated by a comma. Example: tags=consumer=consumer@xyz.com,owner=producer@xyz.com
- * :orderBy <optional param> is the field by which results should be ordered.
- * Supports ordering by "name","type"
- * :offset <optional param> start showing results from the offset, used for pagination. Integer, Default is 0
- * :numResults <optional param> number of results to show per request, used for pagination. Integer, Default is -1 (means all results)
+ * :entity-type Valid options are cluster, feed or process.
+ * fields <optional param> Fields of entity that the user wants to view, separated by commas.
+ * Valid options are STATUS, TAGS, PIPELINES.
+ * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,PIPELINES:clickLogs
+ * Supported filter fields are NAME, STATUS, PIPELINES.
+ * Query will do an AND among filterBy fields.
+ * tags <optional param> Return list of entities that have specified tags, separated by a comma. Query will do AND on tag values.
+ * Example: tags=consumer=consumer@xyz.com,owner=producer@xyz.com
+ * orderBy <optional param> Field by which results should be ordered.
+ * Supports ordering by "name".
+ * offset <optional param> Show results from the offset, used for pagination. Defaults to 0.
+ * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10.
---++ Results
List of the entities.
@@ -68,25 +69,25 @@ GET http://localhost:15000/api/entities/list/feed?fields=status
---+++ Rest Call
<verbatim>
-GET http://localhost:15000/api/entities/list/feed?filterBy=STATUS:RUNNING,PIPELINES:dataReplication&fields=status,pipelines,tags&tags=consumer=consumer@xyz.com&orderBy=name&offset=2&numResults=2
+GET http://localhost:15000/api/entities/list/process?filterBy=STATUS:RUNNING,PIPELINES:dataReplication&fields=status,pipelines,tags&tags=consumer=consumer@xyz.com&orderBy=name&offset=2&numResults=2
</verbatim>
---+++ Result
<verbatim>
{
"entity": [
{
- "name" : "SampleInput2",
- "type" : "feed",
+ "name" : "SampleProcess1",
+ "type" : "process",
"status": "RUNNING",
"pipelines": "dataReplication",
- "tags":consumer=consumer@xyz.com
+ "tags": "consumer=consumer@xyz.com"
},
{
- "name": "SampleInput3",
- "type": "feed",
+ "name": "SampleProcess3",
+ "type": "process",
"status": "RUNNING",
"pipelines": "dataReplication",
- "tags":consumer=consumer@xyz.com,owner=producer@xyz.com
+ "tags": "consumer=consumer@xyz.com,owner=producer@xyz.com"
}
]
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/InstanceList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceList.twiki b/docs/src/site/twiki/restapi/InstanceList.twiki
index c2059f2..76eaa33 100644
--- a/docs/src/site/twiki/restapi/InstanceList.twiki
+++ b/docs/src/site/twiki/restapi/InstanceList.twiki
@@ -8,19 +8,21 @@
Get list of all instances of a given entity.
---++ Parameters
- * :entity-type can either be a feed or a process.
- * :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
- * end <optional param> is the end time of the instance that you want to refer to
- * colo <optional param> colo on which the query should be run
- * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
- * filterBy <optional param> Filter results by a given list of field,value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
- * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER,STARTEDAFTER
- * Only the instances that match both filters are returned.
- * orderBy <optional param> is the instance field by which results should be ordered.
- * Supports ordering by "status","startTime","endTime","cluster"
- * offset <optional param> start showing results from the offset, used for pagination. Integer, Default is 0
- * numResults <optional param> number of results to show per request, used for pagination. Integer, Default is -1 (means all results)
+ * :entity-type Valid options are cluster, feed or process.
+ * :entity-name Name of the entity.
+ * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * By default, it is set to (end - (10 * entityFrequency)).
+ * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * Default is set to now.
+ * colo <optional param> Colo on which the query should be run.
+ * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default).
+ * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
+ * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER, STARTEDAFTER.
+ * Query will do an AND among filterBy fields.
+ * orderBy <optional param> Field by which results should be ordered.
+ * Supports ordering by "status","startTime","endTime","cluster".
+ * offset <optional param> Show results from the offset, used for pagination. Defaults to 0.
+ * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10.
---++ Results
List of instances of given entity.
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/InstanceLogs.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceLogs.twiki b/docs/src/site/twiki/restapi/InstanceLogs.twiki
index 56d5fe1..354eed4 100644
--- a/docs/src/site/twiki/restapi/InstanceLogs.twiki
+++ b/docs/src/site/twiki/restapi/InstanceLogs.twiki
@@ -8,20 +8,23 @@
Get log of a specific instance of an entity.
---++ Parameters
- * :entity-type can either be a feed or a process.
- * :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
- * end <optional param> is the end time of the instance that you want to refer to
- * colo <optional param> colo on which the query should be run
- * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
- * filterBy <optional param> Filter results by a given list of field,value pairs. Example: filterBy=STATUS:SUCCEEDED,CLUSTER:primary-cluster
- * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER,STARTEDAFTER
- * Only the instances that match both filters are returned.
- * orderBy <optional param> is the instance field by which results should be ordered.
- * Supports ordering by "status","startTime","endTime","cluster"
- * offset <optional param> start showing results from the offset, used for pagination. Integer, Default is 0
- * numResults <optional param> number of results to show per request, used for pagination. Integer, Default is -1 (means all results)
-
+ * :entity-type Valid options are cluster, feed or process.
+ * :entity-name Name of the entity.
+ * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * By default, it is set to (end - (10 * entityFrequency)).
+ * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * Default is set to now.
+ * colo <optional param> Colo on which the query should be run.
+ * runId <optional param> Run Id.
+ * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default).
+ * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
+ * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER, STARTEDAFTER.
+ * Query will do an AND among filterBy fields.
+ * orderBy <optional param> Field by which results should be ordered.
+ * Supports ordering by "status","startTime","endTime","cluster".
+ * offset <optional param> Show results from the offset, used for pagination. Defaults to 0.
+ * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10.
+
---++ Results
Log of specified instance.
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/InstanceRunning.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceRunning.twiki b/docs/src/site/twiki/restapi/InstanceRunning.twiki
index 0d04008..47037d6 100644
--- a/docs/src/site/twiki/restapi/InstanceRunning.twiki
+++ b/docs/src/site/twiki/restapi/InstanceRunning.twiki
@@ -8,17 +8,18 @@
Get a list of instances currently running for a given entity.
---++ Parameters
- * :entity-type can either be a feed or a process.
- * :entity-name is name of the entity.
- * colo <optional param> colo on which the query should be run
- * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
- * filterBy <optional param> Filter results by a given list of field,value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
- * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER,STARTEDAFTER
- * Only the instances that match both filters are returned.
- * orderBy <optional param> is the instance field by which results should be ordered.
- * Supports ordering by "status","startTime","endTime","cluster"
- * offset <optional param> start showing results from the offset, used for pagination. Integer, Default is 0
- * numResults <optional param> number of results to show per request, used for pagination. Integer, Default is -1 (means all results)
+ * :entity-type Valid options are cluster, feed or process.
+ * :entity-name Name of the entity.
+ * colo <optional param> Colo on which the query should be run.
+ * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default).
+ * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=CLUSTER:primary-cluster
+ * Supported filter fields are CLUSTER, SOURCECLUSTER, STARTEDAFTER.
+ * Query will do an AND among filterBy fields.
+ * orderBy <optional param> Field by which results should be ordered.
+ * Supports ordering by "status","startTime","endTime","cluster".
+ * offset <optional param> Show results from the offset, used for pagination. Defaults to 0.
+ * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10.
+
---++ Results
List of instances currently running.
@@ -48,7 +49,7 @@ GET http://localhost:15000/api/instance/running/process/SampleProcess?colo=*
---+++ Rest Call
<verbatim>
-GET http://localhost:15000/api/instance/running/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&filterBy=STATUS:RUNNING,CLUSTER:primary-cluster&orderBy=startTime&offset=2&numResults=2
+GET http://localhost:15000/api/instance/running/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&filterBy=CLUSTER:primary-cluster&orderBy=startTime&offset=2&numResults=2
</verbatim>
---+++ Result
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/InstanceStatus.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceStatus.twiki b/docs/src/site/twiki/restapi/InstanceStatus.twiki
index 5160252..69634de 100644
--- a/docs/src/site/twiki/restapi/InstanceStatus.twiki
+++ b/docs/src/site/twiki/restapi/InstanceStatus.twiki
@@ -8,19 +8,22 @@
Get status of a specific instance of an entity.
---++ Parameters
- * :entity-type can either be a feed or a process.
- * :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
- * end <optional param> is the end time of the instance that you want to refer to
- * colo <optional param> colo on which the query should be run
- * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
- * filterBy <optional param> Filter results by a given list of field,value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
- * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER,STARTEDAFTER
- * Only the instances that match both filters are returned.
- * orderBy <optional param> is the instance field by which results should be ordered.
- * Supports ordering by "status","startTime","endTime","cluster"
- * offset <optional param> start showing results from the offset, used for pagination. Integer, Default is 0
- * numResults <optional param> number of results to show per request, used for pagination. Integer, Default is -1 (means all results)
+ * :entity-type Valid options are cluster, feed or process.
+ * :entity-name Name of the entity.
+ * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * By default, it is set to (end - (10 * entityFrequency)).
+ * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * Default is set to now.
+ * colo <optional param> Colo on which the query should be run.
+ * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default).
+ * filterBy <optional param> Filter results by list of field:value pairs. Example: filterBy=STATUS:RUNNING,CLUSTER:primary-cluster
+ * Supported filter fields are STATUS, CLUSTER, SOURCECLUSTER, STARTEDAFTER.
+ * Query will do an AND among filterBy fields.
+ * orderBy <optional param> Field by which results should be ordered.
+ * Supports ordering by "status","startTime","endTime","cluster".
+ * offset <optional param> Show results from the offset, used for pagination. Defaults to 0.
+ * numResults <optional param> Number of results to show per request, used for pagination. Only integers > 0 are valid, Default is 10.
+
---++ Results
Status of the specified instance.
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/InstanceSummary.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceSummary.twiki b/docs/src/site/twiki/restapi/InstanceSummary.twiki
index 4fa5780..9d1041e 100644
--- a/docs/src/site/twiki/restapi/InstanceSummary.twiki
+++ b/docs/src/site/twiki/restapi/InstanceSummary.twiki
@@ -8,12 +8,14 @@
Get summary of instance/instances of an entity.
---++ Parameters
- * :entity-type can either be a feed or a process.
- * :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
- * end <optional param> is the end time of the instance that you want to refer to
- * colo <optional param> colo on which the query should be run
- * lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
+ * :entity-type Valid options are cluster, feed or process.
+ * :entity-name Name of the entity.
+ * start <optional param> Show instances from this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * By default, it is set to (end - (10 * entityFrequency)).
+ * end <optional param> Show instances up to this date. Date format is yyyy-MM-dd'T'HH:mm'Z'.
+ * Default is set to now.
+ * colo <optional param> Colo on which the query should be run.
+ * lifecycle <optional param> Valid lifecycles for feed are Eviction/Replication(default) and for process is Execution(default).
---++ Results
Summary of the instances over the specified time range
@@ -22,7 +24,6 @@ Summary of the instances over the specified time range
---+++ Rest Call
<verbatim>
GET http://localhost:15000/api/instance/summary/process/WordCount?colo=*&start=2014-01-21T13:00Z&end=2014-01-21T16:00Z
-Remote-User: suhas
</verbatim>
---+++ Result
<verbatim>
@@ -38,7 +39,7 @@ Remote-User: suhas
"entry":
{
"key":"SUCCEEDED",
- "value":"3"
+ "key2":"value"
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/docs/src/site/twiki/restapi/ResourceList.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/ResourceList.twiki b/docs/src/site/twiki/restapi/ResourceList.twiki
index 4b377e8..9284810 100644
--- a/docs/src/site/twiki/restapi/ResourceList.twiki
+++ b/docs/src/site/twiki/restapi/ResourceList.twiki
@@ -48,21 +48,21 @@ See also: [[../Security.twiki][Security in Falcon]]
| DELETE | [[EntityDelete][api/entities/delete/:entity-type/:entity-name]] | Delete the entity |
| GET | [[EntityStatus][api/entities/status/:entity-type/:entity-name]] | Get the status of the entity |
| GET | [[EntityDefinition][api/entities/definition/:entity-type/:entity-name]] | Get the definition of the entity |
-| GET | [[EntityList][api/entities/list/:entity-type?fields=:fields&filterBy=:filterBy&tags=:tags&orderBy=:orderBy&offset=:offset&numResults=:numResults]] | Get the list of entities |
+| GET | [[EntityList][api/entities/list/:entity-type]] | Get the list of entities |
| GET | [[EntityDependencies][api/entities/dependencies/:entity-type/:entity-name]] | Get the dependencies of the entity |
---++ REST Call on Feed and Process Instances
| *Call Type* | *Resource* | *Description* |
-| GET | [[InstanceRunning][api/instance/running/:entity-type/:entity-name?colo=:colo&lifecycle=:lifecycle&...]] | List of running instances. |
-| GET | [[InstanceList][api/instance/list/:entity-type/:entity-name?start=:start&end=:end&colo&lifecycle=:lifecycle&...]] | List of instances |
-| GET | [[InstanceStatus][api/instance/status/:entity-type/:entity-name?start=:start&end=:end&colo=:colo&lifecycle=:lifecycle&...]] | Status of a given instance |
+| GET | [[InstanceRunning][api/instance/running/:entity-type/:entity-name]] | List of running instances. |
+| GET | [[InstanceList][api/instance/list/:entity-type/:entity-name]] | List of instances |
+| GET | [[InstanceStatus][api/instance/status/:entity-type/:entity-name]] | Status of a given instance |
| POST | [[InstanceKill][api/instance/kill/:entity-type/:entity-name]] | Kill a given instance |
| POST | [[InstanceSuspend][api/instance/suspend/:entity-type/:entity-name]] | Suspend a running instance |
| POST | [[InstanceResume][api/instance/resume/:entity-type/:entity-name]] | Resume a given instance |
| POST | [[InstanceRerun][api/instance/rerun/:entity-type/:entity-name]] | Rerun a given instance |
-| GET | [[InstanceLogs][api/instance/logs/:entity-type/:entity-name?start=:start&end=:end&colo=:colo&lifecycle=:lifecycle&...]] | Get logs of a given instance |
-| GET | [[InstanceSummary][api/instance/summary/:entity-type/:entity-name?start=:start&end=:end&colo=:colo&lifecycle=:lifecycle]] | Return summary of instances for an entity |
+| GET | [[InstanceLogs][api/instance/logs/:entity-type/:entity-name]] | Get logs of a given instance |
+| GET | [[InstanceSummary][api/instance/summary/:entity-type/:entity-name]] | Return summary of instances for an entity |
---++ REST Call on Lineage Graph
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
index 84ae446..a6d1b29 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
@@ -60,7 +60,9 @@ import java.util.*;
public abstract class AbstractEntityManager {
private static final Logger LOG = LoggerFactory.getLogger(AbstractEntityManager.class);
private static final Logger AUDIT = LoggerFactory.getLogger("AUDIT");
+
protected static final int XML_DEBUG_LEN = 10 * 1024;
+ protected static final String DEFAULT_NUM_RESULTS = "10";
private AbstractWorkflowEngine workflowEngine;
protected ConfigurationStore configStore = ConfigurationStore.get();
@@ -728,14 +730,22 @@ public abstract class AbstractEntityManager {
elements[offset] is included. Size 10, offset 10, return empty list.
Size 10, offset 5, count 3, return elements[5,6,7].
Size 10, offset 5, count >= 5, return elements[5,6,7,8,9]
- When count is -1, return elements starting from elements[offset] until the end */
+ return elements starting from elements[offset] until the end OR offset+numResults*/
+
+ if (numresults < 1) {
+ LOG.error("Value for param numResults should be > than 0 : {}", numresults);
+ throw FalconWebException.newException("Value for param numResults should be > than 0 : " + numresults,
+ Response.Status.BAD_REQUEST);
+ }
+
+ if (offset < 0) { offset = 0; }
if (offset >= arraySize || arraySize == 0) {
// No elements to return
return 0;
}
int retLen = arraySize - offset;
- if (retLen > numresults && numresults != -1) {
+ if (retLen > numresults) {
retLen = numresults;
}
return retLen;
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
index 141ee9c..6862b99 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
@@ -24,6 +24,7 @@ import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.parser.ValidationException;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
+import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.logging.LogProvider;
import org.apache.falcon.resource.InstancesResult.Instance;
@@ -43,6 +44,11 @@ import java.util.*;
public abstract class AbstractInstanceManager extends AbstractEntityManager {
private static final Logger LOG = LoggerFactory.getLogger(AbstractInstanceManager.class);
+ private static final long MINUTE_IN_MILLIS = 60000L;
+ private static final long HOUR_IN_MILLIS = 3600000L;
+ private static final long DAY_IN_MILLIS = 86400000L;
+ private static final long MONTH_IN_MILLIS = 2592000000L;
+
protected void checkType(String type) {
if (StringUtils.isEmpty(type)) {
throw FalconWebException.newInstanceException("entity type is empty",
@@ -111,13 +117,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
checkType(type);
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
- validateParams(type, entity, startStr, endStr);
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
+
// LifeCycle lifeCycleObject = EntityUtil.getLifeCycle(lifeCycle);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return getInstanceResultSubset(wfEngine.getStatus(entityObject, start, end, lifeCycles),
+ return getInstanceResultSubset(wfEngine.getStatus(entityObject,
+ startAndEndDate.first, startAndEndDate.second, lifeCycles),
filterBy, orderBy, offset, numResults);
} catch (Throwable e) {
LOG.error("Failed to get instances status", e);
@@ -132,14 +139,12 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
checkType(type);
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
- validateParams(type, entity, startStr, endStr);
-
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.getSummary(entityObject, start, end, lifeCycles);
+ return wfEngine.getSummary(entityObject, startAndEndDate.first, startAndEndDate.second, lifeCycles);
} catch (Throwable e) {
LOG.error("Failed to get instances status", e);
throw FalconWebException.newInstanceSummaryException(e, Response.Status.BAD_REQUEST);
@@ -276,14 +281,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
Collections.sort(instanceSet, new Comparator<Instance>() {
@Override
public int compare(Instance i1, Instance i2) {
- return i1.getStartTime().compareTo(i2.getStartTime());
+ return i2.getStartTime().compareTo(i1.getStartTime()); //default desc
}
});
} else if (orderBy.equals("endTime")) {
Collections.sort(instanceSet, new Comparator<Instance>() {
@Override
public int compare(Instance i1, Instance i2) {
- return i1.getEndTime().compareTo(i2.getEndTime());
+ return i2.getEndTime().compareTo(i1.getEndTime()); //default desc
}
});
}//Default : no sort
@@ -303,13 +308,12 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
throw new FalconException("For displaying wf-params there can't be more than one lifecycle "
+ lifeCycles);
}
- validateParams(type, entity, startTime, null);
-
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
- Date start = EntityUtil.parseDateUTC(startTime);
- Date end = getEndDate(start, null);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startTime, null);
+
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.getInstanceParams(entityObject, start, end, lifeCycles);
+ return wfEngine.getInstanceParams(entityObject, startAndEndDate.first, startAndEndDate.second, lifeCycles);
} catch (Throwable e) {
LOG.error("Failed to display params of an instance", e);
throw FalconWebException.newInstanceException(e, Response.Status.BAD_REQUEST);
@@ -326,15 +330,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
audit(request, entity, type, "INSTANCE_KILL");
- validateParams(type, entity, startStr, endStr);
-
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.killInstances(entityObject, start, end, props, lifeCycles);
+ return wfEngine.killInstances(entityObject,
+ startAndEndDate.first, startAndEndDate.second, props, lifeCycles);
} catch (Throwable e) {
LOG.error("Failed to kill instances", e);
throw FalconWebException.newInstanceException(e, Response.Status.BAD_REQUEST);
@@ -351,15 +354,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
audit(request, entity, type, "INSTANCE_SUSPEND");
- validateParams(type, entity, startStr, endStr);
-
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.suspendInstances(entityObject, start, end, props, lifeCycles);
+ return wfEngine.suspendInstances(entityObject,
+ startAndEndDate.first, startAndEndDate.second, props, lifeCycles);
} catch (Throwable e) {
LOG.error("Failed to suspend instances", e);
throw FalconWebException.newInstanceException(e, Response.Status.BAD_REQUEST);
@@ -376,15 +378,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
audit(request, entity, type, "INSTANCE_RESUME");
- validateParams(type, entity, startStr, endStr);
-
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.resumeInstances(entityObject, start, end, props, lifeCycles);
+ return wfEngine.resumeInstances(entityObject,
+ startAndEndDate.first, startAndEndDate.second, props, lifeCycles);
} catch (Throwable e) {
LOG.error("Failed to resume instances", e);
throw FalconWebException.newInstanceException(e, Response.Status.BAD_REQUEST);
@@ -400,15 +401,14 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
try {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
audit(request, entity, type, "INSTANCE_RERUN");
- validateParams(type, entity, startStr, endStr);
-
- Date start = EntityUtil.parseDateUTC(startStr);
- Date end = getEndDate(start, endStr);
+ validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
- return wfEngine.reRunInstances(entityObject, start, end, props, lifeCycles);
+ return wfEngine.reRunInstances(entityObject,
+ startAndEndDate.first, startAndEndDate.second, props, lifeCycles);
} catch (Exception e) {
LOG.error("Failed to rerun instances", e);
throw FalconWebException.newInstanceException(e, Response.Status.BAD_REQUEST);
@@ -427,75 +427,72 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
return props;
}
- private Date getEndDate(Date start, String endStr) throws FalconException {
- Date end;
- if (StringUtils.isEmpty(endStr)) {
- end = new Date(start.getTime() + 1000); // next sec
- } else {
- end = EntityUtil.parseDateUTC(endStr);
+ private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr)
+ throws FalconException {
+ Pair<Date, Date> clusterStartEndDates = EntityUtil.getEntityStartEndDates(entityObject);
+ Frequency frequency = EntityUtil.getFrequency(entityObject);
+ Date endDate = getEndDate(endStr, clusterStartEndDates.second);
+ Date startDate = getStartDate(startStr, endDate, clusterStartEndDates.first, frequency);
+
+ if (startDate.after(endDate)) {
+ throw new FalconException("Specified End date " + SchemaHelper.getDateFormat().format(endDate)
+ + " is before the entity was scheduled " + SchemaHelper.getDateFormat().format(startDate));
}
- return end;
+ return new Pair<Date, Date>(startDate, endDate);
}
- private void validateParams(String type, String entity, String startStr,
- String endStr) throws FalconException {
- validateNotEmpty("entityType", type);
- validateNotEmpty("entityName", entity);
- validateNotEmpty("start", startStr);
-
- Entity entityObject = EntityUtil.getEntity(type, entity);
- validateDateRange(entityObject, startStr, endStr);
+ private Date getEndDate(String endStr, Date clusterEndDate) throws FalconException {
+ Date endDate = StringUtils.isEmpty(endStr) ? new Date() : EntityUtil.parseDateUTC(endStr);
+ if (endDate.after(clusterEndDate)) {
+ endDate = clusterEndDate;
+ }
+ return endDate;
}
- private void validateDateRange(Entity entity, String start, String end) throws FalconException {
- Set<String> clusters = EntityUtil.getClustersDefined(entity);
- Pair<Date, String> clusterMinStartDate = null;
- Pair<Date, String> clusterMaxEndDate = null;
- for (String cluster : clusters) {
- if (clusterMinStartDate == null || clusterMinStartDate.first.after(
- EntityUtil.getStartTime(entity, cluster))) {
- clusterMinStartDate = Pair.of(EntityUtil.getStartTime(entity, cluster), cluster);
- }
- if (clusterMaxEndDate == null || clusterMaxEndDate.first.before(EntityUtil.getEndTime(entity, cluster))) {
- clusterMaxEndDate = Pair.of(EntityUtil.getEndTime(entity, cluster), cluster);
+ private Date getStartDate(String startStr, Date end,
+ Date clusterStartDate, Frequency frequency) throws FalconException {
+ Date start;
+ final int dateMultiplier = 10;
+ if (StringUtils.isEmpty(startStr)) {
+ // set startDate to endDate - 10 times frequency
+ long startMillis = end.getTime();
+
+ switch (frequency.getTimeUnit().getCalendarUnit()){
+ case Calendar.MINUTE :
+ startMillis -= frequency.getFrequencyAsInt() * MINUTE_IN_MILLIS * dateMultiplier;
+ break;
+
+ case Calendar.HOUR :
+ startMillis -= frequency.getFrequencyAsInt() * HOUR_IN_MILLIS * dateMultiplier;
+ break;
+
+ case Calendar.DATE :
+ startMillis -= frequency.getFrequencyAsInt() * DAY_IN_MILLIS * dateMultiplier;
+ break;
+
+ case Calendar.MONTH :
+ startMillis -= frequency.getFrequencyAsInt() * MONTH_IN_MILLIS * dateMultiplier;
+ break;
+
+ default:
+ break;
}
- }
- validateDateRangeFor(entity, clusterMinStartDate, clusterMaxEndDate,
- start, end);
- }
-
- private void validateDateRangeFor(Entity entity, Pair<Date, String> clusterMinStart,
- Pair<Date, String> clusterMaxEnd, String start,
- String end) throws FalconException{
+ start = new Date(startMillis);
+ } else {
+ start = EntityUtil.parseDateUTC(startStr);
+ }
- Date instStart = EntityUtil.parseDateUTC(start);
- if (instStart.before(clusterMinStart.first)) {
- throw new ValidationException("Start date " + start + " is before "
- + entity.getEntityType() + "'s start "
- + SchemaHelper.formatDateUTC(clusterMinStart.first)
- + " for cluster " + clusterMinStart.second);
+ if (start.before(clusterStartDate)) {
+ start = clusterStartDate;
}
- if (StringUtils.isNotEmpty(end)) {
- Date instEnd = EntityUtil.parseDateUTC(end);
- if (instStart.after(instEnd)) {
- throw new ValidationException("Start date " + start
- + " is after end date " + end);
- }
+ return start;
+ }
- if (instEnd.after(clusterMaxEnd.first)) {
- throw new ValidationException("End date " + end + " is after "
- + entity.getEntityType() + "'s end "
- + SchemaHelper.formatDateUTC(clusterMaxEnd.first)
- + " for cluster " + clusterMaxEnd.second);
- }
- } else if (instStart.after(clusterMaxEnd.first)) {
- throw new ValidationException("Start date " + start + " is after "
- + entity.getEntityType() + "'s end "
- + SchemaHelper.formatDateUTC(clusterMaxEnd.first)
- + " for cluster " + clusterMaxEnd.second);
- }
+ private void validateParams(String type, String entity) throws FalconException {
+ validateNotEmpty("entityType", type);
+ validateNotEmpty("entityName", entity);
}
private void validateNotEmpty(String field, String param) throws ValidationException {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
index 970ea74..bfce5f9 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
@@ -45,7 +45,7 @@ import java.util.*;
*/
@Path("instance")
public class InstanceManagerProxy extends AbstractInstanceManager {
-
+ private static final String DEFAULT_NUM_RESULTS = "10";
private final Map<String, Channel> processInstanceManagerChannels = new HashMap<String, Channel>();
public InstanceManagerProxy() {
@@ -85,7 +85,7 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") final String filterBy,
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") final Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
return new InstanceProxy() {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -115,7 +115,7 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") final String filterBy,
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") final Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
return new InstanceProxy() {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -141,7 +141,7 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") final String filterBy,
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") final Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
return new InstanceProxy() {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -210,7 +210,7 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") final String filterBy,
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") final Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
return new InstanceProxy() {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
index 7b8bdf5..f743005 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
@@ -301,7 +301,8 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana
@DefaultValue("") @QueryParam("tags") String tags,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS)
+ @QueryParam("numResults") Integer resultsPerPage) {
return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
index e8c54f3..2c0fa25 100644
--- a/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
+++ b/prism/src/test/java/org/apache/falcon/resource/EntityManagerTest.java
@@ -114,7 +114,7 @@ public class EntityManagerTest extends AbstractEntityManager {
Entity process2 = buildProcess("processAuthUser", System.getProperty("user.name"), "", "");
configStore.publish(EntityType.PROCESS, process2);
- EntityList entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ EntityList entityList = this.getEntityList("process", "", "", "", "", 0, 10);
Assert.assertNotNull(entityList.getElements());
Assert.assertEquals(entityList.getElements().length, 2);
@@ -123,7 +123,7 @@ public class EntityManagerTest extends AbstractEntityManager {
*/
StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
CurrentUser.authenticate(System.getProperty("user.name"));
- entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ entityList = this.getEntityList("process", "", "", "", "", 0, 10);
Assert.assertNotNull(entityList.getElements());
Assert.assertEquals(entityList.getElements().length, 2);
@@ -131,7 +131,7 @@ public class EntityManagerTest extends AbstractEntityManager {
* Only one entity should be returned when the auth is enabled.
*/
CurrentUser.authenticate("fakeUser");
- entityList = this.getEntityList("process", "", "", "", "", 0, -1);
+ entityList = this.getEntityList("process", "", "", "", "", 0, 10);
Assert.assertNotNull(entityList.getElements());
Assert.assertEquals(entityList.getElements().length, 1);
@@ -182,6 +182,15 @@ public class EntityManagerTest extends AbstractEntityManager {
entityList = this.getEntityList("process", "pipelines", "",
"consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 10, 2);
Assert.assertEquals(entityList.getElements().length, 0);
+
+ // Test negative value for numResults, should throw an exception.
+ try {
+ this.getEntityList("process", "pipelines", "",
+ "consumer=consumer@xyz.com, owner=producer@xyz.com", "name", 10, -1);
+ Assert.assertTrue(false);
+ } catch (Throwable e) {
+ Assert.assertTrue(true);
+ }
}
private Entity buildProcess(String name, String username, String tags, String pipelines) {
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
index 8762cc1..a2aaac2 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
@@ -48,7 +48,7 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") String filterBy,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
return super.getRunningInstances(type, entity, colo, lifeCycles, filterBy, orderBy, offset, resultsPerPage);
}
@@ -71,7 +71,7 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") String filterBy,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
return super.getInstances(type, entity, startStr, endStr, colo, lifeCycles,
filterBy, orderBy, offset, resultsPerPage);
}
@@ -91,7 +91,7 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") String filterBy,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles,
filterBy, orderBy, offset, resultsPerPage);
}
@@ -126,7 +126,7 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("filterBy") String filterBy,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles,
filterBy, orderBy, offset, resultsPerPage);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
index f993f81..58a3bd2 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
@@ -64,7 +64,8 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@DefaultValue("") @QueryParam("tags") String tags,
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue("-1") @QueryParam("numResults") Integer resultsPerPage) {
+ @DefaultValue(DEFAULT_NUM_RESULTS)
+ @QueryParam("numResults") Integer resultsPerPage) {
return super.getEntityList(type, fields, filterBy, tags, orderBy, offset, resultsPerPage);
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/1db8712d/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index b65aa46..4ac978c 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -405,6 +405,27 @@ public class FalconCLIIT {
+ overlay.get("outputFeedName")
+ " -start " + SchemaHelper.getDateFormat().format(new Date())
+ " -filterBy INVALID:FILTER -offset 0 -numResults 1"));
+
+ // testcase : start str is older than entity schedule time.
+ Assert.assertEquals(0,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date(10000))
+ + " -orderBy startTime -offset 0 -numResults 1"));
+ // testcase : end str is in future
+ long futureTimeinMilliSecs = (new Date()).getTime()+ 86400000;
+ Assert.assertEquals(0,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -start " + SchemaHelper.getDateFormat().format(new Date(10000))
+ + " -end " + SchemaHelper.getDateFormat().format(new Date(futureTimeinMilliSecs))
+ + " -orderBy startTime -offset 0 -numResults 1"));
+ // Both start and end dates are optional
+ Assert.assertEquals(0,
+ executeWithURL("instance -running -type feed -lifecycle eviction -name "
+ + overlay.get("outputFeedName")
+ + " -orderBy startTime -offset 0 -numResults 1"));
+
Assert.assertEquals(0,
executeWithURL("instance -status -type process -name "
+ overlay.get("processName")