You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by aj...@apache.org on 2015/07/28 11:10:07 UTC
falcon git commit: FALCON-1324 Pagination API breaks backward
compatibility. Contributed by Ajay Yadava
Repository: falcon
Updated Branches:
refs/heads/master cc80a1754 -> 6e7634e8b
FALCON-1324 Pagination API breaks backward compatibility. Contributed by Ajay Yadava
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/6e7634e8
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/6e7634e8
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/6e7634e8
Branch: refs/heads/master
Commit: 6e7634e8b4cff58894fbde5df97579e7ddfaf005
Parents: cc80a17
Author: Ajay Yadava <aj...@gmail.com>
Authored: Tue Jul 28 14:28:39 2015 +0530
Committer: Ajay Yadava <aj...@gmail.com>
Committed: Tue Jul 28 14:28:39 2015 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../main/java/org/apache/falcon/cli/FalconCLI.java | 8 ++++----
.../java/org/apache/falcon/client/FalconClient.java | 9 ++++-----
common/src/main/resources/runtime.properties | 3 ++-
.../falcon/resource/AbstractEntityManager.java | 16 ++++++++++++++--
.../falcon/resource/AbstractInstanceManager.java | 3 +--
.../falcon/resource/proxy/InstanceManagerProxy.java | 12 ++++++++----
.../proxy/SchedulableEntityManagerProxy.java | 2 +-
src/conf/runtime.properties | 8 +++++++-
.../org/apache/falcon/resource/InstanceManager.java | 12 ++++++++----
.../falcon/resource/SchedulableEntityManager.java | 2 +-
11 files changed, 52 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 45d01b0..548325f 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -9,6 +9,8 @@ Trunk (Unreleased)
FALCON-796 Enable users to triage data processing issues through falcon (Ajay Yadava)
IMPROVEMENTS
+ FALCON-1324 Pagination API breaks backward compatibility(Ajay Yadava).
+
FALCON-668 FeedReplicator improvement to include more DistCP options(Sowmya Ramesh via Ajay Yadava)
FALCON-1320 Adding equals() and hashCode() method in LineageGraphResult.Edge(Pragya Mittal via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
index e393f82..148f789 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -246,8 +246,7 @@ public class FalconCLI {
String orderBy = commandLine.getOptionValue(ORDER_BY_OPT);
String sortOrder = commandLine.getOptionValue(SORT_ORDER_OPT);
Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
- Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT),
- FalconClient.DEFAULT_NUM_RESULTS, "numResults");
+ Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT), null, "numResults");
colo = getColo(colo);
String instanceAction = "instance";
@@ -347,7 +346,8 @@ public class FalconCLI {
OUT.get().println(result);
}
- private Integer parseIntegerInput(String optionValue, int defaultVal, String optionName) throws FalconCLIException {
+ private Integer parseIntegerInput(String optionValue, Integer defaultVal, String optionName)
+ throws FalconCLIException {
Integer integer = defaultVal;
if (optionValue != null) {
try {
@@ -418,7 +418,7 @@ public class FalconCLI {
String feedInstancePath = commandLine.getOptionValue(PATH_OPT);
Integer offset = parseIntegerInput(commandLine.getOptionValue(OFFSET_OPT), 0, "offset");
Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT),
- FalconClient.DEFAULT_NUM_RESULTS, "numResults");
+ null, "numResults");
Integer numInstances = parseIntegerInput(commandLine.getOptionValue(NUM_INSTANCES_OPT), 7, "numInstances");
EntityType entityTypeEnum = null;
if (optionsList.contains(LIST_OPT)) {
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/client/src/main/java/org/apache/falcon/client/FalconClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/client/FalconClient.java b/client/src/main/java/org/apache/falcon/client/FalconClient.java
index 555bdb7..d507371 100644
--- a/client/src/main/java/org/apache/falcon/client/FalconClient.java
+++ b/client/src/main/java/org/apache/falcon/client/FalconClient.java
@@ -93,7 +93,6 @@ public class FalconClient {
private static final String TEMPLATE_SUFFIX = "-template.xml";
private static final String PROPERTIES_SUFFIX = ".properties";
- public static final int DEFAULT_NUM_RESULTS = 10;
public static final HostnameVerifier ALL_TRUSTING_HOSTNAME_VERIFIER = new HostnameVerifier() {
@Override
@@ -441,7 +440,7 @@ public class FalconClient {
String filterBy, String orderBy, String sortOrder) throws FalconCLIException {
return sendInstanceRequest(Instances.SUMMARY, type, entity, start, end, null,
- null, colo, lifeCycles, filterBy, orderBy, sortOrder, 0, DEFAULT_NUM_RESULTS)
+ null, colo, lifeCycles, filterBy, orderBy, sortOrder, 0, null)
.getEntity(InstancesSummaryResult.class);
}
@@ -450,7 +449,7 @@ public class FalconClient {
throws FalconCLIException {
return sendInstanceRequest(Instances.LISTING, type, entity, start, end, null,
- null, colo, null, "", "", "", 0, DEFAULT_NUM_RESULTS).getEntity(FeedInstanceResult.class);
+ null, colo, null, "", "", "", 0, null).getEntity(FeedInstanceResult.class);
}
public InstancesResult killInstances(String type, String entity, String start,
@@ -767,7 +766,7 @@ public class FalconClient {
String runid, String colo,
List<LifeCycle> lifeCycles) throws FalconCLIException {
return sendInstanceRequest(instances, type, entity, start, end, props,
- runid, colo, lifeCycles, "", "", "", 0, DEFAULT_NUM_RESULTS)
+ runid, colo, lifeCycles, "", "", "", 0, null)
.getEntity(InstancesResult.class);
}
@@ -776,7 +775,7 @@ public class FalconClient {
String runid, String colo, List<LifeCycle> lifeCycles,
Boolean isForced) throws FalconCLIException {
return sendInstanceRequest(instances, type, entity, start, end, props,
- runid, colo, lifeCycles, "", "", "", 0, DEFAULT_NUM_RESULTS, isForced).getEntity(InstancesResult.class);
+ runid, colo, lifeCycles, "", "", "", 0, null, isForced).getEntity(InstancesResult.class);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/common/src/main/resources/runtime.properties
----------------------------------------------------------------------
diff --git a/common/src/main/resources/runtime.properties b/common/src/main/resources/runtime.properties
index 3b32463..2159e1a 100644
--- a/common/src/main/resources/runtime.properties
+++ b/common/src/main/resources/runtime.properties
@@ -23,7 +23,8 @@
*.falcon.replication.workflow.maxmaps=5
*.falcon.replication.workflow.mapbandwidth=100
-*.webservices.default.max.results.per.page=100
+*.webservices.default.max.results.per.page=3000
+*.webservices.default.results.per.page=3000
# Default configs to handle replication for late arriving feeds.
*.feed.late.allowed=true
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
index a3801e9..478088d 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractEntityManager.java
@@ -77,7 +77,7 @@ public abstract class AbstractEntityManager {
private static MemoryLocks memoryLocks = MemoryLocks.getInstance();
protected static final int XML_DEBUG_LEN = 10 * 1024;
- protected static final String DEFAULT_NUM_RESULTS = "10";
+ protected static final Integer DEFAULT_NUM_RESULTS = getDefaultResultsPerPage();
protected static final int MAX_RESULTS = getMaxResultsPerPage();
private AbstractWorkflowEngine workflowEngine;
@@ -92,7 +92,7 @@ public abstract class AbstractEntityManager {
}
private static int getMaxResultsPerPage() {
- Integer result = 100;
+ Integer result = 3000;
final String key = "webservices.default.max.results.per.page";
String value = RuntimeProperties.get().getProperty(key, result.toString());
try {
@@ -103,6 +103,18 @@ public abstract class AbstractEntityManager {
return result;
}
+ private static int getDefaultResultsPerPage() {
+ Integer result = 3000;
+ final String key = "webservices.default.results.per.page";
+ String value = RuntimeProperties.get().getProperty(key, result.toString());
+ try {
+ result = Integer.valueOf(value);
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid value:{} for key:{} in runtime.properties", value, key);
+ }
+ return result;
+ }
+
protected void checkColo(String colo) {
if (!DeploymentUtil.getCurrentColo().equals(colo)) {
throw FalconWebException.newException(
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
index 310e73b..bf42c85 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
@@ -76,7 +76,6 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
private static final long HOUR_IN_MILLIS = 3600000L;
protected static final long DAY_IN_MILLIS = 86400000L;
private static final long MONTH_IN_MILLIS = 2592000000L;
- protected static final String DEFAULT_NUM_RESULTS = "10";
protected EntityType checkType(String type) {
if (StringUtils.isEmpty(type)) {
@@ -840,7 +839,7 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr)
throws FalconException {
- return getStartAndEndDate(entityObject, startStr, endStr, Integer.parseInt(DEFAULT_NUM_RESULTS));
+ return getStartAndEndDate(entityObject, startStr, endStr, DEFAULT_NUM_RESULTS);
}
private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr, int numResults)
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
index ac3e5db..465a636 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/InstanceManagerProxy.java
@@ -100,7 +100,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("") @QueryParam("sortOrder") final String sortOrder,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
+ @QueryParam("numResults") final Integer numResults) {
+ final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults;
return new InstanceProxy<InstancesResult>(InstancesResult.class) {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -131,7 +132,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("") @QueryParam("sortOrder") final String sortOrder,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
+ @QueryParam("numResults") Integer numResults) {
+ final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults;
return new InstanceProxy<InstancesResult>(InstancesResult.class) {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -158,7 +160,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("") @QueryParam("sortOrder") final String sortOrder,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
+ @QueryParam("numResults") final Integer numResults) {
+ final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults;
return new InstanceProxy<InstancesResult>(InstancesResult.class) {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
@@ -251,7 +254,8 @@ public class InstanceManagerProxy extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") final String orderBy,
@DefaultValue("") @QueryParam("sortOrder") final String sortOrder,
@DefaultValue("0") @QueryParam("offset") final Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") final Integer resultsPerPage) {
+ @QueryParam("numResults") final Integer numResults) {
+ final Integer resultsPerPage = numResults == null ? DEFAULT_NUM_RESULTS : numResults;
return new InstanceProxy<InstancesResult>(InstancesResult.class) {
@Override
protected InstancesResult doExecute(String colo) throws FalconException {
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
index eb281d3..b9aef71 100644
--- a/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
+++ b/prism/src/main/java/org/apache/falcon/resource/proxy/SchedulableEntityManagerProxy.java
@@ -471,11 +471,11 @@ public class SchedulableEntityManagerProxy extends AbstractSchedulableEntityMana
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("asc") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS)
@QueryParam("numResults") Integer resultsPerPage) {
if (StringUtils.isNotEmpty(type)) {
type = type.substring(1);
}
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getEntityList(fields, nameSubsequence, tagKeywords, type, tags, filterBy,
orderBy, sortOrder, offset, resultsPerPage);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/src/conf/runtime.properties
----------------------------------------------------------------------
diff --git a/src/conf/runtime.properties b/src/conf/runtime.properties
index 58dee3d..7a099ee 100644
--- a/src/conf/runtime.properties
+++ b/src/conf/runtime.properties
@@ -29,7 +29,13 @@ prism.falcon.local.endpoint=https://localhost:15443
# falcon server should have the following properties
falcon.current.colo=local
-webservices.default.max.results.per.page=100
+
+
+#default number of results per call, if numResults option is not specified then this value is used.
+webservices.default.results.per.page=3000
+# maximum number of results allowed to be returned from server, acts as an upper limit for numResults option
+webservices.default.max.results.per.page=3000
+
# retry count - to fetch the status from the workflow engine
workflow.status.retry.count=30
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
index 9c5538b..cbc936e 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/InstanceManager.java
@@ -55,7 +55,8 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
+ @QueryParam("numResults") Integer resultsPerPage) {
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getRunningInstances(type, entity, colo, lifeCycles, filterBy,
orderBy, sortOrder, offset, resultsPerPage);
}
@@ -80,7 +81,8 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
+ @QueryParam("numResults") Integer resultsPerPage) {
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getInstances(type, entity, startStr, endStr, colo, lifeCycles,
filterBy, orderBy, sortOrder, offset, resultsPerPage);
}
@@ -101,7 +103,8 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
+ @QueryParam("numResults") Integer resultsPerPage) {
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getStatus(type, entity, startStr, endStr, colo, lifeCycles,
filterBy, orderBy, sortOrder, offset, resultsPerPage);
}
@@ -155,7 +158,8 @@ public class InstanceManager extends AbstractInstanceManager {
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS) @QueryParam("numResults") Integer resultsPerPage) {
+ @QueryParam("numResults") Integer resultsPerPage) {
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getLogs(type, entity, startStr, endStr, colo, runId, lifeCycles,
filterBy, orderBy, sortOrder, offset, resultsPerPage);
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/6e7634e8/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
----------------------------------------------------------------------
diff --git a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
index ed6f44e..9a3d6be 100644
--- a/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
+++ b/webapp/src/main/java/org/apache/falcon/resource/SchedulableEntityManager.java
@@ -76,11 +76,11 @@ public class SchedulableEntityManager extends AbstractSchedulableEntityManager {
@DefaultValue("") @QueryParam("orderBy") String orderBy,
@DefaultValue("asc") @QueryParam("sortOrder") String sortOrder,
@DefaultValue("0") @QueryParam("offset") Integer offset,
- @DefaultValue(DEFAULT_NUM_RESULTS)
@QueryParam("numResults") Integer resultsPerPage) {
if (StringUtils.isNotEmpty(type)) {
type = type.substring(1);
}
+ resultsPerPage = resultsPerPage == null ? DEFAULT_NUM_RESULTS : resultsPerPage;
return super.getEntityList(fields, nameSubsequence, tagKeywords, type, tags, filterBy,
orderBy, sortOrder, offset, resultsPerPage);
}