You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by ve...@apache.org on 2014/10/27 04:55:40 UTC
[1/4] git commit: FALCON-710 start/end is optional for lifecycle
operations. Contributed by Balu Vellanki
Repository: incubator-falcon
Updated Branches:
refs/heads/master b80f1b165 -> bf959ea9f
FALCON-710 start/end is optional for lifecycle operations. Contributed by Balu Vellanki
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/cdb6b941
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/cdb6b941
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/cdb6b941
Branch: refs/heads/master
Commit: cdb6b94104c3fa24e070512cb0e66430ecae4e58
Parents: b80f1b1
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Sun Oct 26 20:05:27 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Sun Oct 26 20:05:27 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +
.../java/org/apache/falcon/cli/FalconCLI.java | 88 +++++++-------------
docs/src/site/twiki/FalconCLI.twiki | 4 +-
docs/src/site/twiki/restapi/InstanceKill.twiki | 7 +-
docs/src/site/twiki/restapi/InstanceRerun.twiki | 5 +-
.../src/site/twiki/restapi/InstanceResume.twiki | 7 +-
.../site/twiki/restapi/InstanceSuspend.twiki | 7 +-
.../resource/AbstractInstanceManager.java | 23 ++++-
.../java/org/apache/falcon/cli/FalconCLIIT.java | 31 ++++---
9 files changed, 90 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 4aca3e7..2e0960c 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -119,6 +119,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-710 start/end is optional for lifecycle operations (Balu Vellanki
+ via Venkatesh Seetharam)
+
FALCON-819 Submission of cluster with registry interface fails on secure
setup (Sowmya Ramesh via Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
index 9cf6339..89fed4f 100644
--- a/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
+++ b/client/src/main/java/org/apache/falcon/cli/FalconCLI.java
@@ -269,12 +269,20 @@ public class FalconCLI {
} else if (optionsList.contains(SUMMARY_OPT)) {
result = client.getSummaryOfInstances(type, entity, start, end, colo, lifeCycles);
} else if (optionsList.contains(KILL_OPT)) {
+ validateNotEmpty(start, START_OPT);
+ validateNotEmpty(end, END_OPT);
result = client.killInstances(type, entity, start, end, colo, clusters, sourceClusters, lifeCycles);
} else if (optionsList.contains(SUSPEND_OPT)) {
+ validateNotEmpty(start, START_OPT);
+ validateNotEmpty(end, END_OPT);
result = client.suspendInstances(type, entity, start, end, colo, clusters, sourceClusters, lifeCycles);
} else if (optionsList.contains(RESUME_OPT)) {
+ validateNotEmpty(start, START_OPT);
+ validateNotEmpty(end, END_OPT);
result = client.resumeInstances(type, entity, start, end, colo, clusters, sourceClusters, lifeCycles);
} else if (optionsList.contains(RERUN_OPT)) {
+ validateNotEmpty(start, START_OPT);
+ validateNotEmpty(end, END_OPT);
result = client.rerunInstances(type, entity, start, end, filePath, colo, clusters, sourceClusters,
lifeCycles);
} else if (optionsList.contains(CONTINUE_OPT)) {
@@ -313,17 +321,9 @@ public class FalconCLI {
String entity, String type,
String colo) throws FalconCLIException {
- if (StringUtils.isEmpty(entity)) {
- throw new FalconCLIException("Missing argument: name");
- }
-
- if (StringUtils.isEmpty(type)) {
- throw new FalconCLIException("Missing argument: type");
- }
-
- if (StringUtils.isEmpty(colo)) {
- throw new FalconCLIException("Missing argument: colo");
- }
+ validateNotEmpty(entity, ENTITY_NAME_OPT);
+ validateNotEmpty(type, ENTITY_TYPE_OPT);
+ validateNotEmpty(colo, COLO_OPT);
if (optionsList.contains(CLUSTERS_OPT)) {
if (optionsList.contains(RUNNING_OPT)
@@ -369,55 +369,55 @@ public class FalconCLI {
Integer numResults = parseIntegerInput(commandLine.getOptionValue(NUM_RESULTS_OPT),
FalconClient.DEFAULT_NUM_RESULTS, "numResults");
Integer numInstances = parseIntegerInput(commandLine.getOptionValue(NUM_INSTANCES_OPT), 7, "numInstances");
- validateEntityType(entityType);
+ validateNotEmpty(entityType, ENTITY_TYPE_OPT);
validateSortOrder(sortOrder);
String entityAction = "entity";
if (optionsList.contains(SUBMIT_OPT)) {
- validateFilePath(filePath, "file");
+ validateNotEmpty(filePath, "file");
validateColo(optionsList);
result = client.submit(entityType, filePath);
} else if (optionsList.contains(UPDATE_OPT)) {
- validateFilePath(filePath, "file");
+ validateNotEmpty(filePath, "file");
validateColo(optionsList);
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
Date effectiveTime = parseDateString(time);
result = client.update(entityType, entityName, filePath, effectiveTime);
} else if (optionsList.contains(SUBMIT_AND_SCHEDULE_OPT)) {
- validateFilePath(filePath, "file");
+ validateNotEmpty(filePath, "file");
validateColo(optionsList);
result = client.submitAndSchedule(entityType, filePath);
} else if (optionsList.contains(VALIDATE_OPT)) {
- validateFilePath(filePath, "file");
+ validateNotEmpty(filePath, "file");
validateColo(optionsList);
result = client.validate(entityType, filePath);
} else if (optionsList.contains(SCHEDULE_OPT)) {
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
colo = getColo(colo);
result = client.schedule(entityType, entityName, colo);
} else if (optionsList.contains(SUSPEND_OPT)) {
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
colo = getColo(colo);
result = client.suspend(entityType, entityName, colo);
} else if (optionsList.contains(RESUME_OPT)) {
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
colo = getColo(colo);
result = client.resume(entityType, entityName, colo);
} else if (optionsList.contains(DELETE_OPT)) {
validateColo(optionsList);
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
result = client.delete(entityType, entityName);
} else if (optionsList.contains(STATUS_OPT)) {
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
colo = getColo(colo);
result = client.getStatus(entityType, entityName, colo);
} else if (optionsList.contains(DEFINITION_OPT)) {
validateColo(optionsList);
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
result = client.getDefinition(entityType, entityName);
} else if (optionsList.contains(DEPENDENCY_OPT)) {
validateColo(optionsList);
- validateEntityName(entityName);
+ validateNotEmpty(entityName, ENTITY_NAME_OPT);
result = client.getDependency(entityType, entityName).toString();
} else if (optionsList.contains(LIST_OPT)) {
validateColo(optionsList);
@@ -428,7 +428,7 @@ public class FalconCLI {
filterTags, orderBy, sortOrder, offset, numResults);
result = entityList != null ? entityList.toString() : "No entity of type (" + entityType + ") found.";
} else if (optionsList.contains(SUMMARY_OPT)) {
- validateCluster(cluster);
+ validateNotEmpty(cluster, CLUSTER_OPT);
validateEntityFields(fields);
validateFilterBy(filterBy, entityAction);
validateOrderBy(orderBy, entityAction);
@@ -442,6 +442,12 @@ public class FalconCLI {
OUT.get().println(result);
}
+ private void validateNotEmpty(String paramVal, String paramName) throws FalconCLIException {
+ if (StringUtils.isEmpty(paramVal)) {
+ throw new FalconCLIException("Missing argument : " + paramName);
+ }
+ }
+
private void validateSortOrder(String sortOrder) throws FalconCLIException {
if (!StringUtils.isEmpty(sortOrder)) {
if (!sortOrder.equalsIgnoreCase("asc") && !sortOrder.equalsIgnoreCase("desc")) {
@@ -451,12 +457,6 @@ public class FalconCLI {
}
}
- private void validateCluster(String cluster) throws FalconCLIException {
- if (StringUtils.isEmpty(cluster)) {
- throw new FalconCLIException("Missing argument: cluster");
- }
- }
-
private String getColo(String colo) throws FalconCLIException, IOException {
if (colo == null) {
Properties prop = getClientProperties();
@@ -465,14 +465,6 @@ public class FalconCLI {
return colo;
}
- private void validateFilePath(String filePath, String argument)
- throws FalconCLIException {
-
- if (StringUtils.isEmpty(filePath)) {
- throw new FalconCLIException("Missing argument: " + argument);
- }
- }
-
private void validateColo(Set<String> optionsList)
throws FalconCLIException {
@@ -545,22 +537,6 @@ public class FalconCLI {
return null;
}
- private void validateEntityName(String entityName)
- throws FalconCLIException {
- if (StringUtils.isEmpty(entityName)) {
- throw new FalconCLIException("Missing argument: name");
- }
- }
-
- private void validateEntityType(String entityType)
- throws FalconCLIException {
-
- if (StringUtils.isEmpty(entityType)) {
- throw new FalconCLIException("Missing argument: type");
- }
- }
-
-
private Options createAdminOptions() {
Options adminOptions = new Options();
Option url = new Option(URL_OPTION, true, "Falcon URL");
@@ -1007,7 +983,7 @@ public class FalconCLI {
String recipeName = commandLine.getOptionValue(RECIPE_NAME);
String recipeToolClass = commandLine.getOptionValue(RECIPE_TOOL_CLASS_NAME);
- validateFilePath(recipeName, RECIPE_NAME);
+ validateNotEmpty(recipeName, RECIPE_NAME);
String result = client.submitRecipe(recipeName, recipeToolClass);
OUT.get().println(result);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/docs/src/site/twiki/FalconCLI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/FalconCLI.twiki b/docs/src/site/twiki/FalconCLI.twiki
index b8d36bb..01e6f1b 100644
--- a/docs/src/site/twiki/FalconCLI.twiki
+++ b/docs/src/site/twiki/FalconCLI.twiki
@@ -110,9 +110,7 @@ $FALCON_HOME/bin/falcon entity -type [cluster|feed|process] -name <<name>> -defi
Kill sub-command is used to kill all the instances of the specified process whose nominal time is between the given start time and end time.
Note:
-1. For all the instance management sub-commands, if end time is not specified, Falcon will perform the actions on all the instances whose instance time falls after the start time.
-
-2. The start time and end time needs to be specified in TZ format.
+1. The start time and end time needs to be specified in TZ format.
Example: 01 Jan 2012 01:00 => 2012-01-01T01:00Z
3. Process name is compulsory parameter for each instance management command.
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/docs/src/site/twiki/restapi/InstanceKill.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceKill.twiki b/docs/src/site/twiki/restapi/InstanceKill.twiki
index b8dbc4a..4cdd4ac 100644
--- a/docs/src/site/twiki/restapi/InstanceKill.twiki
+++ b/docs/src/site/twiki/restapi/InstanceKill.twiki
@@ -5,12 +5,13 @@
* <a href="#Examples">Examples</a>
---++ Description
-Kill a currently running instance.
+Kill currently running instance(s) of an entity.
---++ Parameters
* :entity-type can either be a feed or a process.
* :entity-name is name of the entity.
- * start start time of the entity.
+ * start is the start time of the instance(s) that you want to refer to
+ * end is the end time of the instance(s) that you want to refer to
* lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
---++ Results
@@ -19,7 +20,7 @@ Result of the kill operation.
---++ Examples
---+++ Rest Call
<verbatim>
-POST http://localhost:15000/api/instance/kill/process/SampleProcess?colo=*&start=2012-04-03T07:00Z
+POST http://localhost:15000/api/instance/kill/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&end=2014-04-03T07:00Z
</verbatim>
---+++ Result
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/docs/src/site/twiki/restapi/InstanceRerun.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceRerun.twiki b/docs/src/site/twiki/restapi/InstanceRerun.twiki
index f622480..d98ae3a 100644
--- a/docs/src/site/twiki/restapi/InstanceRerun.twiki
+++ b/docs/src/site/twiki/restapi/InstanceRerun.twiki
@@ -5,12 +5,13 @@
* <a href="#Examples">Examples</a>
---++ Description
-Rerun a specific instance of an entity.
+Rerun instances of an entity.
---++ Parameters
* :entity-type can either be a feed or a process.
* :entity-name is name of the entity.
* start is the start time of the instance that you want to refer to
+ * end is the end time of the instance that you want to refer to
* lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
---++ Results
@@ -19,7 +20,7 @@ Results of the rerun command.
---++ Examples
---+++ Rest Call
<verbatim>
-POST http://localhost:15000/api/instance/rerun/process/SampleProcess?colo=*&start=2012-04-03T07:00Z
+POST http://localhost:15000/api/instance/rerun/process/SampleProcess?colo=*&start=2013-04-03T07:00Z&end=2014-04-03T07:00Z
</verbatim>
---+++ Result
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/docs/src/site/twiki/restapi/InstanceResume.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceResume.twiki b/docs/src/site/twiki/restapi/InstanceResume.twiki
index 0ea8669..ab1d06b 100644
--- a/docs/src/site/twiki/restapi/InstanceResume.twiki
+++ b/docs/src/site/twiki/restapi/InstanceResume.twiki
@@ -5,12 +5,13 @@
* <a href="#Examples">Examples</a>
---++ Description
-Resume a specific instance of an entity.
+Resume suspended instances of an entity.
---++ Parameters
* :entity-type can either be a feed or a process.
* :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
+ * start is the start time of the instance(s) that you want to refer to
+ * end is the end time of the instance(s) that you want to refer to
* lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
---++ Results
Results of the resume command.
@@ -18,7 +19,7 @@ Results of the resume command.
---++ Examples
---+++ Rest Call
<verbatim>
-POST http://localhost:15000/api/instance/resume/process/SampleProcess?colo=*&start=2012-04-03T07:00Z
+POST http://localhost:15000/api/instance/resume/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&end=2014-04-03T07:00Z
</verbatim>
---+++ Result
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/docs/src/site/twiki/restapi/InstanceSuspend.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/restapi/InstanceSuspend.twiki b/docs/src/site/twiki/restapi/InstanceSuspend.twiki
index c2adfdc..c4404b5 100644
--- a/docs/src/site/twiki/restapi/InstanceSuspend.twiki
+++ b/docs/src/site/twiki/restapi/InstanceSuspend.twiki
@@ -5,12 +5,13 @@
* <a href="#Examples">Examples</a>
---++ Description
-Suspend a specific instance of an entity.
+Suspend instances of an entity.
---++ Parameters
* :entity-type can either be a feed or a process.
* :entity-name is name of the entity.
- * start is the start time of the instance that you want to refer to
+ * start is the start time of the instance(s) that you want to refer to
+ * end is the end time of the instance(s) that you want to refer to
* lifecycle <optional param> can be Eviction/Replication(default) for feed and Execution(default) for process.
---++ Results
@@ -19,7 +20,7 @@ Results of the suspend command.
---++ Examples
---+++ Rest Call
<verbatim>
-POST http://localhost:15000/api/instance/suspend/process/SampleProcess?colo=*&start=2012-04-03T07:00Z
+POST http://localhost:15000/api/instance/suspend/process/SampleProcess?colo=*&start=2012-04-03T07:00Z&end=2014-04-03T07:00Z
</verbatim>
---+++ Result
<verbatim>
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
index 9df6a2b..fce28ad 100644
--- a/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
+++ b/prism/src/main/java/org/apache/falcon/resource/AbstractInstanceManager.java
@@ -367,7 +367,8 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
- Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDateForLifecycleOperations(
+ entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
@@ -389,7 +390,8 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
- Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDateForLifecycleOperations(
+ entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
@@ -411,7 +413,8 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
- Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDateForLifecycleOperations(
+ entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
@@ -432,7 +435,8 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
lifeCycles = checkAndUpdateLifeCycle(lifeCycles, type);
validateParams(type, entity);
Entity entityObject = EntityUtil.getEntity(type, entity);
- Pair<Date, Date> startAndEndDate = getStartAndEndDate(entityObject, startStr, endStr);
+ Pair<Date, Date> startAndEndDate = getStartAndEndDateForLifecycleOperations(
+ entityObject, startStr, endStr);
Properties props = getProperties(request);
AbstractWorkflowEngine wfEngine = getWorkflowEngine();
@@ -456,6 +460,17 @@ public abstract class AbstractInstanceManager extends AbstractEntityManager {
return props;
}
+ private Pair<Date, Date> getStartAndEndDateForLifecycleOperations(Entity entityObject,
+ String startStr, String endStr)
+ throws FalconException {
+
+ if (StringUtils.isEmpty(startStr) || StringUtils.isEmpty(endStr)) {
+ throw new FalconException("Start and End dates cannot be empty for Instance POST apis");
+ }
+
+ return getStartAndEndDate(entityObject, startStr, endStr);
+ }
+
private Pair<Date, Date> getStartAndEndDate(Entity entityObject, String startStr, String endStr)
throws FalconException {
Pair<Date, Date> clusterStartEndDates = EntityUtil.getEntityStartEndDates(entityObject);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/cdb6b941/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
index 8cd86cc..c9b03c5 100644
--- a/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
+++ b/webapp/src/test/java/org/apache/falcon/cli/FalconCLIIT.java
@@ -573,7 +573,8 @@ public class FalconCLIIT {
+ overlay.get("processName")
+ " -start " + START_INSTANCE + " -end " + START_INSTANCE));
- Assert.assertEquals(0,
+ // No end date, should fail.
+ Assert.assertEquals(-1,
executeWithURL("instance -suspend -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
+ " -start "+ SchemaHelper.getDateFormat().format(new Date())));
@@ -586,7 +587,8 @@ public class FalconCLIIT {
Assert.assertEquals(0,
executeWithURL("instance -resume -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())));
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())));
}
private static final String START_INSTANCE = "2012-04-20T00:00Z";
@@ -611,22 +613,23 @@ public class FalconCLIIT {
+ overlay.get("processName")
+ " -start " + START_INSTANCE + " -end " + START_INSTANCE));
- Assert.assertEquals(0,
+ // Fail due to no end date
+ Assert.assertEquals(-1,
executeWithURL("instance -kill -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
+ " -start "+ SchemaHelper.getDateFormat().format(new Date())));
- Assert.assertEquals(
- 0,
+ Assert.assertEquals(0,
executeWithURL("instance -rerun -type process -name "
+ overlay.get("processName")
- + " -start " + START_INSTANCE + " -file "
- + createTempJobPropertiesFile()));
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE
+ + " -file " + createTempJobPropertiesFile()));
Assert.assertEquals(0,
executeWithURL("instance -rerun -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
+ " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end "+ SchemaHelper.getDateFormat().format(new Date())
+ " -file "+ createTempJobPropertiesFile()));
}
@@ -820,18 +823,24 @@ public class FalconCLIIT {
Assert.assertEquals(0,
executeWithURL("instance -kill -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())));
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())));
- Assert.assertEquals(
- 0,
+ Assert.assertEquals(-1,
executeWithURL("instance -continue -type process -name "
+ overlay.get("processName")
+ " -start " + START_INSTANCE));
Assert.assertEquals(0,
+ executeWithURL("instance -continue -type process -name "
+ + overlay.get("processName")
+ + " -start " + START_INSTANCE + " -end " + START_INSTANCE));
+
+ Assert.assertEquals(0,
executeWithURL("instance -continue -type feed -lifecycle eviction -name "
+ overlay.get("outputFeedName")
- + " -start "+ SchemaHelper.getDateFormat().format(new Date())));
+ + " -start "+ SchemaHelper.getDateFormat().format(new Date())
+ + " -end " + SchemaHelper.getDateFormat().format(new Date())));
}
public void testInvalidCLIInstanceCommands() throws Exception {
[2/4] git commit: FALCON-828 Add hadoop version info as part of
falcon admin version API. Contributed by Venkatesh Seetharam
Posted by ve...@apache.org.
FALCON-828 Add hadoop version info as part of falcon admin version API. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/73269d23
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/73269d23
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/73269d23
Branch: refs/heads/master
Commit: 73269d2373fde667694ff432a3ef97efa07a2219
Parents: cdb6b94
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Sun Oct 26 20:10:46 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Sun Oct 26 20:10:46 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../java/org/apache/falcon/resource/admin/AdminResource.java | 7 +++++++
2 files changed, 10 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73269d23/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2e0960c..fdeff18 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -37,6 +37,9 @@ Trunk (Unreleased)
FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
IMPROVEMENTS
+ FALCON-828 Add hadoop version info as part of falcon admin version API
+ (Venkatesh Seetharam)
+
FALCON-762 Support feed listing for file system storage (Srikanth Sundarrajan)
FALCON-20 Remove dependency on custom InMobi DistCp (Sowmya Ramesh via
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/73269d23/prism/src/main/java/org/apache/falcon/resource/admin/AdminResource.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/resource/admin/AdminResource.java b/prism/src/main/java/org/apache/falcon/resource/admin/AdminResource.java
index 41fb1c7..2798534 100644
--- a/prism/src/main/java/org/apache/falcon/resource/admin/AdminResource.java
+++ b/prism/src/main/java/org/apache/falcon/resource/admin/AdminResource.java
@@ -23,6 +23,7 @@ import org.apache.falcon.util.BuildProperties;
import org.apache.falcon.util.DeploymentProperties;
import org.apache.falcon.util.RuntimeProperties;
import org.apache.falcon.util.StartupProperties;
+import org.apache.hadoop.util.VersionInfo;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -83,9 +84,15 @@ public class AdminResource {
property.value = DeploymentProperties.get().getProperty("deploy.mode");
props.add(property);
+ property = new Property();
+ property.key = "Hadoop";
+ property.value = VersionInfo.getVersion() + "-r" + VersionInfo.getRevision();
+ props.add(property);
+
version = new PropertyList();
version.properties = props;
}
+
return version;
}
[3/4] git commit: FALCON-831 Operation on non existing entity throws
internal server error. Contributed by Venkatesh Seetharam
Posted by ve...@apache.org.
FALCON-831 Operation on non existing entity throws internal server error. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/03d3531f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/03d3531f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/03d3531f
Branch: refs/heads/master
Commit: 03d3531f16c9b2d74dcc30cc01cd9afff11db261
Parents: 73269d2
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Sun Oct 26 20:31:10 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Sun Oct 26 20:31:10 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../security/FalconAuthorizationFilter.java | 13 ++++++++---
.../security/FalconAuthorizationFilterTest.java | 23 ++++++++++++++++++++
3 files changed, 36 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/03d3531f/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index fdeff18..9261af3 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -122,6 +122,9 @@ Trunk (Unreleased)
OPTIMIZATIONS
BUG FIXES
+ FALCON-831 Operation on non existing entity throws internal server error
+ (Venkatesh Seetharam)
+
FALCON-710 start/end is optional for lifecycle operations (Balu Vellanki
via Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/03d3531f/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
index 3daa419..884bd73 100644
--- a/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuthorizationFilter.java
@@ -19,6 +19,8 @@
package org.apache.falcon.security;
import org.apache.falcon.FalconException;
+import org.apache.falcon.FalconWebException;
+import org.apache.hadoop.security.authorize.AuthorizationException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -29,6 +31,7 @@ import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.core.Response;
import java.io.IOException;
import java.util.ArrayList;
@@ -65,9 +68,13 @@ public class FalconAuthorizationFilter implements Filter {
if (isAuthorizationEnabled) {
LOG.info("Authorizing user={} against request={}", CurrentUser.getUser(), requestParts);
- authorizationProvider.authorizeResource(requestParts.getResource(),
- requestParts.getAction(), requestParts.getEntityType(),
- requestParts.getEntityName(), CurrentUser.getProxyUGI());
+ try {
+ authorizationProvider.authorizeResource(requestParts.getResource(),
+ requestParts.getAction(), requestParts.getEntityType(),
+ requestParts.getEntityName(), CurrentUser.getProxyUGI());
+ } catch (AuthorizationException e) {
+ throw FalconWebException.newException(e.getMessage(), Response.Status.UNAUTHORIZED);
+ }
}
filterChain.doFilter(request, response);
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/03d3531f/prism/src/test/java/org/apache/falcon/security/FalconAuthorizationFilterTest.java
----------------------------------------------------------------------
diff --git a/prism/src/test/java/org/apache/falcon/security/FalconAuthorizationFilterTest.java b/prism/src/test/java/org/apache/falcon/security/FalconAuthorizationFilterTest.java
index 289e232..6e4885a 100644
--- a/prism/src/test/java/org/apache/falcon/security/FalconAuthorizationFilterTest.java
+++ b/prism/src/test/java/org/apache/falcon/security/FalconAuthorizationFilterTest.java
@@ -152,6 +152,29 @@ public class FalconAuthorizationFilterTest {
}
}
+ @Test (expectedExceptions = Exception.class)
+ public void testDoFilterForEntityWithInvalidEntity() throws Exception {
+ CurrentUser.authenticate("falcon");
+ Filter filter = new FalconAuthorizationFilter();
+ synchronized (StartupProperties.get()) {
+ filter.init(mockConfig);
+ }
+
+ try {
+ StartupProperties.get().setProperty("falcon.security.authorization.enabled", "true");
+
+ String uri = "/entities/suspend/process/bad-entity";
+ StringBuffer requestUrl = new StringBuffer("http://localhost" + uri);
+ Mockito.when(mockRequest.getRequestURL()).thenReturn(requestUrl);
+ Mockito.when(mockRequest.getRequestURI()).thenReturn("/api" + uri);
+ Mockito.when(mockRequest.getPathInfo()).thenReturn(uri);
+
+ filter.doFilter(mockRequest, mockResponse, mockChain);
+ } finally {
+ filter.destroy();
+ }
+ }
+
public void addClusterEntity() throws Exception {
clusterEntity = EntityBuilderTestUtil.buildCluster(CLUSTER_ENTITY_NAME);
configStore.publish(EntityType.CLUSTER, clusterEntity);
[4/4] git commit: FALCON-834 Propagate request id in the response to
help trace and debug failures in merlin. Contributed by Venkatesh Seetharam
Posted by ve...@apache.org.
FALCON-834 Propagate request id in the response to help trace and debug failures in merlin. Contributed by Venkatesh Seetharam
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/bf959ea9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/bf959ea9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/bf959ea9
Branch: refs/heads/master
Commit: bf959ea9fa78da7ea5e652295f833473123e8fd8
Parents: 03d3531
Author: Venkatesh Seetharam <ve...@apache.org>
Authored: Sun Oct 26 20:35:13 2014 -0700
Committer: Venkatesh Seetharam <ve...@apache.org>
Committed: Sun Oct 26 20:35:13 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 3 +++
.../apache/falcon/security/FalconAuditFilter.java | 15 ++++++++++++++-
.../falcon/security/FalconAuthenticationFilter.java | 4 ----
3 files changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf959ea9/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 9261af3..6259566 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -37,6 +37,9 @@ Trunk (Unreleased)
FALCON-263 API to get workflow parameters. (pavan kumar kolamuri via Shwetha GS)
IMPROVEMENTS
+ FALCON-834 Propagate request id in the response to help trace and debug
+ failures in merlin (Venkatesh Seetharam)
+
FALCON-828 Add hadoop version info as part of falcon admin version API
(Venkatesh Seetharam)
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf959ea9/prism/src/main/java/org/apache/falcon/security/FalconAuditFilter.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuditFilter.java b/prism/src/main/java/org/apache/falcon/security/FalconAuditFilter.java
index 21e28b3..9a9b400 100644
--- a/prism/src/main/java/org/apache/falcon/security/FalconAuditFilter.java
+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuditFilter.java
@@ -21,6 +21,7 @@ package org.apache.falcon.security;
import org.apache.falcon.aspect.GenericAlert;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.util.Servlets;
+import org.apache.log4j.NDC;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -31,16 +32,21 @@ import javax.servlet.ServletException;
import javax.servlet.ServletRequest;
import javax.servlet.ServletResponse;
import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
import java.io.IOException;
import java.util.Date;
+import java.util.UUID;
/**
- * This records audit information as part of the filter after processing the request.
+ * This records audit information as part of the filter after processing the request
+ * and also introduces a UUID into request and response for tracing requests in logs.
*/
public class FalconAuditFilter implements Filter {
private static final Logger LOG = LoggerFactory.getLogger(FalconAuditFilter.class);
+ public static final String REQUEST_ID = "requestId";
+
@Override
public void init(FilterConfig filterConfig) throws ServletException {
LOG.info("FalconAuditFilter initialization started");
@@ -51,11 +57,18 @@ public class FalconAuditFilter implements Filter {
ServletResponse response,
FilterChain filterChain) throws IOException, ServletException {
final String requestTimeISO9601 = SchemaHelper.formatDateUTC(new Date());
+ // generate a unique id and shove it into NDC
+ final String requestId = UUID.randomUUID().toString();
+ NDC.push(requestId);
try {
filterChain.doFilter(request, response);
} finally {
recordAudit((HttpServletRequest) request, requestTimeISO9601);
+
+ // put the request id into the response so users can trace logs for this request
+ ((HttpServletResponse) response).setHeader(REQUEST_ID, requestId);
+ NDC.pop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/bf959ea9/prism/src/main/java/org/apache/falcon/security/FalconAuthenticationFilter.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/security/FalconAuthenticationFilter.java b/prism/src/main/java/org/apache/falcon/security/FalconAuthenticationFilter.java
index 4edde75..fa30f0e 100644
--- a/prism/src/main/java/org/apache/falcon/security/FalconAuthenticationFilter.java
+++ b/prism/src/main/java/org/apache/falcon/security/FalconAuthenticationFilter.java
@@ -42,7 +42,6 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
-import java.util.UUID;
/**
* This enforces authentication as part of the filter before processing the request.
@@ -177,9 +176,7 @@ public class FalconAuthenticationFilter
"User can't be a superuser:" + BLACK_LISTED_USERS_KEY);
} else {
try {
- String requestId = UUID.randomUUID().toString();
NDC.push(user + ":" + httpRequest.getMethod() + "/" + httpRequest.getPathInfo());
- NDC.push(requestId);
CurrentUser.authenticate(user);
LOG.info("Request from user: {}, URL={}", user,
Servlets.getRequestURI(httpRequest));
@@ -187,7 +184,6 @@ public class FalconAuthenticationFilter
filterChain.doFilter(servletRequest, servletResponse);
} finally {
NDC.pop();
- NDC.pop();
}
}
}