You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/01/06 11:40:07 UTC
[2/2] falcon git commit: FALCON-1720 Rerun API does not rerun
succeeded instances (by Pavan Kolamuri)
FALCON-1720 Rerun API does not rerun succeeded instances (by Pavan Kolamuri)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/ccb6df38
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/ccb6df38
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/ccb6df38
Branch: refs/heads/master
Commit: ccb6df38bf086135d28bce9c0c92d6fd23e19459
Parents: 7cde36c
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Wed Jan 6 15:35:31 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Wed Jan 6 15:35:31 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../falcon/workflow/engine/OozieDAGEngine.java | 24 +++++++++++++++-----
.../AbstractSchedulerManagerJerseyIT.java | 2 +-
.../InstanceSchedulerManagerJerseyIT.java | 12 +++++++++-
.../local-process-noinputs-template.xml | 2 +-
5 files changed, 33 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ed1ab4..5677175 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -90,6 +90,8 @@ Proposed Release Version: 0.9
OPTIMIZATIONS
BUG FIXES
+ FALCON-1720 Rerun API does not rerun succeeded instances (Pavan Kolamuri via Pallavi Rao)
+
FALCON-1719 Retry does not update the state of the instance in the database (Pavan Kolamuri via Pallavi Rao)
FALCON-1710 dependency API sets totalResults as 0 by default(Praveen Adlakha via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java
index 1425a97..1d0e126 100644
--- a/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java
+++ b/scheduler/src/main/java/org/apache/falcon/workflow/engine/OozieDAGEngine.java
@@ -237,17 +237,29 @@ public class OozieDAGEngine implements DAGEngine {
String jobId = instance.getExternalID();
try {
WorkflowJob jobInfo = client.getJobInfo(jobId);
- Properties jobprops = OozieUtils.toProperties(jobInfo.getConf());
- if (props != null) {
- jobprops.putAll(props);
+ if (props == null) {
+ props = new Properties();
}
//if user has set any of these oozie rerun properties then force rerun flag is ignored
- if (!jobprops.containsKey(OozieClient.RERUN_FAIL_NODES)
- && !jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) {
- jobprops.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced));
+ if (!props.containsKey(OozieClient.RERUN_FAIL_NODES)
+ && !props.containsKey(OozieClient.RERUN_SKIP_NODES)) {
+ props.put(OozieClient.RERUN_FAIL_NODES, String.valueOf(!isForced));
}
+
+ Properties jobprops = OozieUtils.toProperties(jobInfo.getConf());
+ jobprops.putAll(props);
+
jobprops.remove(OozieClient.COORDINATOR_APP_PATH);
jobprops.remove(OozieClient.BUNDLE_APP_PATH);
+ // In case if both props exists one should be removed otherwise it will fail.
+ // This case will occur when user runs workflow with skip-nodes property and
+ // try to do force rerun or rerun with fail-nodes property.
+ if (jobprops.containsKey(OozieClient.RERUN_FAIL_NODES)
+ && jobprops.containsKey(OozieClient.RERUN_SKIP_NODES)) {
+ LOG.warn("Both " + OozieClient.RERUN_SKIP_NODES + " and " + OozieClient.RERUN_FAIL_NODES
+ + " are present in workflow params removing" + OozieClient.RERUN_SKIP_NODES);
+ jobprops.remove(OozieClient.RERUN_SKIP_NODES);
+ }
client.reRun(jobId, jobprops);
assertStatus(instance.getExternalID(), Job.Status.PREP, Job.Status.RUNNING, Job.Status.SUCCEEDED);
LOG.info("Rerun job {} of entity {} of time {} on cluster {}", jobId, instance.getEntity().getName(),
http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java
index f053b76..0a3e984 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/AbstractSchedulerManagerJerseyIT.java
@@ -62,7 +62,7 @@ public class AbstractSchedulerManagerJerseyIT extends FalconUnitTestBase {
private static final String DB_BASE_DIR = "target/test-data/falcondb";
protected static String dbLocation = DB_BASE_DIR + File.separator + "data.db";
protected static String url = "jdbc:derby:"+ dbLocation +";create=true";
- protected static final String DB_SQL_FILE = DB_BASE_DIR + File.separator + "out.sql";
+ protected static final String DB_SQL_FILE = dbLocation + File.separator + "out.sql";
protected LocalFileSystem localFS = new LocalFileSystem();
http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java
----------------------------------------------------------------------
diff --git a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java
index 1523b76..18c36ff 100644
--- a/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java
+++ b/webapp/src/test/java/org/apache/falcon/resource/InstanceSchedulerManagerJerseyIT.java
@@ -64,7 +64,7 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe
}
@Test
- public void testKillInstances() throws Exception {
+ public void testKillAndRerunInstances() throws Exception {
UnitTestContext context = new UnitTestContext();
Map<String, String> overlay = context.getUniqueOverlay();
@@ -84,6 +84,16 @@ public class InstanceSchedulerManagerJerseyIT extends AbstractSchedulerManagerJe
processName, START_INSTANCE);
Assert.assertEquals(status, InstancesResult.WorkflowStatus.KILLED);
+ result = falconUnitClient.rerunInstances(EntityType.PROCESS.toString(),
+ processName, START_INSTANCE, END_TIME, colo, null, null, null, null, true, null);
+ assertStatus(result);
+
+ waitForStatus(EntityType.PROCESS.name(), context.processName, START_INSTANCE,
+ InstancesResult.WorkflowStatus.RUNNING);
+ status = getClient().getInstanceStatus(EntityType.PROCESS.name(),
+ processName, START_INSTANCE);
+ Assert.assertEquals(status, InstancesResult.WorkflowStatus.RUNNING);
+
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/ccb6df38/webapp/src/test/resources/local-process-noinputs-template.xml
----------------------------------------------------------------------
diff --git a/webapp/src/test/resources/local-process-noinputs-template.xml b/webapp/src/test/resources/local-process-noinputs-template.xml
index aabdc6a..a7388d8 100644
--- a/webapp/src/test/resources/local-process-noinputs-template.xml
+++ b/webapp/src/test/resources/local-process-noinputs-template.xml
@@ -38,5 +38,5 @@
<property name="sundayThisWeek" value="${currentWeek('SUN', 0, 0)}"/>
</properties>
<workflow engine="oozie" path="##workflow.path##" lib="##workflow.lib.path##"/>
- <retry policy="periodic" delay="minutes(10)" attempts="3"/>
+ <retry policy="periodic" delay="minutes(10)" attempts="1"/>
</process>