You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2018/07/30 13:37:01 UTC
oozie git commit: OOZIE-2718 Improve -dryrun for bundles (zhengxb2005,
asalamon74 via andras.piros)
Repository: oozie
Updated Branches:
refs/heads/master 647682940 -> 2a26670ea
OOZIE-2718 Improve -dryrun for bundles (zhengxb2005, asalamon74 via andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/2a26670e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/2a26670e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/2a26670e
Branch: refs/heads/master
Commit: 2a26670eae11c46086dcf683f0896bd278f116c5
Parents: 6476829
Author: Andras Piros <an...@cloudera.com>
Authored: Mon Jul 30 15:35:18 2018 +0200
Committer: Andras Piros <an...@cloudera.com>
Committed: Mon Jul 30 15:35:18 2018 +0200
----------------------------------------------------------------------
.../java/org/apache/oozie/cli/OozieCLI.java | 13 +++--
.../command/bundle/BundleSubmitXCommand.java | 8 ---
.../bundle/TestBundleSubmitXCommand.java | 14 +-----
.../command/coord/TestCoordUpdateXCommand.java | 5 --
docs/src/site/twiki/DG_CommandLineTool.twiki | 53 +++++++++++++++++++-
release-log.txt | 1 +
6 files changed, 63 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
index c289971..9507a70 100644
--- a/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
+++ b/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
@@ -326,8 +326,8 @@ public class OozieCLI {
Option debug = new Option(DEBUG_OPTION, false, "Use debug mode to see debugging statements on stdout");
Option rerun = new Option(RERUN_OPTION, true,
"rerun a job (coordinator requires -action or -date, bundle requires -coordinator or -date)");
- Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2) or coordinator (since 2.0) job without"
- + " actually executing it");
+ Option dryrun = new Option(DRYRUN_OPTION, false, "Dryrun a workflow (since 3.3.2), a coordinator (since 2.0) "
+ + " or a bundle (since 5.1) job without actually executing it");
Option update = new Option(UPDATE_OPTION, true, "Update coord definition and properties");
Option showdiff = new Option(SHOWDIFF_OPTION, true,
"Show diff of the new coord definition and properties with the existing one (default true)");
@@ -1020,7 +1020,14 @@ public class OozieCLI {
String dryrunStr = wc.dryrun(getConfiguration(wc, commandLine));
if (dryrunStr.equals("OK")) { // workflow
System.out.println("OK");
- } else { // coordinator
+ }
+ else if (dryrunStr.contains("<bundle-app")) {
+ // bundle
+ System.out.println("***Bundle job after parsing: ***");
+ System.out.println(dryrunStr);
+ }
+ else {
+ // coordinator
String[] dryrunStrs = dryrunStr.split("action for new instance");
int arraysize = dryrunStrs.length;
System.out.println("***coordJob after parsing: ***");
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
index f94e871..c06759e 100644
--- a/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/bundle/BundleSubmitXCommand.java
@@ -143,14 +143,6 @@ public class BundleSubmitXCommand extends SubmitTransitionXCommand {
LogUtils.setLogInfo(bundleBean);
if (dryrun) {
- Date startTime = bundleBean.getStartTime();
- long startTimeMilli = startTime.getTime();
- long endTimeMilli = startTimeMilli + (3600 * 1000);
- Date jobEndTime = bundleBean.getEndTime();
- Date endTime = new Date(endTimeMilli);
- if (endTime.compareTo(jobEndTime) > 0) {
- endTime = jobEndTime;
- }
jobId = bundleBean.getId();
LOG.info("[" + jobId + "]: Update status to PREP");
bundleBean.setStatus(Job.Status.PREP);
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
index 9c7a516..7c5f5b6 100644
--- a/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/bundle/TestBundleSubmitXCommand.java
@@ -82,17 +82,14 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
jobConf.set(OozieClient.BUNDLE_APP_PATH, appPath.toString());
BundleSubmitXCommand command = new BundleSubmitXCommand(true, jobConf);
- BundleJobBean bundleBean = (BundleJobBean) command.getJob();
- bundleBean.setStartTime(new Date());
- bundleBean.setEndTime(new Date());
command.call();
+ assertEquals(Job.Status.PREP, command.getJob().getStatus());
// result includes bundle-submit-job.xml file instead of jobId since this is a dryRun mode
String result = command.submit();
// bundle-submit-job.xml contains the Apache license but this result should not contain the comment block
assertTrue("submit result should not contain <!-- ", !result.contains("<!--"));
assertTrue("submit result should not contain --> ", !result.contains("-->"));
-
}
public void testCoordJobNameParameterization() throws Exception {
@@ -102,9 +99,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
jobConf.set("coord1.starttime", "2009-02-01T00:00Z");
BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
- final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
- bundleBean.setStartTime(new Date());
- bundleBean.setEndTime(new Date());
final String jobId = command.call();
sleep(2000);
new BundleStartXCommand(jobId).call();
@@ -129,9 +123,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
jobConf.set("coordName2", "coord");
BundleSubmitXCommand command = new BundleSubmitXCommand(true, jobConf);
- BundleJobBean bundleBean = (BundleJobBean) command.getJob();
- bundleBean.setStartTime(new Date());
- bundleBean.setEndTime(new Date());
try {
command.call();
}
@@ -148,9 +139,6 @@ public class TestBundleSubmitXCommand extends XDataTestCase {
jobConf.set("coord1.starttime", "2009-02-01T00:00Z");
BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
- final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
- bundleBean.setStartTime(new Date());
- bundleBean.setEndTime(new Date());
final String jobId = command.call();
sleep(2000);
new BundleStartXCommand(jobId).call();
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java
index 1528a52..c002df6 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordUpdateXCommand.java
@@ -25,14 +25,12 @@ import java.io.Reader;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.io.Writer;
-import java.util.Date;
import java.util.List;
import java.util.regex.Matcher;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.BundleActionBean;
-import org.apache.oozie.BundleJobBean;
import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.XException;
import org.apache.oozie.client.CoordinatorAction;
@@ -371,9 +369,6 @@ public class TestCoordUpdateXCommand extends XDataTestCase {
jobConf.set("coordName2", "coord2");
jobConf.set("isEnabled", "true");
BundleSubmitXCommand command = new BundleSubmitXCommand(jobConf);
- final BundleJobBean bundleBean = (BundleJobBean) command.getJob();
- bundleBean.setStartTime(new Date());
- bundleBean.setEndTime(new Date());
final String jobId = command.call();
sleep(2000);
new BundleStartXCommand(jobId).call();
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/docs/src/site/twiki/DG_CommandLineTool.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_CommandLineTool.twiki b/docs/src/site/twiki/DG_CommandLineTool.twiki
index d1d302a..8d33c50 100644
--- a/docs/src/site/twiki/DG_CommandLineTool.twiki
+++ b/docs/src/site/twiki/DG_CommandLineTool.twiki
@@ -53,8 +53,8 @@ oozie job <OPTIONS> : job operations
-diff <arg> Show diff of the new coord definition and properties with the
existing one (default true)
-doas <arg> doAs user, impersonates as the specified user
- -dryrun Dryrun a workflow (since 3.3.2) or coordinator (since 2.0)
- job without actually executing it
+ -dryrun Dryrun a coordinator (since 2.0), a workflow (since 3.3.2) or a bundle (since 5.1)
+ without actually executing it.
-failed re-runs the failed workflow actions of the coordinator actions (requires -rerun)
-filter <arg> <key><comparator><value>[;<key><comparator><value>]*
(All Coordinator actions satisfying the filters will be retrieved).
@@ -890,6 +890,55 @@ specified path must be an HDFS path.
If the workflow is accepted (i.e. Oozie is able to successfully read and parse it), it will return ="OK"=; otherwise, it will return
an error message describing why it was rejected.
+---+++ Dryrun of Bundle Job
+
+* This feature is only supported in Oozie 5.1 or later.
+
+Example:
+
+<verbatim>
+
+$ oozie job -oozie http://localhost:11000/oozie -dryrun -config job.properties
+***Bundle job after parsing: ***
+<bundle-app xmlns="uri:oozie:bundle:0.2" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" name="My Bundle">
+ <parameters>
+ <property>
+ <name>oozie.use.system.libpath</name>
+ <value>true</value>
+ </property>
+ </parameters>
+ <controls>
+ <kick-off-time>2017-02-14T00:13Z</kick-off-time>
+ </controls>
+ <coordinator name="My Coordinator-0">
+ <app-path>hdfs://localhost:8020/user/hue/oozie/deployments/_admin_-oozie-24-1487060026.39</app-path>
+ <configuration>
+ <property>
+ <name>wf_application_path</name>
+ <value>hdfs://localhost:8020/user/hue/oozie/workspaces/hue-oozie-1486786607.01</value>
+ </property>
+ <property>
+ <name>start_date</name>
+ <value>2017-02-14T08:12Z</value>
+ </property>
+ <property>
+ <name>end_date</name>
+ <value>2017-02-21T08:12Z</value>
+ </property>
+ </configuration>
+ </coordinator>
+</bundle-app>
+
+</verbatim>
+
+The =dryrun= option tests running a bundle job with given job properties and does not create the job.
+
+The parameters for the job must be provided in a file, either a Java Properties file (.properties) or a Hadoop XML
+Configuration file (.xml). This file must be specified with the <code>-config</code> option.
+
+If the bundle job is accepted (i.e. Oozie is able to successfully read and parse it), it will return the parsed bundle job in xml;
+otherwise, it will return an error message describing why it was rejected.
+
---+++ Updating coordinator definition and properties
Existing coordinator definition will be replaced by new definition. The refreshed coordinator would keep the same coordinator ID, state, and coordinator actions.
All created coord action(including in WAITING) will use old configuration.
http://git-wip-us.apache.org/repos/asf/oozie/blob/2a26670e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index acd5949..fb0e020 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.1.0 release (trunk - unreleased)
+OOZIE-2718 Improve -dryrun for bundles (zhengxb2005, asalamon74 via andras.piros)
OOZIE-3156 amend Retry SSH action check when cannot connect to remote host (txsing, matijhs via andras.piros)
OOZIE-3303 Oozie UI does not work after Jetty 9.3 upgrade (asalamon74 via gezapeti)
OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti)