You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2017/05/10 21:11:03 UTC
[1/9] oozie git commit: Revert "OOZIE-2873 Check El Functions before
submitting the coordinator (satishsaley)"
Repository: oozie
Updated Branches:
refs/heads/oya 40ee3200f -> 80854d151
Revert "OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley)"
This reverts commit f669852b332af78aaf658f69e785fad31a1aa392.
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3026e5ae
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3026e5ae
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3026e5ae
Branch: refs/heads/oya
Commit: 3026e5aeb51520278f2648a8b7a69e3132013b32
Parents: f669852
Author: satishsaley <sa...@apache.org>
Authored: Mon May 8 09:25:31 2017 -0700
Committer: satishsaley <sa...@apache.org>
Committed: Mon May 8 09:25:31 2017 -0700
----------------------------------------------------------------------
.../command/coord/CoordSubmitXCommand.java | 25 ---------
.../command/coord/TestCoordSubmitXCommand.java | 30 ----------
.../resources/coord-invalid-el-function.xml | 35 ------------
.../resources/coord-invalid-output-instance.xml | 58 --------------------
.../coord-multiple-input-instance4.xml | 2 +-
.../coord-multiple-input-start-instance2.xml | 2 +-
.../coord-multiple-output-instance4.xml | 2 +-
release-log.txt | 1 -
8 files changed, 3 insertions(+), 152 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
index 0c50621..969336d 100644
--- a/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
+++ b/core/src/main/java/org/apache/oozie/command/coord/CoordSubmitXCommand.java
@@ -245,7 +245,6 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
LogUtils.setLogInfo(coordJob);
if (!dryrun) {
- verifyElFunctions(coordJob);
queueMaterializeTransitionXCommand(jobId);
}
else {
@@ -1349,28 +1348,4 @@ public class CoordSubmitXCommand extends SubmitTransitionXCommand {
@Override
public void performWrites() throws CommandException {
}
-
- /**
- * Verify el functions using dryrun. Throws an exception if dry run throws
- * an exception and marks the coordJob as FAILED.
- *
- * @param coordJob
- * @throws Exception
- */
- private void verifyElFunctions(CoordinatorJobBean coordJob) throws Exception {
- Job.Status prevStatus = coordJob.getStatus();
- try {
- // dry run will catch issues with el functions if any
- getDryRun(coordJob);
- }
- catch (Exception e) {
- prevStatus = CoordinatorJob.Status.FAILED;
- throw e;
- }
- finally {
- coordJob.setStatus(prevStatus);
- coordJob.resetPending();
- }
- }
-
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
index 91b3792..dcc97ad 100644
--- a/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
+++ b/core/src/test/java/org/apache/oozie/command/coord/TestCoordSubmitXCommand.java
@@ -20,7 +20,6 @@ package org.apache.oozie.command.coord;
import java.io.File;
import java.io.FileWriter;
-import java.io.IOException;
import java.io.Reader;
import java.io.Writer;
import java.net.URI;
@@ -1570,33 +1569,4 @@ public class TestCoordSubmitXCommand extends XDataTestCase {
.contains("tmp/coord/b/2009/03"));
}
- public void testFailureOnSubmitWithInvalidInstance() throws Exception {
- testFailure("coord-invalid-output-instance.xml");
- }
-
- public void testFailureOnSubmitWithInvalidElFunction() throws Exception {
- testFailure("coord-invalid-el-function.xml");
- }
-
- private void testFailure(String coordinatorXml) throws IOException {
- Configuration conf = new XConfiguration();
- File appPathFile = new File(getTestCaseDir(), "coordinator.xml");
-
- Reader reader = IOUtils.getResourceAsReader(coordinatorXml, -1);
- Writer writer = new FileWriter(new File(getTestCaseDir(), "coordinator.xml"));
- IOUtils.copyCharStream(reader, writer);
-
- conf.set(OozieClient.COORDINATOR_APP_PATH, appPathFile.toURI().toString());
- conf.set(OozieClient.USER_NAME, getTestUser());
- CoordSubmitXCommand sc = new CoordSubmitXCommand(conf);
-
- try {
- sc.call();
- fail("Should throw an exception");
- }
- catch (CommandException e) {
- assertEquals(Job.Status.FAILED, sc.getJob().getStatus());
- assertEquals(e.getErrorCode(), ErrorCode.E0803);
- }
- }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/resources/coord-invalid-el-function.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-invalid-el-function.xml b/core/src/test/resources/coord-invalid-el-function.xml
deleted file mode 100644
index 11dc478..0000000
--- a/core/src/test/resources/coord-invalid-el-function.xml
+++ /dev/null
@@ -1,35 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}"
- start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
- <controls>
- <concurrency>2</concurrency>
- <execution>LIFO</execution>
- </controls>
- <action>
- <workflow>
- <app-path>hdfs:///tmp/workflows/</app-path>
- <configuration>
- <property>
- <name>yyyymmdd</name>
- <value>${coord:formatTime(coord:nominalTime(), 'DAY')}</value>
- </property>
- </configuration>
- </workflow>
- </action>
-</coordinator-app>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/resources/coord-invalid-output-instance.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-invalid-output-instance.xml b/core/src/test/resources/coord-invalid-output-instance.xml
deleted file mode 100644
index a90706e..0000000
--- a/core/src/test/resources/coord-invalid-output-instance.xml
+++ /dev/null
@@ -1,58 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<coordinator-app xmlns="uri:oozie:coordinator:0.2" name="NAME" frequency="${coord:days(1)}"
- start="2009-02-01T01:00Z" end="2009-02-03T23:59Z" timezone="UTC">
- <controls>
- <concurrency>2</concurrency>
- <execution>LIFO</execution>
- </controls>
- <datasets>
- <dataset name="a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
- <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
- </dataset>
- <dataset name="local_a" frequency="${coord:days(7)}" initial-instance="2009-02-01T01:00Z" timezone="UTC">
- <uri-template>file:///tmp/coord/workflows/${YEAR}/${DAY}</uri-template>
- </dataset>
- </datasets>
- <input-events>
- <data-in name="A" dataset="a">
- <instance>${coord:future(0, 1)}</instance>
- </data-in>
- </input-events>
- <!-- Invalid coordinator application coord:latest()/future(), not permitted in output-events -->
- <output-events>
- <data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:future(0,2)}</instance>
- </data-out>
- </output-events>
- <action>
- <workflow>
- <app-path>hdfs:///tmp/workflows/</app-path>
- <configuration>
- <property>
- <name>inputA</name>
- <value>${coord:dataIn('A')}</value>
- </property>
- <property>
- <name>inputB</name>
- <value>${coord:dataOut('LOCAL_A')}</value>
- </property>
- </configuration>
- </workflow>
- </action>
-</coordinator-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/resources/coord-multiple-input-instance4.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-input-instance4.xml b/core/src/test/resources/coord-multiple-input-instance4.xml
index e092465..023df26 100644
--- a/core/src/test/resources/coord-multiple-input-instance4.xml
+++ b/core/src/test/resources/coord-multiple-input-instance4.xml
@@ -35,7 +35,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:current(0)}</instance>
+ <instance>${coord:future(0,2)}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/resources/coord-multiple-input-start-instance2.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-input-start-instance2.xml b/core/src/test/resources/coord-multiple-input-start-instance2.xml
index 6865d39..83d3773 100644
--- a/core/src/test/resources/coord-multiple-input-start-instance2.xml
+++ b/core/src/test/resources/coord-multiple-input-start-instance2.xml
@@ -36,7 +36,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:current(0)}</instance>
+ <instance>${coord:future(0,2)}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/core/src/test/resources/coord-multiple-output-instance4.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/coord-multiple-output-instance4.xml b/core/src/test/resources/coord-multiple-output-instance4.xml
index 9ed4e40..32d0786 100644
--- a/core/src/test/resources/coord-multiple-output-instance4.xml
+++ b/core/src/test/resources/coord-multiple-output-instance4.xml
@@ -35,7 +35,7 @@
</input-events>
<output-events>
<data-out name="LOCAL_A" dataset="local_a">
- <instance>${coord:current(0)}</instance>
+ <instance>${coord:formatTime(coord:current(0),'yyyy-MM-dd')}</instance>
</data-out>
</output-events>
<action>
http://git-wip-us.apache.org/repos/asf/oozie/blob/3026e5ae/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 944e623..d998b2d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,6 +1,5 @@
-- Oozie 4.4.0 release (trunk - unreleased)
-OOZIE-2873 Check El Functions before submitting the coordinator (satishsaley)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)
[3/9] oozie git commit: Revert "OOZIE-2751 LocalOozieClient is
missing methods from OozieClient (abhishekbafna via rkanter)"
Posted by ge...@apache.org.
Revert "OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)"
This reverts commit 30e673e60a746f855f50bf03b74810e19130e422.
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/ffd9951f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/ffd9951f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/ffd9951f
Branch: refs/heads/oya
Commit: ffd9951f30a66ae08a59943c2db62ad6d482f437
Parents: 30e673e
Author: Robert Kanter <rk...@apache.org>
Authored: Mon May 8 15:41:30 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon May 8 15:41:30 2017 -0700
----------------------------------------------------------------------
docs/src/site/twiki/DG_QuickStart.twiki | 6 +++---
docs/src/site/twiki/ENG_Building.twiki | 6 +++---
pom.xml | 6 +++---
release-log.txt | 1 -
4 files changed, 9 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/ffd9951f/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index 7ea87c7..fdc8374 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -16,7 +16,7 @@ For detailed install and configuration instructions refer to [[AG_Install][Oozie
* Unix box (tested on Mac OS X and Linux)
* Java JDK 1.7+
* Maven 3.0.1+
- * Hadoop 2.6.0+
+ * Hadoop 0.20.2+
* Pig 0.7+
JDK commands (java, javac) must be in the command path.
@@ -45,7 +45,7 @@ available to customise the versions of the dependencies:
profile depending on the hadoop version used.
-Ptez - Bundle tez jars in hive and pig sharelibs. Useful if you want to use tez
+as the execution engine for those applications.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.6.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
@@ -70,7 +70,7 @@ More details on building Oozie can be found on the [[ENG_Building][Building Oozi
* Unix (tested in Linux and Mac OS X)
* Java 1.7+
* Hadoop
- * [[http://hadoop.apache.org][Apache Hadoop]] (tested with 1.2.1 & 2.6.0+)
+ * [[http://hadoop.apache.org][Apache Hadoop]] (tested with 1.2.1 & 2.4.0+)
* ExtJS library (optional, to enable Oozie webconsole)
* [[http://archive.cloudera.com/gplextras/misc/ext-2.2.zip][ExtJS 2.2]]
http://git-wip-us.apache.org/repos/asf/oozie/blob/ffd9951f/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index 535f1b0..019f8f1 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -11,7 +11,7 @@
* Unix box (tested on Mac OS X and Linux)
* Java JDK 1.7+
* [[http://maven.apache.org/][Maven 3.0.1+]]
- * [[http://hadoop.apache.org/core/releases.html][Hadoop 2.6.0+]]
+ * [[http://hadoop.apache.org/core/releases.html][Hadoop 0.20.2+]]
* [[http://hadoop.apache.org/pig/releases.html][Pig 0.7+]]
JDK commands (java, javac) must be in the command path.
@@ -114,7 +114,7 @@ specified in the =test.properties= file (which is loaded by the =XTestCase= clas
*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-1 or Hadoop-2) you wish to build Oozie against specifically. It will
substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. Default version is 1.2.1
-for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.6.0*.
+for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.4.0*.
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
@@ -215,7 +215,7 @@ the versions of the dependencies:
profile depending on the hadoop version used.
-Ptez - Bundle tez jars in hive and pig sharelibs. Useful if you want to use tez
as the execution engine for those applications.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.6.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/ffd9951f/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ebe1d68..5c6edf1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -78,7 +78,7 @@
</oozie.test.default.config.file>
<oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file>
- <hadoop.version>2.6.0</hadoop.version>
+ <hadoop.version>2.4.0</hadoop.version>
<hadoop.majorversion>2</hadoop.majorversion>
<hadooplib.version>hadoop-${hadoop.majorversion}-${project.version}</hadooplib.version>
<oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version>
@@ -261,7 +261,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>${hadoop.version}</version>
+ <version>2.4.0</version>
</dependency>
<!-- client -->
@@ -1957,7 +1957,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <hadoop.version>2.6.0</hadoop.version>
+ <hadoop.version>2.4.0</hadoop.version>
<hadoop.majorversion>2</hadoop.majorversion>
<pig.classifier>h2</pig.classifier>
<sqoop.classifier>hadoop200</sqoop.classifier>
http://git-wip-us.apache.org/repos/asf/oozie/blob/ffd9951f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index cb141cb..d998b2d 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,6 +1,5 @@
-- Oozie 4.4.0 release (trunk - unreleased)
-OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)
[4/9] oozie git commit: OOZIE-2751 LocalOozieClient is missing
methods from OozieClient (abhishekbafna via rkanter)
Posted by ge...@apache.org.
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java b/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
index 80c8ec4..9473821 100644
--- a/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/V1JobsServlet.java
@@ -30,27 +30,25 @@ import javax.servlet.http.HttpServletResponse;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.BaseEngineException;
import org.apache.oozie.BulkResponseInfo;
-import org.apache.oozie.BundleJobBean;
+import org.apache.oozie.BundleEngine;
+import org.apache.oozie.BundleEngineException;
import org.apache.oozie.BundleJobInfo;
import org.apache.oozie.CoordinatorEngine;
-import org.apache.oozie.BundleEngine;
import org.apache.oozie.CoordinatorEngineException;
-import org.apache.oozie.BundleEngineException;
-import org.apache.oozie.CoordinatorJobBean;
import org.apache.oozie.CoordinatorJobInfo;
import org.apache.oozie.DagEngine;
import org.apache.oozie.DagEngineException;
import org.apache.oozie.ErrorCode;
-import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.OozieJsonFactory;
import org.apache.oozie.WorkflowsInfo;
import org.apache.oozie.cli.OozieCLI;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.rest.BulkResponseImpl;
import org.apache.oozie.client.rest.JsonTags;
import org.apache.oozie.client.rest.RestConstants;
+import org.apache.oozie.service.BundleEngineService;
import org.apache.oozie.service.CoordinatorEngineService;
import org.apache.oozie.service.DagEngineService;
-import org.apache.oozie.service.BundleEngineService;
import org.apache.oozie.service.Services;
import org.apache.oozie.util.XLog;
import org.apache.oozie.util.XmlUtils;
@@ -308,12 +306,12 @@ public class V1JobsServlet extends BaseJobsServlet {
* request object
*/
private JSONObject getWorkflowJobs(HttpServletRequest request) throws XServletException {
- JSONObject json = new JSONObject();
+ JSONObject json;
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
- String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
+ String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
? "GMT" : request.getParameter(RestConstants.TIME_ZONE_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
@@ -321,12 +319,7 @@ public class V1JobsServlet extends BaseJobsServlet {
len = (len < 1) ? 50 : len;
DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(getUser(request));
WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
- List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows();
- json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows, timeZoneId));
- json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
- json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
- json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());
-
+ json = OozieJsonFactory.getWFJSONObject(jobs, timeZoneId);
}
catch (DagEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
@@ -341,12 +334,12 @@ public class V1JobsServlet extends BaseJobsServlet {
*/
@SuppressWarnings("unchecked")
private JSONObject getCoordinatorJobs(HttpServletRequest request) throws XServletException {
- JSONObject json = new JSONObject();
+ JSONObject json;
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
- String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
+ String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
? "GMT" : request.getParameter(RestConstants.TIME_ZONE_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
@@ -355,12 +348,7 @@ public class V1JobsServlet extends BaseJobsServlet {
CoordinatorEngine coordEngine = Services.get().get(CoordinatorEngineService.class).getCoordinatorEngine(
getUser(request));
CoordinatorJobInfo jobs = coordEngine.getCoordJobs(filter, start, len);
- List<CoordinatorJobBean> jsonJobs = jobs.getCoordJobs();
- json.put(JsonTags.COORDINATOR_JOBS, CoordinatorJobBean.toJSONArray(jsonJobs, timeZoneId));
- json.put(JsonTags.COORD_JOB_TOTAL, jobs.getTotal());
- json.put(JsonTags.COORD_JOB_OFFSET, jobs.getStart());
- json.put(JsonTags.COORD_JOB_LEN, jobs.getLen());
-
+ json = OozieJsonFactory.getCoordJSONObject(jobs, timeZoneId);
}
catch (CoordinatorEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
@@ -370,12 +358,12 @@ public class V1JobsServlet extends BaseJobsServlet {
@SuppressWarnings("unchecked")
private JSONObject getBundleJobs(HttpServletRequest request) throws XServletException {
- JSONObject json = new JSONObject();
+ JSONObject json;
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
String lenStr = request.getParameter(RestConstants.LEN_PARAM);
- String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
+ String timeZoneId = request.getParameter(RestConstants.TIME_ZONE_PARAM) == null
? "GMT" : request.getParameter(RestConstants.TIME_ZONE_PARAM);
int start = (startStr != null) ? Integer.parseInt(startStr) : 1;
start = (start < 1) ? 1 : start;
@@ -384,13 +372,7 @@ public class V1JobsServlet extends BaseJobsServlet {
BundleEngine bundleEngine = Services.get().get(BundleEngineService.class).getBundleEngine(getUser(request));
BundleJobInfo jobs = bundleEngine.getBundleJobs(filter, start, len);
- List<BundleJobBean> jsonJobs = jobs.getBundleJobs();
-
- json.put(JsonTags.BUNDLE_JOBS, BundleJobBean.toJSONArray(jsonJobs, timeZoneId));
- json.put(JsonTags.BUNDLE_JOB_TOTAL, jobs.getTotal());
- json.put(JsonTags.BUNDLE_JOB_OFFSET, jobs.getStart());
- json.put(JsonTags.BUNDLE_JOB_LEN, jobs.getLen());
-
+ json = OozieJsonFactory.getBundleJSONObject(jobs, timeZoneId);
}
catch (BundleEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
@@ -505,110 +487,76 @@ public class V1JobsServlet extends BaseJobsServlet {
int len = (lenStr != null) ? Integer.parseInt(lenStr) : 50;
len = (len < 1) ? 50 : len;
- JSONObject json = new JSONObject();
+ JSONObject json;
List<String> ids = new ArrayList<String>();
if (jobType.equals("wf")) {
- WorkflowsInfo jobs = null;
+ WorkflowsInfo jobs;
DagEngine dagEngine = Services.get().get(DagEngineService.class).getDagEngine(getUser(request));
- if (action.equals(RestConstants.JOB_ACTION_KILL)) {
- try {
- jobs = dagEngine.killJobs(filter, start, len);
- }
- catch (DagEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- } else if (action.equals(RestConstants.JOB_ACTION_SUSPEND)) {
- try {
- jobs = dagEngine.suspendJobs(filter, start, len);
- }
- catch (DagEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- } else if (action.equals(RestConstants.JOB_ACTION_RESUME)) {
- try {
- jobs = dagEngine.resumeJobs(filter, start, len);
- }
- catch (DagEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
+ try {
+ switch (action) {
+ case RestConstants.JOB_ACTION_KILL:
+ jobs = dagEngine.killJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_SUSPEND:
+ jobs = dagEngine.suspendJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_RESUME:
+ jobs = dagEngine.resumeJobs(filter, start, len);
+ break;
+ default:
+ throw new DagEngineException(ErrorCode.E0301, action);
}
+ } catch (DagEngineException ex) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
-
- json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jobs.getWorkflows(), timeZoneId));
- json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
- json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
- json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());
-
+ json = OozieJsonFactory.getWFJSONObject(jobs, timeZoneId);
}
else if (jobType.equals("bundle")) {
- BundleJobInfo jobs = null;
- BundleEngine bundleEngine = Services.get().get(BundleEngineService.class).
- getBundleEngine(getUser(request));
- if (action.equals(RestConstants.JOB_ACTION_KILL)) {
- try {
- jobs = bundleEngine.killJobs(filter, start, len);
- }
- catch (BundleEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- }
- else if (action.equals(RestConstants.JOB_ACTION_SUSPEND)) {
- try {
- jobs = bundleEngine.suspendJobs(filter, start, len);
- }
- catch (BundleEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- }
- else if (action.equals(RestConstants.JOB_ACTION_RESUME)) {
- try {
- jobs = bundleEngine.resumeJobs(filter, start, len);
- }
- catch (BundleEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
+ BundleJobInfo jobs;
+ BundleEngine bundleEngine = Services.get().get(BundleEngineService.class).getBundleEngine(getUser(request));
+ try {
+ switch (action) {
+ case RestConstants.JOB_ACTION_KILL:
+ jobs = bundleEngine.killJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_SUSPEND:
+ jobs = bundleEngine.suspendJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_RESUME:
+ jobs = bundleEngine.resumeJobs(filter, start, len);
+ break;
+ default:
+ throw new BundleEngineException(ErrorCode.E0301, action);
}
+ } catch (BundleEngineException ex) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
-
- json.put(JsonTags.BUNDLE_JOBS, BundleJobBean.toJSONArray(jobs.getBundleJobs(), timeZoneId));
- json.put(JsonTags.BUNDLE_JOB_TOTAL, jobs.getTotal());
- json.put(JsonTags.BUNDLE_JOB_OFFSET, jobs.getStart());
- json.put(JsonTags.BUNDLE_JOB_LEN, jobs.getLen());
+ json = OozieJsonFactory.getBundleJSONObject(jobs, timeZoneId);
}
else {
- CoordinatorJobInfo jobs = null;
+ CoordinatorJobInfo jobs;
CoordinatorEngine coordEngine = Services.get().get(CoordinatorEngineService.class).
getCoordinatorEngine(getUser(request));
- if (action.equals(RestConstants.JOB_ACTION_KILL)) {
- try {
- jobs = coordEngine.killJobs(filter, start, len);
- }
- catch (CoordinatorEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- }
- else if (action.equals(RestConstants.JOB_ACTION_SUSPEND)) {
- try {
- jobs = coordEngine.suspendJobs(filter, start, len);
- }
- catch (CoordinatorEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
+ try {
+ switch (action) {
+ case RestConstants.JOB_ACTION_KILL:
+ jobs = coordEngine.killJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_SUSPEND:
+ jobs = coordEngine.suspendJobs(filter, start, len);
+ break;
+ case RestConstants.JOB_ACTION_RESUME:
+ jobs = coordEngine.resumeJobs(filter, start, len);
+ break;
+ default:
+ throw new CoordinatorEngineException(ErrorCode.E0301, action);
}
+ } catch (CoordinatorEngineException ex) {
+ throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
}
- else if (action.equals(RestConstants.JOB_ACTION_RESUME)) {
- try {
- jobs = coordEngine.resumeJobs(filter, start, len);
- }
- catch (CoordinatorEngineException ex) {
- throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
- }
- }
-
- json.put(JsonTags.COORDINATOR_JOBS, CoordinatorJobBean.toJSONArray(jobs.getCoordJobs(), timeZoneId));
- json.put(JsonTags.COORD_JOB_TOTAL, jobs.getTotal());
- json.put(JsonTags.COORD_JOB_OFFSET, jobs.getStart());
- json.put(JsonTags.COORD_JOB_LEN, jobs.getLen());
+ json = OozieJsonFactory.getCoordJSONObject(jobs, timeZoneId);
}
-
json.put(JsonTags.JOB_IDS, toJSONArray(ids));
return json;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
index 4decd52..177bff7 100644
--- a/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
+++ b/core/src/test/java/org/apache/oozie/TestLocalOozieClientCoord.java
@@ -28,12 +28,16 @@ import java.util.List;
import java.util.NoSuchElementException;
import java.util.Properties;
+import org.apache.commons.lang.StringUtils;
import org.apache.oozie.client.CoordinatorJob;
+import org.apache.oozie.client.Job;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
import org.apache.oozie.local.LocalOozie;
import org.apache.oozie.service.Services;
import org.apache.oozie.test.XDataTestCase;
+import org.json.simple.JSONArray;
+import org.json.simple.JSONObject;
public class TestLocalOozieClientCoord extends XDataTestCase {
@@ -81,7 +85,7 @@ public class TestLocalOozieClientCoord extends XDataTestCase {
public void testHeaderMethods() {
OozieClient client = LocalOozie.getCoordClient();
client.setHeader("h", "v");
- assertNull(client.getHeader("h"));
+ assertTrue("no-op, should be null/empty", StringUtils.isBlank(client.getHeader("h")));
Iterator<String> hit = client.getHeaderNames();
assertFalse(hit.hasNext());
try {
@@ -92,7 +96,7 @@ public class TestLocalOozieClientCoord extends XDataTestCase {
// expected
}
client.removeHeader("h");
- assertNull(client.getHeader("h"));
+ assertTrue("no-op, should be null/empty", StringUtils.isBlank(client.getHeader("h")));
}
public void testGetJobsInfo() {
@@ -192,4 +196,68 @@ public class TestLocalOozieClientCoord extends XDataTestCase {
List<CoordinatorJob> list = client.getCoordJobsInfo("", 1, 5);
assertEquals(2, list.size());
}
+
+ public void testJobsOperations() throws Exception {
+ final OozieClient client = LocalOozie.getCoordClient();
+
+ // Just in case, check that there are no Coord job records left by previous tests:
+ List<CoordinatorJob> list0 = client.getCoordJobsInfo("", 1, 100);
+ assertEquals(0, list0.size());
+ Properties conf = client.createConfiguration();
+ String appPath = storedCoordAppPath();
+
+ conf.setProperty(OozieClient.COORDINATOR_APP_PATH, appPath);
+ final String jobId0 = client.run(conf);
+ final String jobId1 = client.run(conf);
+ final String jobId2 = client.run(conf);
+ waitFor(client, jobId0);
+ waitFor(client, jobId1);
+ waitFor(client, jobId2);
+ list0 = client.getCoordJobsInfo("name=NAME", 1, 10);
+ assertEquals(3, list0.size());
+
+ JSONObject jsonObject = client.suspendJobs("name=NAME", "coord", 1, 3);
+ assertEquals(3, jsonObject.get("total"));
+ assertEquals(3, ((JSONArray) jsonObject.get("coordinatorjobs")).size());
+ assertEquals(Job.Status.SUSPENDED, client.getCoordJobInfo(jobId0).getStatus());
+ assertEquals(Job.Status.SUSPENDED, client.getCoordJobInfo(jobId1).getStatus());
+ assertEquals(Job.Status.SUSPENDED, client.getCoordJobInfo(jobId2).getStatus());
+
+ jsonObject = client.resumeJobs("name=NAME", "coord", 1, 3);
+ assertEquals(3, jsonObject.get("total"));
+ assertEquals(3, ((JSONArray) jsonObject.get("coordinatorjobs")).size());
+ assertEquals(Job.Status.RUNNING, client.getCoordJobInfo(jobId0).getStatus());
+ assertEquals(Job.Status.RUNNING, client.getCoordJobInfo(jobId1).getStatus());
+ assertEquals(Job.Status.RUNNING, client.getCoordJobInfo(jobId2).getStatus());
+
+ jsonObject = client.killJobs("name=NAME", "coord", 1, 3);
+ assertEquals(3, jsonObject.get("total"));
+ assertEquals(3, ((JSONArray) jsonObject.get("coordinatorjobs")).size());
+ assertEquals(Job.Status.KILLED, client.getCoordJobInfo(jobId0).getStatus());
+ assertEquals(Job.Status.KILLED, client.getCoordJobInfo(jobId1).getStatus());
+ assertEquals(Job.Status.KILLED, client.getCoordJobInfo(jobId2).getStatus());
+ }
+
+ private void waitFor(final OozieClient client, final String jobId) {
+ waitFor(10 * 1000, new Predicate() {
+ @Override
+ public boolean evaluate() throws Exception {
+ Job.Status status = client.getCoordJobInfo(jobId).getStatus();
+ return status != Job.Status.PREP;
+ }
+ });
+ }
+
+ private String storedCoordAppPath() throws Exception {
+ String appPath = getTestCaseFileUri("coordinator.xml");
+ String appXml = "<coordinator-app name=\"NAME\" frequency=\"${coord:minutes(20)}\" "
+ + "start=\"2009-02-01T01:00Z\" end=\"2009-02-01T03:00Z\" timezone=\"UTC\" "
+ + "xmlns=\"uri:oozie:coordinator:0.1\"> <controls> <timeout>10</timeout> <concurrency>1</concurrency> "
+ + "<execution>LIFO</execution> </controls> "
+ + " <action> <workflow> "
+ + "<app-path>hdfs:///tmp/workflows/</app-path> "
+ +" </workflow> </action> </coordinator-app>";
+ writeToFile(appXml, appPath);
+ return appPath;
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d998b2d..cb141cb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)
[2/9] oozie git commit: OOZIE-2751 LocalOozieClient is missing
methods from OozieClient (abhishekbafna via rkanter)
Posted by ge...@apache.org.
OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/30e673e6
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/30e673e6
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/30e673e6
Branch: refs/heads/oya
Commit: 30e673e60a746f855f50bf03b74810e19130e422
Parents: 3026e5a
Author: Robert Kanter <rk...@apache.org>
Authored: Mon May 8 15:38:51 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon May 8 15:38:51 2017 -0700
----------------------------------------------------------------------
docs/src/site/twiki/DG_QuickStart.twiki | 6 +++---
docs/src/site/twiki/ENG_Building.twiki | 6 +++---
pom.xml | 6 +++---
release-log.txt | 1 +
4 files changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/30e673e6/docs/src/site/twiki/DG_QuickStart.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_QuickStart.twiki b/docs/src/site/twiki/DG_QuickStart.twiki
index fdc8374..7ea87c7 100644
--- a/docs/src/site/twiki/DG_QuickStart.twiki
+++ b/docs/src/site/twiki/DG_QuickStart.twiki
@@ -16,7 +16,7 @@ For detailed install and configuration instructions refer to [[AG_Install][Oozie
* Unix box (tested on Mac OS X and Linux)
* Java JDK 1.7+
* Maven 3.0.1+
- * Hadoop 0.20.2+
+ * Hadoop 2.6.0+
* Pig 0.7+
JDK commands (java, javac) must be in the command path.
@@ -45,7 +45,7 @@ available to customise the versions of the dependencies:
profile depending on the hadoop version used.
-Ptez - Bundle tez jars in hive and pig sharelibs. Useful if you want to use tez
+as the execution engine for those applications.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.6.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
@@ -70,7 +70,7 @@ More details on building Oozie can be found on the [[ENG_Building][Building Oozi
* Unix (tested in Linux and Mac OS X)
* Java 1.7+
* Hadoop
- * [[http://hadoop.apache.org][Apache Hadoop]] (tested with 1.2.1 & 2.4.0+)
+ * [[http://hadoop.apache.org][Apache Hadoop]] (tested with 1.2.1 & 2.6.0+)
* ExtJS library (optional, to enable Oozie webconsole)
* [[http://archive.cloudera.com/gplextras/misc/ext-2.2.zip][ExtJS 2.2]]
http://git-wip-us.apache.org/repos/asf/oozie/blob/30e673e6/docs/src/site/twiki/ENG_Building.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/ENG_Building.twiki b/docs/src/site/twiki/ENG_Building.twiki
index 019f8f1..535f1b0 100644
--- a/docs/src/site/twiki/ENG_Building.twiki
+++ b/docs/src/site/twiki/ENG_Building.twiki
@@ -11,7 +11,7 @@
* Unix box (tested on Mac OS X and Linux)
* Java JDK 1.7+
* [[http://maven.apache.org/][Maven 3.0.1+]]
- * [[http://hadoop.apache.org/core/releases.html][Hadoop 0.20.2+]]
+ * [[http://hadoop.apache.org/core/releases.html][Hadoop 2.6.0+]]
* [[http://hadoop.apache.org/pig/releases.html][Pig 0.7+]]
JDK commands (java, javac) must be in the command path.
@@ -114,7 +114,7 @@ specified in the =test.properties= file (which is loaded by the =XTestCase= clas
*hadoop.version* =(*)=: indicates the Hadoop version(Hadoop-1 or Hadoop-2) you wish to build Oozie against specifically. It will
substitute this value in the Oozie POM properties and pull the corresponding Hadoop artifacts from Maven. Default version is 1.2.1
-for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.4.0*.
+for Hadoop-1 (the most common case). For Hadoop-2, the version you can pass is *2.6.0*.
*generateSite* (*): generates Oozie documentation, default is undefined (no documentation is generated)
@@ -215,7 +215,7 @@ the versions of the dependencies:
profile depending on the hadoop version used.
-Ptez - Bundle tez jars in hive and pig sharelibs. Useful if you want to use tez
as the execution engine for those applications.
--Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.4.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
+-Dhadoop.version=<version> - default 1.2.1 for hadoop-1, 2.6.0 for hadoop-2 and 3.0.0-SNAPSHOT for hadoop-3
-Dhadoop.auth.version=<version> - defaults to hadoop version
-Ddistcp.version=<version> - defaults to hadoop version
-Dpig.version=<version> - default 0.16.0
http://git-wip-us.apache.org/repos/asf/oozie/blob/30e673e6/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 5c6edf1..ebe1d68 100644
--- a/pom.xml
+++ b/pom.xml
@@ -78,7 +78,7 @@
</oozie.test.default.config.file>
<oozie.test.config.file>${oozie.test.default.config.file}</oozie.test.config.file>
- <hadoop.version>2.4.0</hadoop.version>
+ <hadoop.version>2.6.0</hadoop.version>
<hadoop.majorversion>2</hadoop.majorversion>
<hadooplib.version>hadoop-${hadoop.majorversion}-${project.version}</hadooplib.version>
<oozie.hadoop-utils.version>hadoop-${hadoop.majorversion}-${project.version}</oozie.hadoop-utils.version>
@@ -261,7 +261,7 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>2.4.0</version>
+ <version>${hadoop.version}</version>
</dependency>
<!-- client -->
@@ -1957,7 +1957,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <hadoop.version>2.4.0</hadoop.version>
+ <hadoop.version>2.6.0</hadoop.version>
<hadoop.majorversion>2</hadoop.majorversion>
<pig.classifier>h2</pig.classifier>
<sqoop.classifier>hadoop200</sqoop.classifier>
http://git-wip-us.apache.org/repos/asf/oozie/blob/30e673e6/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index d998b2d..cb141cb 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2851 spelling mistakes in examples (Artem Ervits via gezapeti)
[8/9] oozie git commit: amend OOZIE-2874 Make the Launcher Mapper
map-only job's InputFormat class pluggable (andras.piros via gezapeti)
Posted by ge...@apache.org.
amend OOZIE-2874 Make the Launcher Mapper map-only job's InputFormat class pluggable (andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/11038820
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/11038820
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/11038820
Branch: refs/heads/oya
Commit: 110388208a232651fb82fa8e856541ef24aff81b
Parents: 5cb3f33
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue May 9 22:38:49 2017 -0700
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Tue May 9 22:40:17 2017 -0700
----------------------------------------------------------------------
.../hadoop/LauncherInputFormatClassLocator.java | 84 ++++++++++++++++++++
.../TestLauncherInputFormatClassLocator.java | 57 +++++++++++++
2 files changed, 141 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/11038820/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
new file mode 100644
index 0000000..4dba776
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherInputFormatClassLocator.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.util.XLog;
+
+class LauncherInputFormatClassLocator {
+ private final XLog LOG = XLog.getLog(getClass());
+
+ static final String HADOOP_INPUT_FORMAT_CLASSNAME = "mapreduce.input.format.class";
+
+ private Class<? extends InputFormat> launcherInputFormatClass;
+
+ /**
+ * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration.
+ * <p/>
+ * Priority:
+ * <ol>
+ * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li>
+ * <li><tt>OozieLauncherInputFormat</tt></li>
+ * </ol>
+ *
+ * @return
+ */
+ Class<? extends InputFormat> locateOrGet() {
+ if (launcherInputFormatClass == null) {
+ launcherInputFormatClass = locate();
+ }
+
+ return launcherInputFormatClass;
+ }
+
+ /**
+ * Get the {@code LauncherMapper} map only MR job's {@link org.apache.hadoop.mapreduce.InputFormat} according to configuration.
+ * <p/>
+ * Priority:
+ * <ol>
+ * <li><tt>oozie.action.launcher.mapreduce.input-format.class-name</tt></li>
+ * <li><tt>OozieLauncherInputFormat</tt></li>
+ * </ol>
+ *
+ * @return
+ */
+ private Class<? extends InputFormat> locate() {
+ String inputFormatClassName = OozieLauncherInputFormat.class.getSimpleName();
+
+ final String configuredClassName = ConfigurationService.get(
+ JavaActionExecutor.OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_INPUT_FORMAT_CLASSNAME);
+ if (configuredClassName != null) {
+ inputFormatClassName = configuredClassName;
+ }
+
+ try {
+ LOG.debug("Locating launcher input format class [{0}].", inputFormatClassName);
+
+ final Class inputFormatClass = Class.forName(inputFormatClassName);
+ LOG.debug("Launcher input format class [{0}] located successfully.", inputFormatClassName);
+
+ return inputFormatClass;
+ } catch (final ClassNotFoundException | ClassCastException e) {
+ LOG.warn("Could not load class [{0}], located [{1}] instead.", inputFormatClassName,
+ OozieLauncherInputFormat.class.getSimpleName());
+ return OozieLauncherInputFormat.class;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/11038820/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
new file mode 100644
index 0000000..a6ccc5a
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestLauncherInputFormatClassLocator.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.mapreduce.InputFormat;
+import org.apache.oozie.service.ConfigurationService;
+
+public class TestLauncherInputFormatClassLocator extends ActionExecutorTestCase {
+
+ public void testInputFormatClassnameNotSet() {
+ ConfigurationService.set(
+ "oozie.action.launcher.mapreduce.input.format.class",
+ "");
+
+ final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
+
+ assertEquals("default launcher input format is found when configuration is left empty",
+ OozieLauncherInputFormat.class,
+ inputFormatClass);
+ }
+
+ public void testInputFormatClassnameDefault() {
+ final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
+
+ assertEquals("default launcher input format is found when configuration is left on default",
+ OozieLauncherInputFormat.class,
+ inputFormatClass);
+ }
+
+ public void testInputFormatClassnameSet() {
+ ConfigurationService.set(
+ "oozie.action.launcher.mapreduce.input.format.class",
+ InputFormat.class.getName());
+
+ final Class inputFormatClass = new LauncherInputFormatClassLocator().locateOrGet();
+
+ assertEquals("Hadoop launcher input format is found when configuration is set",
+ InputFormat.class,
+ inputFormatClass);
+ }
+}
\ No newline at end of file
[5/9] oozie git commit: OOZIE-2751 LocalOozieClient is missing
methods from OozieClient (abhishekbafna via rkanter)
Posted by ge...@apache.org.
OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/f5554dd3
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/f5554dd3
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/f5554dd3
Branch: refs/heads/oya
Commit: f5554dd315c730286edbb333b4008e48620e2acc
Parents: ffd9951
Author: Robert Kanter <rk...@apache.org>
Authored: Mon May 8 15:44:53 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon May 8 15:44:53 2017 -0700
----------------------------------------------------------------------
.../org/apache/oozie/client/OozieClient.java | 4 +-
.../apache/oozie/client/rest/RestConstants.java | 2 +-
.../main/java/org/apache/oozie/BaseEngine.java | 18 +-
.../org/apache/oozie/BaseLocalOozieClient.java | 601 +++++++++++++++++++
.../java/org/apache/oozie/LocalOozieClient.java | 245 +-------
.../apache/oozie/LocalOozieClientBundle.java | 93 +++
.../org/apache/oozie/LocalOozieClientCoord.java | 328 ++--------
.../oozie/OozieClientOperationHandler.java | 173 ++++++
.../java/org/apache/oozie/OozieJsonFactory.java | 55 ++
.../java/org/apache/oozie/local/LocalOozie.java | 93 ++-
.../org/apache/oozie/servlet/V0JobsServlet.java | 12 +-
.../org/apache/oozie/servlet/V1JobsServlet.java | 184 ++----
.../apache/oozie/TestLocalOozieClientCoord.java | 72 ++-
release-log.txt | 1 +
14 files changed, 1233 insertions(+), 648 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/client/src/main/java/org/apache/oozie/client/OozieClient.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/OozieClient.java b/client/src/main/java/org/apache/oozie/client/OozieClient.java
index 7370808..dbb7cbd 100644
--- a/client/src/main/java/org/apache/oozie/client/OozieClient.java
+++ b/client/src/main/java/org/apache/oozie/client/OozieClient.java
@@ -66,7 +66,7 @@ import org.codehaus.jackson.type.TypeReference;
/**
- * Client API to submit and manage Oozie workflow jobs against an Oozie intance.
+ * Client API to submit and manage Oozie workflow jobs against an Oozie instance.
* <p>
* This class is thread safe.
* <p>
@@ -1978,7 +1978,7 @@ public class OozieClient {
* @param map the map
* @return the string
*/
- private String mapToString(Map<String, String> map) {
+ protected String mapToString(Map<String, String> map) {
StringBuilder sb = new StringBuilder();
Iterator<Entry<String, String>> it = map.entrySet().iterator();
while (it.hasNext()) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
index f477531..74843dc 100644
--- a/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
+++ b/client/src/main/java/org/apache/oozie/client/rest/RestConstants.java
@@ -221,7 +221,7 @@ public interface RestConstants {
String USER_PARAM = "user";
- public static final String COORD_ACTION_MISSING_DEPENDENCIES = "missing-dependencies";
+ String COORD_ACTION_MISSING_DEPENDENCIES = "missing-dependencies";
String ADMIN_PURGE = "purge";
String PURGE_WF_AGE = "wf";
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/BaseEngine.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/BaseEngine.java b/core/src/main/java/org/apache/oozie/BaseEngine.java
index 2780ec2..6a8ebff 100644
--- a/core/src/main/java/org/apache/oozie/BaseEngine.java
+++ b/core/src/main/java/org/apache/oozie/BaseEngine.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.client.CoordinatorJob;
+import org.apache.oozie.client.OozieClientException;
import org.apache.oozie.client.WorkflowJob;
import org.apache.oozie.command.CommandException;
import org.apache.oozie.executor.jpa.JPAExecutorException;
@@ -320,7 +321,7 @@ public abstract class BaseEngine {
/**
* Stream job log.
*
- * @param xLogStreamer the log streamer
+ * @param logStreamer the log streamer
* @param jobId the job id
* @param writer the writer
* @throws IOException Signals that an I/O exception has occurred.
@@ -329,4 +330,19 @@ public abstract class BaseEngine {
protected abstract void streamJobLog(XLogStreamer logStreamer, String jobId, Writer writer) throws IOException,
BaseEngineException;
+ interface BaseEngineCallable<V> {
+ V callOrThrow() throws BaseEngineException;
+ }
+
+ static <V> V callOrRethrow(final BaseEngineCallable<V> callable) throws OozieClientException {
+ try {
+ return callable.callOrThrow();
+ } catch (final BaseEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ static <V> V throwNoOp() throws OozieClientException {
+ throw new OozieClientException(ErrorCode.E0301.toString(), "no-op");
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java b/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java
new file mode 100644
index 0000000..38fb006
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/BaseLocalOozieClient.java
@@ -0,0 +1,601 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie;
+
+import com.google.common.base.Preconditions;
+import org.apache.oozie.client.BulkResponse;
+import org.apache.oozie.client.BundleJob;
+import org.apache.oozie.client.CoordinatorAction;
+import org.apache.oozie.client.CoordinatorJob;
+import org.apache.oozie.client.JMSConnectionInfo;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.OozieClientException;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.client.WorkflowJob;
+import org.apache.oozie.client.rest.RestConstants;
+import org.apache.oozie.util.XConfiguration;
+import org.json.simple.JSONObject;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.oozie.BaseEngine.BaseEngineCallable;
+import static org.apache.oozie.BaseEngine.callOrRethrow;
+import static org.apache.oozie.BaseEngine.throwNoOp;
+import static org.apache.oozie.OozieClientOperationHandler.OozieOperationJob;
+import static org.apache.oozie.OozieClientOperationHandler.OozieJobOperationCaller;
+
+abstract class BaseLocalOozieClient extends OozieClient {
+
+ private final BaseEngine engine;
+
+ BaseLocalOozieClient(final BaseEngine engine) {
+ Preconditions.checkNotNull(engine);
+ this.engine = engine;
+ }
+
+ @Override
+ public String getOozieUrl() {
+ return "localoozie";
+ }
+
+ @Override
+ public String getProtocolUrl() throws OozieClientException {
+ return "localoozie";
+ }
+
+ @Override
+ public synchronized void validateWSVersion() throws OozieClientException {
+ }
+
+ @Override
+ public Properties createConfiguration() {
+ final Properties conf = new Properties();
+ if (engine != null) {
+ conf.setProperty(USER_NAME, engine.getUser());
+ }
+ conf.setProperty(GROUP_NAME, "users");
+ return conf;
+ }
+
+ // no-operation
+ @Override
+ public void setHeader(final String name, final String value) {
+ }
+
+ // no-operation
+ @Override
+ public String getHeader(final String name) {
+ return "";
+ }
+
+ // no-operation
+ @Override
+ public void removeHeader(final String name) {
+ }
+
+ // no-operation
+ @Override
+ public Iterator<String> getHeaderNames() {
+ return Collections.<String> emptySet().iterator();
+ }
+
+ @Override
+ public Map<String, String> getHeaders() {
+ return Collections.emptyMap();
+ }
+
+ @Override
+ public String submit(final Properties conf) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.submitJob(new XConfiguration(conf), false);
+ }
+ });
+ }
+
+ @Override
+ public void start(final String jobId) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.start(jobId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public String run(final Properties conf) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.submitJob(new XConfiguration(conf), true);
+ }
+ });
+ }
+
+ @Override
+ public void reRun(final String jobId, final Properties conf) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.reRun(jobId, new XConfiguration(conf));
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void suspend(final String jobId) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.suspend(jobId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void resume(final String jobId) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.resume(jobId);
+ return null;
+ }
+ });
+
+ }
+
+ @Override
+ public void kill(final String jobId) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.kill(jobId);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public String dryrun(final Properties conf) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.dryRunSubmit(new XConfiguration(conf));
+ }
+ });
+ }
+
+ @Override
+ public String getStatus(final String jobId) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.getJobStatus(jobId);
+ }
+ });
+ }
+
+ @Override
+ public String getJobDefinition(final String jobId) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.getDefinition(jobId);
+ }
+ });
+ }
+
+ @Override
+ public String getJobId(final String externalId) throws OozieClientException {
+ return callOrRethrow(new BaseEngineCallable<String>() {
+ @Override
+ public String callOrThrow() throws BaseEngineException {
+ return engine.getJobIdForExternalId(externalId);
+ }
+ });
+ }
+
+ @Override
+ public void slaEnableAlert(final String bundleId, final String actions, final String dates, final String coords)
+ throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.enableSLAAlert(bundleId, actions, dates, coords);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void slaEnableAlert(String jobIds, String actions, String dates) throws OozieClientException {
+ slaEnableAlert(jobIds, actions, dates, null);
+ }
+
+ @Override
+ public void slaDisableAlert(final String bundleId, final String actions, final String dates, final String coords)
+ throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.disableSLAAlert(bundleId, actions, dates, coords);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void slaDisableAlert(String jobIds, String actions, String dates) throws OozieClientException {
+ slaDisableAlert(jobIds, actions, dates, null);
+ }
+
+ @Override
+ public void slaChange(final String bundleId, final String actions, final String dates, final String coords,
+ final String newSlaParams)
+ throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.changeSLA(bundleId, actions, dates, coords, newSlaParams);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public void slaChange(String jobIds, String actions, String dates, String newSlaParams) throws OozieClientException {
+ slaChange(jobIds, actions, dates, null, newSlaParams);
+ }
+
+ @Override
+ public void slaChange(String bundleId, String actions, String dates, String coords, Map<String, String> newSlaParams)
+ throws OozieClientException {
+ slaChange(bundleId, actions, dates, coords, mapToString(newSlaParams));
+ }
+
+ @Override
+ public void change(final String jobId, final String changeValue) throws OozieClientException {
+ callOrRethrow(new BaseEngineCallable<Void>() {
+ @Override
+ public Void callOrThrow() throws BaseEngineException {
+ engine.change(jobId, changeValue);
+ return null;
+ }
+ });
+ }
+
+ @Override
+ public JSONObject bulkModifyJobs(final String actionType, final String filter, final String jobType, final int start,
+ final int len)
+ throws OozieClientException {
+ final JSONObject jsonObject;
+ switch (actionType) {
+ case RestConstants.JOB_ACTION_KILL:
+ jsonObject = killJobs(filter, jobType, start, len);
+ break;
+ case RestConstants.JOB_ACTION_SUSPEND:
+ jsonObject = suspendJobs(filter, jobType, start, len);
+ break;
+ case RestConstants.JOB_ACTION_RESUME:
+ jsonObject = resumeJobs(filter, jobType, start, len);
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid actionType passed. actionType: " + actionType);
+ }
+ return jsonObject;
+ }
+
+ @Override
+ public JSONObject killJobs(final String filter, final String jobType, final int start, final int len)
+ throws OozieClientException {
+ OozieClientOperationHandler handler = new OozieClientOperationHandler(engine);
+ final OozieOperationJob operation = handler.getOperationHandler(RestConstants.JOB_ACTION_KILL, filter, start, len);
+
+
+ return callOrRethrow(new BaseEngineCallable<JSONObject>() {
+ @Override
+ public JSONObject callOrThrow() throws BaseEngineException {
+ return new OozieJobOperationCaller().call(jobType, operation);
+ }
+ });
+ }
+
+ @Override
+ public JSONObject suspendJobs(final String filter, final String jobType, final int start, final int len)
+ throws OozieClientException {
+ OozieClientOperationHandler handler = new OozieClientOperationHandler(engine);
+ final OozieOperationJob operation = handler.getOperationHandler(RestConstants.JOB_ACTION_SUSPEND, filter, start, len);
+
+ return callOrRethrow(new BaseEngineCallable<JSONObject>() {
+ @Override
+ public JSONObject callOrThrow() throws BaseEngineException {
+ return new OozieJobOperationCaller().call(jobType, operation);
+ }
+ });
+ }
+
+ @Override
+ public JSONObject resumeJobs(final String filter, final String jobType, final int start, final int len)
+ throws OozieClientException {
+ OozieClientOperationHandler handler = new OozieClientOperationHandler(engine);
+ final OozieOperationJob operation = handler.getOperationHandler(RestConstants.JOB_ACTION_RESUME, filter, start, len);
+
+ return callOrRethrow(new BaseEngineCallable<JSONObject>() {
+ @Override
+ public JSONObject callOrThrow() throws BaseEngineException {
+ return new OozieJobOperationCaller().call(jobType, operation);
+ }
+ });
+ }
+
+
+ @Override
+ public WorkflowJob getJobInfo(final String jobId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public WorkflowJob getJobInfo(final String jobId, final int start, final int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<WorkflowJob> getJobsInfo(final String filter, final int start, final int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<WorkflowJob> getJobsInfo(final String filter) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<CoordinatorAction> kill(final String jobId, final String rangeType, final String scope)
+ throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<BulkResponse> getBulkInfo(final String filter, final int start, final int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String updateCoord(String jobId, Properties conf, String dryrun, String showDiff) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String updateCoord(String jobId, String dryrun, String showDiff) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public WorkflowAction getWorkflowActionInfo(String actionId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public BundleJob getBundleJobInfo(String jobId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public CoordinatorJob getCoordJobInfo(String jobId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public CoordinatorJob getCoordJobInfo(String jobId, String filter, int start, int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public CoordinatorJob getCoordJobInfo(String jobId, String filter, int start, int len, String order)
+ throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<WorkflowJob> getWfsForCoordAction(String coordActionId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public CoordinatorAction getCoordActionInfo(String actionId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh, boolean noCleanup)
+ throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<CoordinatorAction> reRunCoord(String jobId, String rerunType, String scope, boolean refresh,
+ boolean noCleanup, boolean failed, Properties props) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Void reRunBundle(String jobId, String coordScope, String dateScope, boolean refresh, boolean noCleanup)
+ throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<CoordinatorJob> getCoordJobsInfo(String filter, int start, int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<BundleJob> getBundleJobsInfo(String filter, int start, int len) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ protected HttpURLConnection createRetryableConnection(URL url, String method) throws IOException {
+ try {
+ return throwNoOp();
+ } catch (OozieClientException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ protected HttpURLConnection createConnection(URL url, String method) throws IOException, OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public List<CoordinatorAction> ignore(String jobId, String scope) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public JMSConnectionInfo getJMSConnectionInfo() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String getJobLog(String jobId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public void getJobAuditLog(String jobId, PrintStream ps) throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public void getJobLog(String jobId, String logRetrievalType, String logRetrievalScope, String logFilter, PrintStream ps)
+ throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public void getJobErrorLog(String jobId, PrintStream ps) throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public void getJobLog(String jobId, String logRetrievalType, String logRetrievalScope, PrintStream ps)
+ throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public String getJMSTopicName(String jobId) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public void getSlaInfo(int start, int len, String filter) throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public void setSystemMode(SYSTEM_MODE status) throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public SYSTEM_MODE getSystemMode() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String updateShareLib() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String listShareLib(String sharelibKey) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String getServerBuildVersion() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public String getClientBuildVersion() {
+ throw new UnsupportedOperationException("Operation not supported.");
+ }
+
+ @Override
+ public String validateXML(String file) throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public void pollJob(String id, int timeout, int interval, boolean verbose) throws OozieClientException {
+ throwNoOp();
+ }
+
+ @Override
+ public List<String> getQueueDump() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Map<String, String> getAvailableOozieServers() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Map<String, String> getServerConfiguration() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Map<String, String> getJavaSystemProperties() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Map<String, String> getOSEnv() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Metrics getMetrics() throws OozieClientException {
+ return throwNoOp();
+ }
+
+ @Override
+ public Instrumentation getInstrumentation() throws OozieClientException {
+ return throwNoOp();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/LocalOozieClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/LocalOozieClient.java b/core/src/main/java/org/apache/oozie/LocalOozieClient.java
index f734f76..123725f 100644
--- a/core/src/main/java/org/apache/oozie/LocalOozieClient.java
+++ b/core/src/main/java/org/apache/oozie/LocalOozieClient.java
@@ -18,18 +18,14 @@
package org.apache.oozie;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Properties;
-
-import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.OozieClientException;
+import org.apache.oozie.client.WorkflowAction;
import org.apache.oozie.client.WorkflowJob;
-import org.apache.oozie.util.XConfiguration;
+
+import java.util.List;
/**
- * Client API to submit and manage Oozie workflow jobs against an Oozie intance. <p> This class is thread safe. <p>
+ * Client API to submit and manage Oozie workflow jobs against an Oozie instance. <p> This class is thread safe. <p>
* Syntax for filter for the {@link #getJobsInfo(String)} {@link #getJobsInfo(String, int, int)} methods:
* <code>[NAME=VALUE][;NAME=VALUE]*</code>. <p> Valid filter names are: <p> <ul> <li>name: the workflow application
* name from the workflow definition.</li> <li>user: the user that submitted the job.</li> <li>group: the group for the
@@ -37,7 +33,7 @@ import org.apache.oozie.util.XConfiguration;
* query will do an OR among all the filter values for the same name. Multiple values must be specified as different
* name value pairs.
*/
-public class LocalOozieClient extends OozieClient {
+public class LocalOozieClient extends BaseLocalOozieClient {
private DagEngine dagEngine;
@@ -47,217 +43,11 @@ public class LocalOozieClient extends OozieClient {
* @param dagEngine the dag engine instance to use.
*/
public LocalOozieClient(DagEngine dagEngine) {
+ super(dagEngine);
this.dagEngine = dagEngine;
}
/**
- * Return the Oozie URL of the workflow client instance. <p> This URL is the base URL fo the Oozie system, with not
- * protocol versioning.
- *
- * @return the Oozie URL of the workflow client instance.
- */
- @Override
- public String getOozieUrl() {
- return "localoozie";
- }
-
- /**
- * Return the Oozie URL used by the client and server for WS communications. <p> This URL is the original URL plus
- * the versioning element path.
- *
- * @return the Oozie URL used by the client and server for communication.
- * @throws org.apache.oozie.client.OozieClientException thrown in the client and the server are not protocol
- * compatible.
- */
- @Override
- public String getProtocolUrl() throws OozieClientException {
- return "localoozie";
- }
-
- /**
- * Validate that the Oozie client and server instances are protocol compatible.
- *
- * @throws org.apache.oozie.client.OozieClientException thrown in the client and the server are not protocol
- * compatible.
- */
- @Override
- public synchronized void validateWSVersion() throws OozieClientException {
- }
-
- /**
- * Create an empty configuration with just the {@link #USER_NAME} set to the JVM user name and the {@link
- * #GROUP_NAME} set to 'other'.
- *
- * @return an empty configuration.
- */
- @Override
- public Properties createConfiguration() {
- Properties conf = new Properties();
- if (dagEngine != null) {
- conf.setProperty(USER_NAME, dagEngine.getUser());
- }
- conf.setProperty(GROUP_NAME, "users");
- return conf;
- }
-
- /**
- * Set a HTTP header to be used in the WS requests by the workflow instance.
- *
- * @param name header name.
- * @param value header value.
- */
- @Override
- public void setHeader(String name, String value) {
- }
-
- /**
- * Get the value of a set HTTP header from the workflow instance.
- *
- * @param name header name.
- * @return header value, <code>null</code> if not set.
- */
- @Override
- public String getHeader(String name) {
- return null;
- }
-
- /**
- * Remove a HTTP header from the workflow client instance.
- *
- * @param name header name.
- */
- @Override
- public void removeHeader(String name) {
- }
-
- /**
- * Return an iterator with all the header names set in the workflow instance.
- *
- * @return header names.
- */
- @Override
- @SuppressWarnings("unchecked")
- public Iterator<String> getHeaderNames() {
- return Collections.EMPTY_SET.iterator();
- }
-
-
- /**
- * Submit a workflow job.
- *
- * @param conf job configuration.
- * @return the job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be submitted.
- */
- @Override
- public String submit(Properties conf) throws OozieClientException {
- try {
- return dagEngine.submitJob(new XConfiguration(conf), false);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Start a workflow job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be started.
- */
- @Override
- public void start(String jobId) throws OozieClientException {
- try {
- dagEngine.start(jobId);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Submit and start a workflow job.
- *
- * @param conf job configuration.
- * @return the job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be submitted.
- */
- @Override
- public String run(Properties conf) throws OozieClientException {
- try {
- return dagEngine.submitJob(new XConfiguration(conf), true);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Rerun a workflow job.
- *
- * @param jobId job Id to rerun.
- * @param conf configuration information for the rerun.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be started.
- */
- @Override
- public void reRun(String jobId, Properties conf) throws OozieClientException {
- try {
- dagEngine.reRun(jobId, new XConfiguration(conf));
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Suspend a workflow job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be suspended.
- */
- @Override
- public void suspend(String jobId) throws OozieClientException {
- try {
- dagEngine.suspend(jobId);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Resume a workflow job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be resume.
- */
- @Override
- public void resume(String jobId) throws OozieClientException {
- try {
- dagEngine.resume(jobId);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Kill a workflow job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job could not be killed.
- */
- @Override
- public void kill(String jobId) throws OozieClientException {
- try {
- dagEngine.kill(jobId);
- }
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
* Get the info of a workflow job.
*
* @param jobId job Id.
@@ -306,20 +96,21 @@ public class LocalOozieClient extends OozieClient {
return getJobsInfo(filter, 1, 100);
}
- /**
- * Return the workflow job Id for an external Id. <p> The external Id must have provided at job creation time.
- *
- * @param externalId external Id given at job creation time.
- * @return the workflow job Id for an external Id, <code>null</code> if none.
- * @throws org.apache.oozie.client.OozieClientException thrown if the operation could not be done.
- */
@Override
- public String getJobId(String externalId) throws OozieClientException {
+ public WorkflowJob getJobInfo(String jobId, int start, int len) throws OozieClientException {
try {
- return dagEngine.getJobIdForExternalId(externalId);
+ return dagEngine.getJob(jobId, start, len);
+ } catch (DagEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
}
- catch (DagEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
+ }
+
+ @Override
+ public WorkflowAction getWorkflowActionInfo(String actionId) throws OozieClientException {
+ try {
+ return dagEngine.getWorkflowAction(actionId);
+ } catch (BaseEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/LocalOozieClientBundle.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/LocalOozieClientBundle.java b/core/src/main/java/org/apache/oozie/LocalOozieClientBundle.java
new file mode 100644
index 0000000..3c229be
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/LocalOozieClientBundle.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie;
+
+import org.apache.oozie.client.BulkResponse;
+import org.apache.oozie.client.BundleJob;
+import org.apache.oozie.client.OozieClientException;
+
+import java.util.List;
+
+/**
+ * Client API to submit and manage Oozie Bundle jobs against an Oozie instance.
+ * <p>
+ * This class is thread safe.
+ * <p>
+ * Syntax for filter for the {@link #getJobsInfo(String)} {@link #getJobsInfo(String, int, int)} methods:
+ * <code>[NAME=VALUE][;NAME=VALUE]*</code>.
+ * <p>
+ * Valid filter names are:
+ * <p>
+ * <ul>
+ * <li>name: the bundle application name from the bundle definition.</li>
+ * <li>user: the user that submitted the job.</li>
+ * <li>group: the group for the job.</li>
+ * <li>status: the status of the job.</li>
+ * </ul>
+ * <p>
+ * The query will do an AND among all the filter names. The query will do an OR among all the filter values for the same
+ * name. Multiple values must be specified as different name value pairs.
+ */
+public class LocalOozieClientBundle extends BaseLocalOozieClient {
+
+ private final BundleEngine bundleEngine;
+
+ public LocalOozieClientBundle(BundleEngine bundleEngine) {
+ super(bundleEngine);
+ this.bundleEngine = bundleEngine;
+ }
+
+ @Override
+ public BundleJob getBundleJobInfo(String jobId) throws OozieClientException {
+ try {
+ return bundleEngine.getBundleJob(jobId);
+ } catch (BundleEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public Void reRunBundle(String jobId, String coordScope, String dateScope, boolean refresh, boolean noCleanup)
+ throws OozieClientException {
+ try {
+ bundleEngine.reRun(jobId, coordScope, dateScope, refresh, noCleanup);
+ } catch (BaseEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ return null;
+ }
+
+ @Override
+ public List<BundleJob> getBundleJobsInfo(String filter, int start, int len) throws OozieClientException {
+ try {
+ return (List) bundleEngine.getBundleJobs(filter, start, len).getBundleJobs();
+ } catch (BundleEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public List<BulkResponse> getBulkInfo(String filter, int start, int len) throws OozieClientException {
+ try {
+ return (List) bundleEngine.getBulkJobs(filter, start, len).getResponses();
+ } catch (BundleEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java b/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java
index 32b0cd0..d1e6972 100644
--- a/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java
+++ b/core/src/main/java/org/apache/oozie/LocalOozieClientCoord.java
@@ -19,8 +19,6 @@
package org.apache.oozie;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.Properties;
@@ -36,7 +34,7 @@ import org.apache.oozie.util.XConfiguration;
/**
* Client API to submit and manage Oozie coordinator jobs against an Oozie
- * intance.
+ * instance.
* <p>
* This class is thread safe.
* <p>
@@ -57,7 +55,7 @@ import org.apache.oozie.util.XConfiguration;
* among all the filter values for the same name. Multiple values must be
* specified as different name value pairs.
*/
-public class LocalOozieClientCoord extends OozieClient {
+public class LocalOozieClientCoord extends BaseLocalOozieClient {
private final CoordinatorEngine coordEngine;
@@ -68,178 +66,11 @@ public class LocalOozieClientCoord extends OozieClient {
* @param coordEngine the engine instance to use.
*/
public LocalOozieClientCoord(CoordinatorEngine coordEngine) {
+ super(coordEngine);
this.coordEngine = coordEngine;
}
/**
- * Return the Oozie URL of the coordinator client instance.
- * <p>
- * This URL is the base URL fo the Oozie system, with not protocol
- * versioning.
- *
- * @return the Oozie URL of the coordinator client instance.
- */
- @Override
- public String getOozieUrl() {
- return "localoozie";
- }
-
- /**
- * Return the Oozie URL used by the client and server for WS communications.
- * <p>
- * This URL is the original URL plus the versioning element path.
- *
- * @return the Oozie URL used by the client and server for communication.
- * @throws org.apache.oozie.client.OozieClientException thrown in the client
- * and the server are not protocol compatible.
- */
- @Override
- public String getProtocolUrl() throws OozieClientException {
- return "localoozie";
- }
-
- /**
- * Validate that the Oozie client and server instances are protocol
- * compatible.
- *
- * @throws org.apache.oozie.client.OozieClientException thrown in the client
- * and the server are not protocol compatible.
- */
- @Override
- public synchronized void validateWSVersion() throws OozieClientException {
- }
-
- /**
- * Create an empty configuration with just the {@link #USER_NAME} set to the
- * JVM user name and the {@link #GROUP_NAME} set to 'other'.
- *
- * @return an empty configuration.
- */
- @Override
- public Properties createConfiguration() {
- Properties conf = new Properties();
- if (coordEngine != null) {
- conf.setProperty(USER_NAME, coordEngine.getUser());
- }
- conf.setProperty(GROUP_NAME, "users");
- return conf;
- }
-
- /**
- * Set a HTTP header to be used in the WS requests by the coordinator
- * instance.
- *
- * @param name header name.
- * @param value header value.
- */
- @Override
- public void setHeader(String name, String value) {
- }
-
- /**
- * Get the value of a set HTTP header from the coordinator instance.
- *
- * @param name header name.
- * @return header value, <code>null</code> if not set.
- */
- @Override
- public String getHeader(String name) {
- return null;
- }
-
- /**
- * Remove a HTTP header from the coordinator client instance.
- *
- * @param name header name.
- */
- @Override
- public void removeHeader(String name) {
- }
-
- /**
- * Return an iterator with all the header names set in the coordinator
- * instance.
- *
- * @return header names.
- */
- @Override
- @SuppressWarnings("unchecked")
- public Iterator<String> getHeaderNames() {
- return Collections.EMPTY_SET.iterator();
- }
-
- /**
- * Submit a coordinator job.
- *
- * @param conf job configuration.
- * @return the job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be submitted.
- */
- @Override
- public String submit(Properties conf) throws OozieClientException {
- try {
- return coordEngine.submitJob(new XConfiguration(conf), false);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Start a coordinator job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be started.
- */
- @Override
- @Deprecated
- public void start(String jobId) throws OozieClientException {
- try {
- coordEngine.start(jobId);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- catch (BaseEngineException bex) {
- throw new OozieClientException(bex.getErrorCode().toString(), bex);
- }
- }
-
- /**
- * Submit and start a coordinator job.
- *
- * @param conf job configuration.
- * @return the job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be submitted.
- */
- @Override
- public String run(Properties conf) throws OozieClientException {
- try {
- return coordEngine.submitJob(new XConfiguration(conf), true);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Rerun a workflow job.
- *
- * @param jobId job Id to rerun.
- * @param conf configuration information for the rerun.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be started.
- */
- @Override
- @Deprecated
- public void reRun(String jobId, Properties conf) throws OozieClientException {
- throw new OozieClientException(ErrorCode.E0301.toString(), "no-op");
- }
-
- /**
* Rerun coordinator actions.
*
* @param jobId coordinator jobId
@@ -310,71 +141,6 @@ public class LocalOozieClientCoord extends OozieClient {
}
/**
- * Suspend a coordinator job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be suspended.
- */
- @Override
- public void suspend(String jobId) throws OozieClientException {
- try {
- coordEngine.suspend(jobId);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Resume a coordinator job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be resume.
- */
- @Override
- public void resume(String jobId) throws OozieClientException {
- try {
- coordEngine.resume(jobId);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Kill a coordinator job.
- *
- * @param jobId job Id.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * could not be killed.
- */
- @Override
- public void kill(String jobId) throws OozieClientException {
- try {
- coordEngine.kill(jobId);
- }
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
- }
-
- /**
- * Get the info of a workflow job.
- *
- * @param jobId job Id.
- * @return the job info.
- * @throws org.apache.oozie.client.OozieClientException thrown if the job
- * info could not be retrieved.
- */
- @Override
- @Deprecated
- public WorkflowJob getJobInfo(String jobId) throws OozieClientException {
- throw new OozieClientException(ErrorCode.E0301.toString(), "no-op");
- }
-
- /**
* Get the info of a coordinator job.
*
* @param jobId job Id.
@@ -412,9 +178,6 @@ public class LocalOozieClientCoord extends OozieClient {
try {
return coordEngine.getCoordJob(jobId, filter, start, len, false);
}
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
catch (BaseEngineException bex) {
throw new OozieClientException(bex.getErrorCode().toString(), bex);
}
@@ -433,32 +196,12 @@ public class LocalOozieClientCoord extends OozieClient {
try {
return coordEngine.getCoordAction(actionId);
}
- catch (CoordinatorEngineException ex) {
- throw new OozieClientException(ex.getErrorCode().toString(), ex);
- }
catch (BaseEngineException bex) {
throw new OozieClientException(bex.getErrorCode().toString(), bex);
}
}
/**
- * Return the info of the workflow jobs that match the filter.
- *
- * @param filter job filter. Refer to the {@link OozieClient} for the filter
- * syntax.
- * @param start jobs offset, base 1.
- * @param len number of jobs to return.
- * @return a list with the workflow jobs info, without node details.
- * @throws OozieClientException thrown if the jobs info could not be
- * retrieved.
- */
- @Override
- @Deprecated
- public List<WorkflowJob> getJobsInfo(String filter, int start, int len) throws OozieClientException {
- throw new OozieClientException(ErrorCode.E0301.toString(), "no-op");
- }
-
- /**
* Return the info of the coordinator jobs that match the filter.
*
* @param filter job filter. Refer to the {@link OozieClient} for the filter
@@ -486,21 +229,58 @@ public class LocalOozieClientCoord extends OozieClient {
}
}
- /**
- * Return the info of the workflow jobs that match the filter.
- * <p>
- * It returns the first 100 jobs that match the filter.
- *
- * @param filter job filter. Refer to the {@link LocalOozieClient} for the
- * filter syntax.
- * @return a list with the workflow jobs info, without node details.
- * @throws org.apache.oozie.client.OozieClientException thrown if the jobs
- * info could not be retrieved.
- */
@Override
- @Deprecated
- public List<WorkflowJob> getJobsInfo(String filter) throws OozieClientException {
- throw new OozieClientException(ErrorCode.E0301.toString(), "no-op");
+ public String updateCoord(String jobId, Properties conf, String dryrun, String showDiff) throws OozieClientException {
+ try {
+ return coordEngine.updateJob(new XConfiguration(conf), jobId, Boolean.valueOf(dryrun), Boolean.valueOf(showDiff));
+ } catch (CoordinatorEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
}
+ @Override
+ public String updateCoord(String jobId, String dryrun, String showDiff) throws OozieClientException {
+ try {
+ return coordEngine.updateJob(new XConfiguration(), jobId, Boolean.valueOf(dryrun), Boolean.valueOf(showDiff));
+ } catch (CoordinatorEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public CoordinatorJob getCoordJobInfo(String jobId, String filter, int start, int len, String order)
+ throws OozieClientException {
+ try {
+ return coordEngine.getCoordJob(jobId, filter, start, len, order.equalsIgnoreCase("DESC"));
+ } catch (BaseEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public List<CoordinatorAction> kill(String jobId, String rangeType, String scope) throws OozieClientException {
+ try {
+ return (List) coordEngine.killActions(jobId, rangeType, scope).getCoordActions();
+ } catch (CoordinatorEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public List<CoordinatorAction> ignore(String jobId, String scope) throws OozieClientException {
+ try {
+ return (List) coordEngine.ignore(jobId, RestConstants.JOB_COORD_SCOPE_ACTION, scope).getCoordActions();
+ } catch (CoordinatorEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
+
+ @Override
+ public List<WorkflowJob> getWfsForCoordAction(String coordActionId) throws OozieClientException {
+ try {
+ return (List) coordEngine.getReruns(coordActionId);
+ } catch (CoordinatorEngineException e) {
+ throw new OozieClientException(e.getErrorCode().toString(), e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/OozieClientOperationHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/OozieClientOperationHandler.java b/core/src/main/java/org/apache/oozie/OozieClientOperationHandler.java
new file mode 100644
index 0000000..8338f4e
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/OozieClientOperationHandler.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie;
+
+import org.apache.oozie.client.rest.RestConstants;
+import org.json.simple.JSONObject;
+
+class OozieClientOperationHandler {
+
+ private static final String WF = "wf";
+ private static final String BUNDLE = "bundle";
+ private static final String COORD = "coord";
+ private static final String GMT = "GMT";
+
+ private BaseEngine engine;
+
+ OozieClientOperationHandler(BaseEngine engine) {
+ this.engine = engine;
+ }
+
+ interface OozieOperationJob {
+ JSONObject BundleJob() throws BundleEngineException;
+
+ JSONObject CoordinatorJob() throws CoordinatorEngineException;
+
+ JSONObject WorkflowsJob() throws DagEngineException;
+ }
+
+ abstract class AbstractOozieOperationJob implements OozieOperationJob {
+ protected final String filter;
+ protected final int start;
+ protected final int len;
+
+ AbstractOozieOperationJob(final String filter, final int start, final int len) {
+ this.filter = filter;
+ this.start = start;
+ this.len = len;
+ }
+
+ @Override
+ public JSONObject BundleJob() throws BundleEngineException {
+ return OozieJsonFactory.getBundleJSONObject(createBundleJobInfo(), GMT);
+ }
+
+ abstract BundleJobInfo createBundleJobInfo() throws BundleEngineException;
+
+ @Override
+ public JSONObject CoordinatorJob() throws CoordinatorEngineException {
+ return OozieJsonFactory.getCoordJSONObject(createCoordinatorJobInfo(), GMT);
+ }
+
+ abstract CoordinatorJobInfo createCoordinatorJobInfo() throws CoordinatorEngineException;
+
+ @Override
+ public JSONObject WorkflowsJob() throws DagEngineException {
+ return OozieJsonFactory.getWFJSONObject(createWorkflowsInfo(), GMT);
+ }
+
+ abstract WorkflowsInfo createWorkflowsInfo() throws DagEngineException;
+ }
+
+
+ static class OozieJobOperationCaller {
+ JSONObject call(final String jobType, final OozieOperationJob job)
+ throws DagEngineException, BundleEngineException, CoordinatorEngineException {
+ switch (jobType) {
+ case WF:
+ return job.WorkflowsJob();
+ case COORD:
+ return job.CoordinatorJob();
+ case BUNDLE:
+ return job.BundleJob();
+ default:
+ throw new IllegalArgumentException("Invalid jobType passed. jobType: " + jobType);
+ }
+ }
+ }
+
+ class KillOperation extends AbstractOozieOperationJob {
+
+ KillOperation(final String filter, final int start, final int len) {
+ super(filter, start, len);
+ }
+
+ @Override
+ BundleJobInfo createBundleJobInfo() throws BundleEngineException {
+ return ((BundleEngine) engine).killJobs(filter, start, len);
+ }
+
+ @Override
+ CoordinatorJobInfo createCoordinatorJobInfo() throws CoordinatorEngineException {
+ return ((CoordinatorEngine) engine).killJobs(filter, start, len);
+ }
+
+ @Override
+ WorkflowsInfo createWorkflowsInfo() throws DagEngineException {
+ return ((DagEngine) engine).killJobs(filter, start, len);
+ }
+ }
+
+ class SuspendingOperation extends AbstractOozieOperationJob {
+
+ SuspendingOperation(final String filter, final int start, final int len) {
+ super(filter, start, len);
+ }
+
+ @Override
+ BundleJobInfo createBundleJobInfo() throws BundleEngineException {
+ return ((BundleEngine) engine).suspendJobs(filter, start, len);
+ }
+
+ @Override
+ CoordinatorJobInfo createCoordinatorJobInfo() throws CoordinatorEngineException {
+ return ((CoordinatorEngine) engine).suspendJobs(filter, start, len);
+ }
+
+ @Override
+ WorkflowsInfo createWorkflowsInfo() throws DagEngineException {
+ return ((DagEngine) engine).suspendJobs(filter, start, len);
+ }
+ }
+
+ class ResumingOperation extends AbstractOozieOperationJob {
+
+ ResumingOperation(final String filter, final int start, final int len) {
+ super(filter, start, len);
+ }
+
+ @Override
+ BundleJobInfo createBundleJobInfo() throws BundleEngineException {
+ return ((BundleEngine) engine).resumeJobs(filter, start, len);
+ }
+
+ @Override
+ CoordinatorJobInfo createCoordinatorJobInfo() throws CoordinatorEngineException {
+ return ((CoordinatorEngine) engine).resumeJobs(filter, start, len);
+ }
+
+ @Override
+ WorkflowsInfo createWorkflowsInfo() throws DagEngineException {
+ return ((DagEngine) engine).resumeJobs(filter, start, len);
+ }
+ }
+
+ OozieOperationJob getOperationHandler(String operation, final String filter, final int start, final int len) {
+ switch (operation) {
+ case RestConstants.JOB_ACTION_KILL:
+ return new KillOperation(filter, start, len);
+ case RestConstants.JOB_ACTION_SUSPEND:
+ return new SuspendingOperation(filter, start, len);
+ case RestConstants.JOB_ACTION_RESUME:
+ return new ResumingOperation(filter, start, len);
+ default:
+ throw new IllegalArgumentException(operation);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/OozieJsonFactory.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/OozieJsonFactory.java b/core/src/main/java/org/apache/oozie/OozieJsonFactory.java
new file mode 100644
index 0000000..b988ca0
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/OozieJsonFactory.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie;
+
+import org.apache.oozie.client.rest.JsonTags;
+import org.json.simple.JSONObject;
+
+public final class OozieJsonFactory {
+
+ private OozieJsonFactory() {
+ }
+
+ public static JSONObject getWFJSONObject(WorkflowsInfo jobs, String timeZoneId) {
+ JSONObject json = new JSONObject();
+ json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jobs.getWorkflows(), timeZoneId));
+ json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
+ json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
+ json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());
+ return json;
+ }
+
+ public static JSONObject getCoordJSONObject(CoordinatorJobInfo jobs, String timeZoneId) {
+ JSONObject json = new JSONObject();
+ json.put(JsonTags.COORDINATOR_JOBS, CoordinatorJobBean.toJSONArray(jobs.getCoordJobs(), timeZoneId));
+ json.put(JsonTags.COORD_JOB_TOTAL, jobs.getTotal());
+ json.put(JsonTags.COORD_JOB_OFFSET, jobs.getStart());
+ json.put(JsonTags.COORD_JOB_LEN, jobs.getLen());
+ return json;
+ }
+
+ public static JSONObject getBundleJSONObject(BundleJobInfo jobs, String timeZoneId) {
+ JSONObject json = new JSONObject();
+ json.put(JsonTags.BUNDLE_JOBS, BundleJobBean.toJSONArray(jobs.getBundleJobs(), timeZoneId));
+ json.put(JsonTags.BUNDLE_JOB_TOTAL, jobs.getTotal());
+ json.put(JsonTags.BUNDLE_JOB_OFFSET, jobs.getStart());
+ json.put(JsonTags.BUNDLE_JOB_LEN, jobs.getLen());
+ return json;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/local/LocalOozie.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/local/LocalOozie.java b/core/src/main/java/org/apache/oozie/local/LocalOozie.java
index bf1b0db..9ab646c 100644
--- a/core/src/main/java/org/apache/oozie/local/LocalOozie.java
+++ b/core/src/main/java/org/apache/oozie/local/LocalOozie.java
@@ -18,11 +18,14 @@
package org.apache.oozie.local;
+import org.apache.oozie.BundleEngine;
import org.apache.oozie.CoordinatorEngine;
import org.apache.oozie.DagEngine;
import org.apache.oozie.LocalOozieClient;
+import org.apache.oozie.LocalOozieClientBundle;
import org.apache.oozie.LocalOozieClientCoord;
import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.service.BundleEngineService;
import org.apache.oozie.service.CallbackService;
import org.apache.oozie.service.CoordinatorEngineService;
import org.apache.oozie.service.DagEngineService;
@@ -114,12 +117,20 @@ public class LocalOozie {
}
/**
- * Return a {@link org.apache.oozie.client.OozieClient} for LocalOozie. <p> The returned instance is configured
- * with the user name of the JVM (the value of the system property 'user.name'). <p> The following methods of the
- * client are NOP in the returned instance: {@link org.apache.oozie.client.OozieClient#validateWSVersion}, {@link
- * org.apache.oozie.client.OozieClient#setHeader}, {@link org.apache.oozie.client.OozieClient#getHeader}, {@link
- * org.apache.oozie.client.OozieClient#removeHeader}, {@link org.apache.oozie.client.OozieClient#getHeaderNames} and
- * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)}.
+ * Return a {@link org.apache.oozie.client.OozieClient} for LocalOozie.
+ * <p>
+ * The returned instance is configured
+ * with the user name of the JVM (the value of the system property 'user.name').
+ * <p>
+ * The following methods of the client are NOP in the returned instance:
+ * {@link org.apache.oozie.client.OozieClient#validateWSVersion},
+ * {@link org.apache.oozie.client.OozieClient#setHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeader},
+ * {@link org.apache.oozie.client.OozieClient#removeHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
*
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie.
*/
@@ -138,8 +149,10 @@ public class LocalOozie {
* {@link org.apache.oozie.client.OozieClient#setHeader},
* {@link org.apache.oozie.client.OozieClient#getHeader},
* {@link org.apache.oozie.client.OozieClient#removeHeader},
- * {@link org.apache.oozie.client.OozieClient#getHeaderNames} and
- * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)}.
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
*
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie.
*/
@@ -151,10 +164,14 @@ public class LocalOozie {
* Return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for a given user.
* <p>
* The following methods of the client are NOP in the returned instance:
- * {@link org.apache.oozie.client.OozieClient#validateWSVersion}, {@link org.apache.oozie.client.OozieClient#setHeader},
- * {@link org.apache.oozie.client.OozieClient#getHeader}, {@link org.apache.oozie.client.OozieClient#removeHeader},
- * {@link org.apache.oozie.client.OozieClient#getHeaderNames}
- * and {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)}.
+ * {@link org.apache.oozie.client.OozieClient#validateWSVersion},
+ * {@link org.apache.oozie.client.OozieClient#setHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeader},
+ * {@link org.apache.oozie.client.OozieClient#removeHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
*
* @param user user name to use in LocalOozie for running workflows.
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie configured for the given user.
@@ -177,8 +194,10 @@ public class LocalOozie {
* {@link org.apache.oozie.client.OozieClient#setHeader},
* {@link org.apache.oozie.client.OozieClient#getHeader},
* {@link org.apache.oozie.client.OozieClient#removeHeader},
- * {@link org.apache.oozie.client.OozieClient#getHeaderNames} and
- * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)}.
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
*
* @param user user name to use in LocalOozie for running coordinator.
* @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie
@@ -193,4 +212,50 @@ public class LocalOozie {
return new LocalOozieClientCoord(coordEngine);
}
+ /**
+ * <p>
+ * The returned instance is configured with the user name of the JVM (the
+ * value of the system property 'user.name').
+ * <p>
+ * The following methods of the client are NOP in the returned instance:
+ * {@link org.apache.oozie.client.OozieClient#validateWSVersion},
+ * {@link org.apache.oozie.client.OozieClient#setHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeader},
+ * {@link org.apache.oozie.client.OozieClient#removeHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
+ *
+ * @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie.
+ */
+ public static OozieClient getClientBundle() {
+ return getClientBundle(System.getProperty("user.name"));
+ }
+
+ /**
+ * <p>
+ * The returned instance is configured with the user name of the JVM (the
+ * value of the system property 'user.name').
+ * <p>
+ * The following methods of the client are NOP in the returned instance:
+ * {@link org.apache.oozie.client.OozieClient#validateWSVersion},
+ * {@link org.apache.oozie.client.OozieClient#setHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeader},
+ * {@link org.apache.oozie.client.OozieClient#removeHeader},
+ * {@link org.apache.oozie.client.OozieClient#getHeaderNames},
+ * {@link org.apache.oozie.client.OozieClient#setSystemMode(OozieClient.SYSTEM_MODE)},
+ * {@link org.apache.oozie.client.OozieClient#getHeaders},
+ * {@link org.apache.oozie.client.OozieClient#getClientBuildVersion}.
+ *
+ * @return a {@link org.apache.oozie.client.OozieClient} for LocalOozie.
+ */
+ public static OozieClient getClientBundle(String user) {
+ if (!localOozieActive) {
+ throw new IllegalStateException("LocalOozie is not initialized");
+ }
+ ParamChecker.notEmpty(user, "user");
+ BundleEngine bundleEngine = Services.get().get(BundleEngineService.class).getBundleEngine(user);
+ return new LocalOozieClientBundle(bundleEngine);
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/f5554dd3/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java b/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java
index 2c79ef0..8c978fb 100644
--- a/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java
+++ b/core/src/main/java/org/apache/oozie/servlet/V0JobsServlet.java
@@ -19,7 +19,6 @@
package org.apache.oozie.servlet;
import java.io.IOException;
-import java.util.List;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -28,7 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.oozie.DagEngine;
import org.apache.oozie.DagEngineException;
import org.apache.oozie.ErrorCode;
-import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.OozieJsonFactory;
import org.apache.oozie.WorkflowsInfo;
import org.apache.oozie.client.OozieClient;
import org.apache.oozie.client.rest.JsonTags;
@@ -96,7 +95,7 @@ public class V0JobsServlet extends BaseJobsServlet {
*/
@Override
protected JSONObject getJobs(HttpServletRequest request) throws XServletException, IOException {
- JSONObject json = new JSONObject();
+ JSONObject json;
try {
String filter = request.getParameter(RestConstants.JOBS_FILTER_PARAM);
String startStr = request.getParameter(RestConstants.OFFSET_PARAM);
@@ -108,12 +107,7 @@ public class V0JobsServlet extends BaseJobsServlet {
DagEngine dagEngine = Services.get().get(DagEngineService.class)
.getDagEngine(getUser(request));
WorkflowsInfo jobs = dagEngine.getJobs(filter, start, len);
- List<WorkflowJobBean> jsonWorkflows = jobs.getWorkflows();
- json.put(JsonTags.WORKFLOWS_JOBS, WorkflowJobBean.toJSONArray(jsonWorkflows, "GMT"));
- json.put(JsonTags.WORKFLOWS_TOTAL, jobs.getTotal());
- json.put(JsonTags.WORKFLOWS_OFFSET, jobs.getStart());
- json.put(JsonTags.WORKFLOWS_LEN, jobs.getLen());
-
+ json = OozieJsonFactory.getWFJSONObject(jobs, "GMT");
}
catch (DagEngineException ex) {
throw new XServletException(HttpServletResponse.SC_BAD_REQUEST, ex);
[7/9] oozie git commit: OOZIE-2824 Fix typos in documentation (lzeke
via gezapeti)
Posted by ge...@apache.org.
OOZIE-2824 Fix typos in documentation (lzeke via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/5cb3f33c
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/5cb3f33c
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/5cb3f33c
Branch: refs/heads/oya
Commit: 5cb3f33c5abec458bc0fc189e18ecea3dba94491
Parents: 4cb29f8
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue May 9 15:30:26 2017 -0700
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Tue May 9 15:30:26 2017 -0700
----------------------------------------------------------------------
docs/src/site/twiki/AG_Install.twiki | 4 ++--
docs/src/site/twiki/AG_Monitoring.twiki | 2 +-
docs/src/site/twiki/CoordinatorFunctionalSpec.twiki | 2 +-
docs/src/site/twiki/DG_ActionAuthentication.twiki | 4 ++--
docs/src/site/twiki/DG_SLAMonitoring.twiki | 2 +-
docs/src/site/twiki/WebServicesAPI.twiki | 6 +++---
docs/src/site/twiki/WorkflowFunctionalSpec.twiki | 2 +-
release-log.txt | 1 +
8 files changed, 12 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
index a863d46..2125442 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -201,7 +201,7 @@ following configuration properties in the oozie-site.xml:
---++ Database Migration
Oozie provides an easy way to switch between databases without losing any data. Oozie servers should be stopped during the
-database migraition process.
+database migration process.
The export of the database can be done using the following command:
<verbatim>
$ bin/oozie-setup.sh export /tmp/oozie_db.zip
@@ -239,7 +239,7 @@ NOTE: The database version of the zip must match the version of the Oozie databa
After starting the Oozie server, the history and the currently running workflows should be available.
*IMPORTANT:* The tool was primarily developed to make the migration from embedded databases (e.g. Derby) to standalone databases
- (e.g. MySQL, Posgresql, Oracle, MS SQL Server), though it will work between any supported databases.
+ (e.g. MySQL, PosgreSQL, Oracle, MS SQL Server), though it will work between any supported databases.
It is *not* optimized to handle databases over 1 Gb. If the database size is larger, it should be purged before migration.
---++ Oozie Configuration
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/AG_Monitoring.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Monitoring.twiki b/docs/src/site/twiki/AG_Monitoring.twiki
index 2a3e9a0..523f851 100644
--- a/docs/src/site/twiki/AG_Monitoring.twiki
+++ b/docs/src/site/twiki/AG_Monitoring.twiki
@@ -177,7 +177,7 @@ by =oozie.service.SchemaCheckerService.check.interval=
By default, the =SchemaCheckerService= will consider "extra" tables, columns, and indexes to be incorrect. Advanced users who have
added additional tables, columns, and indexes can tell Oozie to ignore these by
-seting =oozie.service.SchemaCheckerService.ignore.extras= to =false=.
+setting =oozie.service.SchemaCheckerService.ignore.extras= to =false=.
The =SchemaCheckerService= currently only supports MySQL, PostgreSQL, and Oracle databases. SQL Server and Derby are currently not
supported.
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
index b8cb261..4e6ba81 100644
--- a/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
+++ b/docs/src/site/twiki/CoordinatorFunctionalSpec.twiki
@@ -3440,7 +3440,7 @@ In this example, the 'nextInstance' will be '2009-01-02T23:00Z' for the first ac
---++++ 6.9.2. coord:dateTzOffset(String baseDate, String timezone) EL Function
The =${coord:dateTzOffset(String baseDate, String timezone)}= EL function calculates the date based on the following
-equation : =newDate = baseDate + (Oozie procesing timezone - timezone)=
+equation : =newDate = baseDate + (Oozie processing timezone - timezone)=
In other words, it offsets the =baseDate= by the difference from Oozie processing timezone to the given =timezone=. It will
account for daylight saving time based on the given =baseDate= and =timezone=.
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/DG_ActionAuthentication.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_ActionAuthentication.twiki b/docs/src/site/twiki/DG_ActionAuthentication.twiki
index 752355a..4d4efa2 100644
--- a/docs/src/site/twiki/DG_ActionAuthentication.twiki
+++ b/docs/src/site/twiki/DG_ActionAuthentication.twiki
@@ -27,8 +27,8 @@ org.apache.oozie.action.hadoop.Credentials to retrieve a delegation token from t
Out of the box, Oozie already comes with support for some credential types
(see [[DG_ActionAuthentication#Built-in_Credentials_Implementations][Built-in Credentials Implementations]]).
-The credential classes that Oozie should load are specified by the following property in oozie-site.xml. The lefthand side of the
-equals sign is the type for the credential type, while the righthand side is the class.
+The credential classes that Oozie should load are specified by the following property in oozie-site.xml. The left hand side of the
+equals sign is the type for the credential type, while the right hand side is the class.
<verbatim>
<property>
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/DG_SLAMonitoring.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/DG_SLAMonitoring.twiki b/docs/src/site/twiki/DG_SLAMonitoring.twiki
index 11a06af..8107788 100644
--- a/docs/src/site/twiki/DG_SLAMonitoring.twiki
+++ b/docs/src/site/twiki/DG_SLAMonitoring.twiki
@@ -166,7 +166,7 @@ In the REST API, the following filters can be applied while fetching SLA informa
* parent_id - Parent id of the workflow job, workflow action or coordinator action
* nominal_start and nominal_end - Start and End range for nominal time of the workflow or coordinator.
* bundle - Bundle Job ID or Bundle App Name. Fetches SLA information for actions of all coordinators in that bundle.
- * event_status - event stauts such as START_MET/START_MISS/DURATION_MET/DURATION_MISS/END_MET/END_MISS
+ * event_status - event status such as START_MET/START_MISS/DURATION_MET/DURATION_MISS/END_MET/END_MISS
* sla_status - sla status such as NOT_STARTED/IN_PROCESS/MET/MISS
multiple event_status and sla_status can be specified with comma separation. When multiple statuses are specified, they are considered as OR.
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/WebServicesAPI.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WebServicesAPI.twiki b/docs/src/site/twiki/WebServicesAPI.twiki
index a63a68c..ef3e602 100644
--- a/docs/src/site/twiki/WebServicesAPI.twiki
+++ b/docs/src/site/twiki/WebServicesAPI.twiki
@@ -1381,7 +1381,7 @@ GET /oozie/v1/job/0000001-111219170928042-oozie-joe-C?show=info&filter=status%21
</verbatim>
This retrieves coordinator actions except for SUCCEEDED status, which is useful for debugging.
-*Retrive information of the retry attempts of the workflow action:*
+*Retrieve information of the retry attempts of the workflow action:*
<verbatim>
GET oozie/v2/job/0000000-161212175234862-oozie-puru-W@pig-node?show=retries
@@ -2037,8 +2037,8 @@ Assuming Oozie is running at =OOZIE_URL=, the following web services end points
There is a difference in the JSON format of Job Information API (*/job) particularly for map-reduce action.
No change for other actions.
-In v1, externalId and consoleUrl point to spawned child job ID, and exteranlChildIDs is null in map-reduce action.
-In v2, externalId and consoleUrl point to launcher job ID, and exteranlChildIDs is spawned child job ID in map-reduce action.
+In v1, externalId and consoleUrl point to spawned child job ID, and externalChildIDs is null in map-reduce action.
+In v2, externalId and consoleUrl point to launcher job ID, and externalChildIDs is spawned child job ID in map-reduce action.
v2 supports retrieving of JMS topic on which job notifications are sent
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
index fa6a84c..6655696 100644
--- a/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
+++ b/docs/src/site/twiki/WorkflowFunctionalSpec.twiki
@@ -2484,7 +2484,7 @@ User-Retry allows user to give certain number of reties (must not exceed system
that problem and fix them when action is in =USER_RETRY= state. If failure or error does not go away after max retries,
the action becomes =FAILED= or =ERROR= and Oozie marks workflow job to =FAILED= or =KILLED=.
-Oozie adminstrator can allow more error codes to be handled for User-Retry. By adding this configuration
+Oozie administrator can allow more error codes to be handled for User-Retry. By adding this configuration
=oozie.service.LiteWorkflowStoreService.user.retry.error.code.ext= to =oozie.site.xml=
and error codes as value, these error codes will be considered as User-Retry after system restart.
http://git-wip-us.apache.org/repos/asf/oozie/blob/5cb3f33c/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index ee65cd1..0128b3b 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2824 Fix typos in documentation (lzeke via gezapeti)
OOZIE-2874 Make the Launcher Mapper map-only job's InputFormat class pluggable (andras.piros via gezapeti)
OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
[6/9] oozie git commit: OOZIE-2874 Make the Launcher Mapper map-only
job's InputFormat class pluggable (andras.piros via gezapeti)
Posted by ge...@apache.org.
OOZIE-2874 Make the Launcher Mapper map-only job's InputFormat class pluggable (andras.piros via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4cb29f81
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4cb29f81
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4cb29f81
Branch: refs/heads/oya
Commit: 4cb29f81492d3b4250f43898a1b3ffae2b28e012
Parents: f5554dd
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Tue May 9 10:43:54 2017 -0700
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Tue May 9 10:43:54 2017 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 22 +++++++++++++-------
.../action/hadoop/LauncherMapperHelper.java | 4 +++-
core/src/main/resources/oozie-default.xml | 8 +++++++
release-log.txt | 1 +
4 files changed, 26 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4cb29f81/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index f62c997..d60a5c7 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.mapred.JobClient;
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
-import org.apache.hadoop.mapreduce.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -103,7 +102,9 @@ public class JavaActionExecutor extends ActionExecutor {
public static final String ACL_MODIFY_JOB = "mapreduce.job.acl-modify-job";
public static final String HADOOP_YARN_TIMELINE_SERVICE_ENABLED = "yarn.timeline-service.enabled";
public static final String HADOOP_YARN_UBER_MODE = "mapreduce.job.ubertask.enable";
- public static final String HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART = "oozie.action.launcher.am.restart.kill.childjobs";
+ public static final String OOZIE_ACTION_LAUNCHER_PREFIX = ActionExecutor.CONF_PREFIX + "launcher.";
+ public static final String HADOOP_YARN_KILL_CHILD_JOBS_ON_AMRESTART =
+ OOZIE_ACTION_LAUNCHER_PREFIX + "am.restart.kill.childjobs";
public static final String HADOOP_MAP_MEMORY_MB = "mapreduce.map.memory.mb";
public static final String HADOOP_CHILD_JAVA_OPTS = "mapred.child.java.opts";
public static final String HADOOP_MAP_JAVA_OPTS = "mapreduce.map.java.opts";
@@ -125,10 +126,11 @@ public class JavaActionExecutor extends ActionExecutor {
protected XLog LOG = XLog.getLog(getClass());
private static final Pattern heapPattern = Pattern.compile("-Xmx(([0-9]+)[mMgG])");
private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir=";
- public static final String CONF_HADOOP_YARN_UBER_MODE = "oozie.action.launcher." + HADOOP_YARN_UBER_MODE;
+ public static final String CONF_HADOOP_YARN_UBER_MODE = OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_YARN_UBER_MODE;
public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader";
public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first";
public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip";
+ private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
public XConfiguration workflowConf = null;
@@ -151,7 +153,7 @@ public class JavaActionExecutor extends ActionExecutor {
public static List<Class> getCommonLauncherClasses() {
List<Class> classes = new ArrayList<Class>();
classes.add(LauncherMapper.class);
- classes.add(OozieLauncherInputFormat.class);
+ classes.add(launcherInputFormatClassLocator.locateOrGet());
classes.add(OozieLauncherOutputFormat.class);
classes.add(OozieLauncherOutputCommitter.class);
classes.add(LauncherMainHadoopUtils.class);
@@ -297,12 +299,12 @@ public class JavaActionExecutor extends ActionExecutor {
// 2. oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable
// 3. oozie.action.launcher.mapreduce.job.ubertask.enable
if (launcherConf.get(HADOOP_YARN_UBER_MODE) == null) {
- if (ConfigurationService.get("oozie.action." + getType() + ".launcher." + HADOOP_YARN_UBER_MODE).length() > 0) {
- if (ConfigurationService.getBoolean("oozie.action." + getType() + ".launcher." + HADOOP_YARN_UBER_MODE)) {
+ if (ConfigurationService.get(getActionTypeLauncherPrefix() + HADOOP_YARN_UBER_MODE).length() > 0) {
+ if (ConfigurationService.getBoolean(getActionTypeLauncherPrefix() + HADOOP_YARN_UBER_MODE)) {
launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true);
}
} else {
- if (ConfigurationService.getBoolean("oozie.action.launcher." + HADOOP_YARN_UBER_MODE)) {
+ if (ConfigurationService.getBoolean(OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_YARN_UBER_MODE)) {
launcherConf.setBoolean(HADOOP_YARN_UBER_MODE, true);
}
}
@@ -312,7 +314,7 @@ public class JavaActionExecutor extends ActionExecutor {
void injectLauncherTimelineServiceEnabled(Configuration launcherConf, Configuration actionConf) {
// Getting delegation token for ATS. If tez-site.xml is present in distributed cache, turn on timeline service.
if (actionConf.get("oozie.launcher." + HADOOP_YARN_TIMELINE_SERVICE_ENABLED) == null
- && ConfigurationService.getBoolean("oozie.action.launcher." + HADOOP_YARN_TIMELINE_SERVICE_ENABLED)) {
+ && ConfigurationService.getBoolean(OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_YARN_TIMELINE_SERVICE_ENABLED)) {
String cacheFiles = launcherConf.get("mapred.cache.files");
if (cacheFiles != null && cacheFiles.contains("tez-site.xml")) {
launcherConf.setBoolean(HADOOP_YARN_TIMELINE_SERVICE_ENABLED, true);
@@ -1778,4 +1780,8 @@ public class JavaActionExecutor extends ActionExecutor {
return workflowConf;
}
+
+ private String getActionTypeLauncherPrefix() {
+ return "oozie.action." + getType() + ".launcher.";
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4cb29f81/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
index 9609fdc..72ed2f1 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
@@ -57,6 +57,8 @@ public class LauncherMapperHelper {
public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag";
+ private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
+
public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
throws HadoopAccessorException, IOException {
String jobId = null;
@@ -160,7 +162,7 @@ public class LauncherMapperHelper {
IOUtils.closeSafely(os);
}
- launcherConf.setInputFormat(OozieLauncherInputFormat.class);
+ launcherConf.setInputFormat(launcherInputFormatClassLocator.locateOrGet());
launcherConf.setOutputFormat(OozieLauncherOutputFormat.class);
launcherConf.setOutputCommitter(OozieLauncherOutputCommitter.class);
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4cb29f81/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --git a/core/src/main/resources/oozie-default.xml b/core/src/main/resources/oozie-default.xml
index e7a48a0..076401d 100644
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@ -1860,6 +1860,14 @@ will be the requeue interval for the actions which are waiting for a long time w
</property>
<property>
+ <name>oozie.action.launcher.mapreduce.input.format.class</name>
+ <value>org.apache.oozie.action.hadoop.OozieLauncherInputFormat</value>
+ <description>
+ Make the Launcher Mapper map-only job's InputFormat class pluggable in order to provide alternative implementations.
+ </description>
+ </property>
+
+ <property>
<name>oozie.action.spark.setup.hadoop.conf.dir</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/4cb29f81/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index cb141cb..ee65cd1 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2874 Make the Launcher Mapper map-only job's InputFormat class pluggable (andras.piros via gezapeti)
OOZIE-2751 LocalOozieClient is missing methods from OozieClient (abhishekbafna via rkanter)
OOZIE-2870 non working examples in oozie documentation coordinator spec (andras.piros via pbacsko)
OOZIE-2827 amend More directly view of the coordinator’s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
[9/9] oozie git commit: Merge branch 'master' into oya
Posted by ge...@apache.org.
Merge branch 'master' into oya
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/80854d15
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/80854d15
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/80854d15
Branch: refs/heads/oya
Commit: 80854d151d67666132a02797de4efdbf3b2af175
Parents: 40ee320 1103882
Author: Gezapeti Cseh <ge...@gmail.com>
Authored: Wed May 10 14:09:31 2017 -0700
Committer: Gezapeti Cseh <ge...@gmail.com>
Committed: Wed May 10 14:09:31 2017 -0700
----------------------------------------------------------------------
.../org/apache/oozie/client/OozieClient.java | 4 +-
.../apache/oozie/client/rest/RestConstants.java | 2 +-
.../main/java/org/apache/oozie/BaseEngine.java | 18 +-
.../org/apache/oozie/BaseLocalOozieClient.java | 601 +++++++++++++++++++
.../java/org/apache/oozie/LocalOozieClient.java | 245 +-------
.../apache/oozie/LocalOozieClientBundle.java | 93 +++
.../org/apache/oozie/LocalOozieClientCoord.java | 328 ++--------
.../oozie/OozieClientOperationHandler.java | 173 ++++++
.../java/org/apache/oozie/OozieJsonFactory.java | 55 ++
.../oozie/action/hadoop/JavaActionExecutor.java | 15 +-
.../oozie/action/hadoop/LauncherHelper.java | 2 +
.../hadoop/LauncherInputFormatClassLocator.java | 84 +++
.../command/coord/CoordSubmitXCommand.java | 25 -
.../java/org/apache/oozie/local/LocalOozie.java | 93 ++-
.../org/apache/oozie/servlet/V0JobsServlet.java | 12 +-
.../org/apache/oozie/servlet/V1JobsServlet.java | 184 ++----
core/src/main/resources/oozie-default.xml | 8 +
.../apache/oozie/TestLocalOozieClientCoord.java | 72 ++-
.../TestLauncherInputFormatClassLocator.java | 57 ++
.../command/coord/TestCoordSubmitXCommand.java | 30 -
.../resources/coord-invalid-el-function.xml | 35 --
.../resources/coord-invalid-output-instance.xml | 58 --
.../coord-multiple-input-instance4.xml | 2 +-
.../coord-multiple-input-start-instance2.xml | 2 +-
.../coord-multiple-output-instance4.xml | 2 +-
docs/src/site/twiki/AG_Install.twiki | 4 +-
docs/src/site/twiki/AG_Monitoring.twiki | 2 +-
.../site/twiki/CoordinatorFunctionalSpec.twiki | 2 +-
.../site/twiki/DG_ActionAuthentication.twiki | 4 +-
docs/src/site/twiki/DG_SLAMonitoring.twiki | 2 +-
docs/src/site/twiki/WebServicesAPI.twiki | 6 +-
.../src/site/twiki/WorkflowFunctionalSpec.twiki | 2 +-
release-log.txt | 4 +-
33 files changed, 1412 insertions(+), 814 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/80854d15/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index f3e5e32,d60a5c7..7a762ab
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@@ -49,14 -48,13 +49,15 @@@ import org.apache.hadoop.fs.FileStatus
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AccessControlException;
-import org.apache.oozie.hadoop.utils.HadoopShims;
+import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.mapred.JobClient;
-import org.apache.hadoop.mapred.JobConf;
-import org.apache.hadoop.mapred.JobID;
-import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.mapred.TaskLog;
+import org.apache.hadoop.mapreduce.filecache.ClientDistributedCacheManager;
+import org.apache.hadoop.mapreduce.v2.util.MRApps;
+import org.apache.hadoop.security.Credentials;
+ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.DiskChecker;
@@@ -144,11 -119,18 +148,12 @@@ public class JavaActionExecutor extend
private static int maxActionOutputLen;
private static int maxExternalStatsSize;
private static int maxFSGlobMax;
- private static final String SUCCEEDED = "SUCCEEDED";
- private static final String KILLED = "KILLED";
- private static final String FAILED = "FAILED";
- private static final String FAILED_KILLED = "FAILED/KILLED";
+
+ protected static final String HADOOP_USER = "user.name";
+
protected XLog LOG = XLog.getLog(getClass());
- private static final Pattern heapPattern = Pattern.compile("-Xmx(([0-9]+)[mMgG])");
private static final String JAVA_TMP_DIR_SETTINGS = "-Djava.io.tmpdir=";
- public static final String CONF_HADOOP_YARN_UBER_MODE = OOZIE_ACTION_LAUNCHER_PREFIX + HADOOP_YARN_UBER_MODE;
- public static final String HADOOP_JOB_CLASSLOADER = "mapreduce.job.classloader";
- public static final String HADOOP_USER_CLASSPATH_FIRST = "mapreduce.user.classpath.first";
- public static final String OOZIE_CREDENTIALS_SKIP = "oozie.credentials.skip";
+ private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
public XConfiguration workflowConf = null;
@@@ -166,15 -150,15 +171,15 @@@
super(type);
}
- public static List<Class> getCommonLauncherClasses() {
- List<Class> classes = new ArrayList<Class>();
- classes.add(LauncherMapper.class);
+ public static List<Class<?>> getCommonLauncherClasses() {
+ List<Class<?>> classes = new ArrayList<Class<?>>();
- classes.add(OozieLauncherInputFormat.class);
+ classes.add(LauncherMain.class);
+ classes.add(launcherInputFormatClassLocator.locateOrGet());
classes.add(OozieLauncherOutputFormat.class);
classes.add(OozieLauncherOutputCommitter.class);
- classes.add(LauncherMainHadoopUtils.class);
- classes.add(HadoopShims.class);
classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
+ classes.add(LauncherAM.class);
+ classes.add(LauncherAMCallbackNotifier.class);
return classes;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/80854d15/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
----------------------------------------------------------------------
diff --cc core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
index 5ac1a16,0000000..442f9a3
mode 100644,000000..100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherHelper.java
@@@ -1,321 -1,0 +1,323 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.math.BigInteger;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.HadoopAccessorException;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.URIHandlerService;
+import org.apache.oozie.service.UserGroupInformationService;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.PropertiesUtils;
+
+// TODO: we're no longer using Launcher Mapper -- give this class a better name
+public class LauncherHelper {
+
+ public static final String OOZIE_ACTION_YARN_TAG = "oozie.action.yarn.tag";
+
++ private static final LauncherInputFormatClassLocator launcherInputFormatClassLocator = new LauncherInputFormatClassLocator();
++
+ public static String getRecoveryId(Configuration launcherConf, Path actionDir, String recoveryId)
+ throws HadoopAccessorException, IOException {
+ String jobId = null;
+ Path recoveryFile = new Path(actionDir, recoveryId);
+ FileSystem fs = Services.get().get(HadoopAccessorService.class)
+ .createFileSystem(launcherConf.get("user.name"),recoveryFile.toUri(), launcherConf);
+
+ if (fs.exists(recoveryFile)) {
+ InputStream is = fs.open(recoveryFile);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+ jobId = reader.readLine();
+ reader.close();
+ }
+ return jobId;
+
+ }
+
+ public static void setupMainClass(Configuration launcherConf, String javaMainClass) {
+ // Only set the javaMainClass if its not null or empty string, this way the user can override the action's main class via
+ // <configuration> property
+ if (javaMainClass != null && !javaMainClass.equals("")) {
+ launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, javaMainClass);
+ }
+ }
+
+ public static void setupLauncherURIHandlerConf(Configuration launcherConf) {
+ for(Map.Entry<String, String> entry : Services.get().get(URIHandlerService.class).getLauncherConfig()) {
+ launcherConf.set(entry.getKey(), entry.getValue());
+ }
+ }
+
+ public static void setupMainArguments(Configuration launcherConf, String[] args) {
+ launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_COUNT, args.length);
+ for (int i = 0; i < args.length; i++) {
+ launcherConf.set(LauncherMapper.CONF_OOZIE_ACTION_MAIN_ARG_PREFIX + i, args[i]);
+ }
+ }
+
+ public static void setupMaxOutputData(Configuration launcherConf, int maxOutputData) {
+ launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA, maxOutputData);
+ }
+
+ /**
+ * Set the maximum value of stats data
+ *
+ * @param launcherConf the oozie launcher configuration
+ * @param maxStatsData the maximum allowed size of stats data
+ */
+ public static void setupMaxExternalStatsSize(Configuration launcherConf, int maxStatsData){
+ launcherConf.setInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE, maxStatsData);
+ }
+
+ /**
+ * Set the maximum number of globbed files/dirs
+ *
+ * @param launcherConf the oozie launcher configuration
+ * @param fsGlobMax the maximum number of files/dirs for FS operation
+ */
+ public static void setupMaxFSGlob(Configuration launcherConf, int fsGlobMax){
+ launcherConf.setInt(LauncherMapper.CONF_OOZIE_ACTION_FS_GLOB_MAX, fsGlobMax);
+ }
+
+ public static void setupLauncherInfo(Configuration launcherConf, String jobId, String actionId, Path actionDir,
+ String recoveryId, Configuration actionConf, String prepareXML) throws IOException, HadoopAccessorException {
+
+ launcherConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
+ launcherConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
+ launcherConf.set(LauncherMapper.OOZIE_ACTION_DIR_PATH, actionDir.toString());
+ launcherConf.set(LauncherMapper.OOZIE_ACTION_RECOVERY_ID, recoveryId);
+ launcherConf.set(LauncherMapper.ACTION_PREPARE_XML, prepareXML);
+
+ actionConf.set(LauncherMapper.OOZIE_JOB_ID, jobId);
+ actionConf.set(LauncherMapper.OOZIE_ACTION_ID, actionId);
+
+ if (Services.get().getConf().getBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", false)) {
+ List<String> purgedEntries = new ArrayList<String>();
+ Collection<String> entries = actionConf.getStringCollection("mapreduce.job.cache.files");
+ for (String entry : entries) {
+ if (entry.contains("#")) {
+ purgedEntries.add(entry);
+ }
+ }
+ actionConf.setStrings("mapreduce.job.cache.files", purgedEntries.toArray(new String[purgedEntries.size()]));
+ launcherConf.setBoolean("oozie.hadoop-2.0.2-alpha.workaround.for.distributed.cache", true);
+ }
+ }
+
+ public static void setupYarnRestartHandling(Configuration launcherJobConf, Configuration actionConf, String launcherTag,
+ long launcherTime)
+ throws NoSuchAlgorithmException {
+ launcherJobConf.setLong(LauncherMain.OOZIE_JOB_LAUNCH_TIME, launcherTime);
+ // Tags are limited to 100 chars so we need to hash them to make sure (the actionId otherwise doesn't have a max length)
+ String tag = getTag(launcherTag);
+ // keeping the oozie.child.mapreduce.job.tags instead of mapreduce.job.tags to avoid killing launcher itself.
+ // mapreduce.job.tags should only go to child job launch by launcher.
+ actionConf.set(LauncherMain.CHILD_MAPREDUCE_JOB_TAGS, tag);
+ }
+
+ public static String getTag(String launcherTag) throws NoSuchAlgorithmException {
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ digest.update(launcherTag.getBytes(), 0, launcherTag.length());
+ String md5 = "oozie-" + new BigInteger(1, digest.digest()).toString(16);
+ return md5;
+ }
+
+ public static boolean isMainDone(RunningJob runningJob) throws IOException {
+ return runningJob.isComplete();
+ }
+
+ public static boolean isMainSuccessful(RunningJob runningJob) throws IOException {
+ boolean succeeded = runningJob.isSuccessful();
+ if (succeeded) {
+ Counters counters = runningJob.getCounters();
+ if (counters != null) {
+ Counters.Group group = counters.getGroup(LauncherMapper.COUNTER_GROUP);
+ if (group != null) {
+ succeeded = group.getCounter(LauncherMapper.COUNTER_LAUNCHER_ERROR) == 0;
+ }
+ }
+ }
+ return succeeded;
+ }
+
+ /**
+ * Determine whether action has external child jobs or not
+ * @param actionData
+ * @return true/false
+ * @throws IOException
+ */
+ public static boolean hasExternalChildJobs(Map<String, String> actionData) throws IOException {
+ return actionData.containsKey(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS);
+ }
+
+ /**
+ * Determine whether action has output data or not
+ * @param actionData
+ * @return true/false
+ * @throws IOException
+ */
+ public static boolean hasOutputData(Map<String, String> actionData) throws IOException {
+ return actionData.containsKey(LauncherMapper.ACTION_DATA_OUTPUT_PROPS);
+ }
+
+ /**
+ * Determine whether action has external stats or not
+ * @param actionData
+ * @return true/false
+ * @throws IOException
+ */
+ public static boolean hasStatsData(Map<String, String> actionData) throws IOException{
+ return actionData.containsKey(LauncherMapper.ACTION_DATA_STATS);
+ }
+
+ /**
+ * Determine whether action has new id (id swap) or not
+ * @param actionData
+ * @return true/false
+ * @throws IOException
+ */
+ public static boolean hasIdSwap(Map<String, String> actionData) throws IOException {
+ return actionData.containsKey(LauncherMapper.ACTION_DATA_NEW_ID);
+ }
+
+ /**
+ * Get the sequence file path storing all action data
+ * @param actionDir
+ * @return
+ */
+ public static Path getActionDataSequenceFilePath(Path actionDir) {
+ return new Path(actionDir, LauncherMapper.ACTION_DATA_SEQUENCE_FILE);
+ }
+
+ /**
+ * Utility function to load the contents of action data sequence file into
+ * memory object
+ *
+ * @param fs Action Filesystem
+ * @param actionDir Path
+ * @param conf Configuration
+ * @return Map action data
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public static Map<String, String> getActionData(final FileSystem fs, final Path actionDir, final Configuration conf)
+ throws IOException, InterruptedException {
+ UserGroupInformationService ugiService = Services.get().get(UserGroupInformationService.class);
+ UserGroupInformation ugi = ugiService.getProxyUser(conf.get(OozieClient.USER_NAME));
+
+ return ugi.doAs(new PrivilegedExceptionAction<Map<String, String>>() {
+ @Override
+ public Map<String, String> run() throws IOException {
+ Map<String, String> ret = new HashMap<String, String>();
+ Path seqFilePath = getActionDataSequenceFilePath(actionDir);
+ if (fs.exists(seqFilePath)) {
+ SequenceFile.Reader seqFile = new SequenceFile.Reader(fs, seqFilePath, conf);
+ Text key = new Text(), value = new Text();
+ while (seqFile.next(key, value)) {
+ ret.put(key.toString(), value.toString());
+ }
+ seqFile.close();
+ }
+ else { // maintain backward-compatibility. to be deprecated
+ org.apache.hadoop.fs.FileStatus[] files = fs.listStatus(actionDir);
+ InputStream is;
+ BufferedReader reader = null;
+ Properties props;
+ if (files != null && files.length > 0) {
+ for (int x = 0; x < files.length; x++) {
+ Path file = files[x].getPath();
+ if (file.equals(new Path(actionDir, "externalChildIds.properties"))) {
+ is = fs.open(file);
+ reader = new BufferedReader(new InputStreamReader(is));
+ ret.put(LauncherMapper.ACTION_DATA_EXTERNAL_CHILD_IDS,
+ IOUtils.getReaderAsString(reader, -1));
+ }
+ else if (file.equals(new Path(actionDir, "newId.properties"))) {
+ is = fs.open(file);
+ reader = new BufferedReader(new InputStreamReader(is));
+ props = PropertiesUtils.readProperties(reader, -1);
+ ret.put(LauncherMapper.ACTION_DATA_NEW_ID, props.getProperty("id"));
+ }
+ else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_OUTPUT_PROPS))) {
+ int maxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_ACTION_MAX_OUTPUT_DATA,
+ 2 * 1024);
+ is = fs.open(file);
+ reader = new BufferedReader(new InputStreamReader(is));
+ ret.put(LauncherMapper.ACTION_DATA_OUTPUT_PROPS, PropertiesUtils
+ .propertiesToString(PropertiesUtils.readProperties(reader, maxOutputData)));
+ }
+ else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_STATS))) {
+ int statsMaxOutputData = conf.getInt(LauncherMapper.CONF_OOZIE_EXTERNAL_STATS_MAX_SIZE,
+ Integer.MAX_VALUE);
+ is = fs.open(file);
+ reader = new BufferedReader(new InputStreamReader(is));
+ ret.put(LauncherMapper.ACTION_DATA_STATS, PropertiesUtils
+ .propertiesToString(PropertiesUtils.readProperties(reader, statsMaxOutputData)));
+ }
+ else if (file.equals(new Path(actionDir, LauncherMapper.ACTION_DATA_ERROR_PROPS))) {
+ is = fs.open(file);
+ reader = new BufferedReader(new InputStreamReader(is));
+ ret.put(LauncherMapper.ACTION_DATA_ERROR_PROPS, IOUtils.getReaderAsString(reader, -1));
+ }
+ }
+ }
+ }
+ return ret;
+ }
+ });
+ }
+
+ public static String getActionYarnTag(Configuration conf, String parentId, WorkflowAction wfAction) {
+ String tag;
+ if ( conf != null && conf.get(OOZIE_ACTION_YARN_TAG) != null) {
+ tag = conf.get(OOZIE_ACTION_YARN_TAG) + "@" + wfAction.getName();
+ } else if (parentId != null) {
+ tag = parentId + "@" + wfAction.getName();
+ } else {
+ tag = wfAction.getId();
+ }
+ return tag;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/80854d15/core/src/main/resources/oozie-default.xml
----------------------------------------------------------------------
diff --cc core/src/main/resources/oozie-default.xml
index 4ab7512,076401d..61b1d88
--- a/core/src/main/resources/oozie-default.xml
+++ b/core/src/main/resources/oozie-default.xml
@@@ -1835,6 -1835,39 +1835,14 @@@ will be the requeue interval for the ac
</property>
<property>
- <name>oozie.action.launcher.mapreduce.job.ubertask.enable</name>
- <value>true</value>
- <description>
- Enables Uber Mode for the launcher job in YARN/Hadoop 2 (no effect in Hadoop 1) for all action types by default.
- This can be overridden on a per-action-type basis by setting
- oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site.xml (where #action-type# is the action
- type; for example, "pig"). And that can be overridden on a per-action basis by setting
- oozie.launcher.mapreduce.job.ubertask.enable in an action's configuration section in a workflow. In summary, the
- priority is this:
- 1. action's configuration section in a workflow
- 2. oozie.action.#action-type#.launcher.mapreduce.job.ubertask.enable in oozie-site
- 3. oozie.action.launcher.mapreduce.job.ubertask.enable in oozie-site
- </description>
- </property>
-
- <property>
- <name>oozie.action.shell.launcher.mapreduce.job.ubertask.enable</name>
- <value>false</value>
- <description>
- The Shell action may have issues with the $PATH environment when using Uber Mode, and so Uber Mode is disabled by
- default for it. See oozie.action.launcher.mapreduce.job.ubertask.enable
- </description>
- </property>
-
- <property>
+ <name>oozie.action.launcher.mapreduce.input.format.class</name>
+ <value>org.apache.oozie.action.hadoop.OozieLauncherInputFormat</value>
+ <description>
+ Make the Launcher Mapper map-only job's InputFormat class pluggable in order to provide alternative implementations.
+ </description>
+ </property>
+
+ <property>
<name>oozie.action.spark.setup.hadoop.conf.dir</name>
<value>false</value>
<description>
http://git-wip-us.apache.org/repos/asf/oozie/blob/80854d15/release-log.txt
----------------------------------------------------------------------