You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2018/08/24 08:54:31 UTC
oozie git commit: OOZIE-3332 [examples] Spark examples should feature
yarn client and cluster modes (daniel.becker via andras.piros)
Repository: oozie
Updated Branches:
refs/heads/master ee275389b -> e08d696e4
OOZIE-3332 [examples] Spark examples should feature yarn client and cluster modes (daniel.becker via andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/e08d696e
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/e08d696e
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/e08d696e
Branch: refs/heads/master
Commit: e08d696e49f0af1ca69ffd1f5733e0e7ef471165
Parents: ee27538
Author: Andras Piros <an...@cloudera.com>
Authored: Fri Aug 24 10:53:15 2018 +0200
Committer: Andras Piros <an...@cloudera.com>
Committed: Fri Aug 24 10:53:15 2018 +0200
----------------------------------------------------------------------
examples/src/main/apps/pyspark/job.properties | 4 ++--
examples/src/main/apps/pyspark/workflow.xml | 1 +
examples/src/main/apps/spark/job.properties | 1 +
examples/src/main/apps/spark/workflow.xml | 1 +
.../src/main/java/org/apache/oozie/example/fluentjob/Spark.java | 1 +
release-log.txt | 1 +
6 files changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/pyspark/job.properties
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/pyspark/job.properties b/examples/src/main/apps/pyspark/job.properties
index b3c70ad..2078a7e 100644
--- a/examples/src/main/apps/pyspark/job.properties
+++ b/examples/src/main/apps/pyspark/job.properties
@@ -18,9 +18,9 @@
nameNode=hdfs://localhost:8020
resourceManager=localhost:8032
+master=yarn
+mode=client
queueName=default
examplesRoot=examples
oozie.use.system.libpath=true
oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/pyspark
-master=yarn-client
-
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/pyspark/workflow.xml
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/pyspark/workflow.xml b/examples/src/main/apps/pyspark/workflow.xml
index 64b7f67..f83ab9b 100644
--- a/examples/src/main/apps/pyspark/workflow.xml
+++ b/examples/src/main/apps/pyspark/workflow.xml
@@ -25,6 +25,7 @@
<resource-manager>${resourceManager}</resource-manager>
<name-node>${nameNode}</name-node>
<master>${master}</master>
+ <mode>${mode}</mode>
<name>Python-Spark-Pi</name>
<jar>pi.py</jar>
</spark>
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/spark/job.properties
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/spark/job.properties b/examples/src/main/apps/spark/job.properties
index 698711a..9866e26 100644
--- a/examples/src/main/apps/spark/job.properties
+++ b/examples/src/main/apps/spark/job.properties
@@ -19,6 +19,7 @@
nameNode=hdfs://localhost:8020
resourceManager=localhost:8032
master=local[*]
+mode=client
queueName=default
examplesRoot=examples
oozie.use.system.libpath=true
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/apps/spark/workflow.xml
----------------------------------------------------------------------
diff --git a/examples/src/main/apps/spark/workflow.xml b/examples/src/main/apps/spark/workflow.xml
index 2ae576b..d3b3564 100644
--- a/examples/src/main/apps/spark/workflow.xml
+++ b/examples/src/main/apps/spark/workflow.xml
@@ -26,6 +26,7 @@
<delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark"/>
</prepare>
<master>${master}</master>
+ <mode>${mode}</mode>
<name>Spark-FileCopy</name>
<class>org.apache.oozie.example.SparkFileCopy</class>
<jar>${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar</jar>
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java b/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java
index 6065933..9527336 100644
--- a/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java
+++ b/examples/src/main/java/org/apache/oozie/example/fluentjob/Spark.java
@@ -42,6 +42,7 @@ public class Spark implements WorkflowFactory {
.withArg("${nameNode}/user/${wf:user()}/${examplesRoot}/input-data/text/data.txt")
.withArg("${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/spark")
.withMaster("${master}")
+ .withMode("${mode}")
.withActionName("Spark File Copy Example")
.withActionClass("org.apache.oozie.example.SparkFileCopy")
.withJar("${nameNode}/user/${wf:user()}/${examplesRoot}/apps/spark/lib/oozie-examples.jar")
http://git-wip-us.apache.org/repos/asf/oozie/blob/e08d696e/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index fd06d45..3c569d0 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.1.0 release (trunk - unreleased)
+OOZIE-3332 [examples] Spark examples should feature yarn client and cluster modes (daniel.becker via andras.piros)
OOZIE-3318 [build] Fix Javadoc check in the pre-commit (kmarton via andras.piros)
OOZIE-3317 [build] Fix false positive precommit reports (kmarton via andras.piros)
OOZIE-3264 Flaky test TestCoordMaterializeTransitionXCommand#testLastOnlyMaterialization (asalamon74 via andras.piros)