You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by pb...@apache.org on 2017/04/19 11:41:51 UTC
oozie git commit: OOZIE-2818 Can't overwrite
oozie.action.max.output.data on a per-workflow basis (asasvari via pbacsko)
Repository: oozie
Updated Branches:
refs/heads/master 406183137 -> fbe5e491e
OOZIE-2818 Can't overwrite oozie.action.max.output.data on a per-workflow basis (asasvari via pbacsko)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/fbe5e491
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/fbe5e491
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/fbe5e491
Branch: refs/heads/master
Commit: fbe5e491e65010ccaa1d3a646bee883a1873da19
Parents: 4061831
Author: Peter Bacsko <pb...@cloudera.com>
Authored: Wed Apr 19 13:41:29 2017 +0200
Committer: Peter Bacsko <pb...@cloudera.com>
Committed: Wed Apr 19 13:41:29 2017 +0200
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 15 ++++++++++++++-
.../oozie/action/hadoop/TestJavaActionExecutor.java | 14 ++++++++++++++
release-log.txt | 1 +
3 files changed, 29 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/fbe5e491/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 39daca7..f62c997 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -40,6 +40,8 @@ import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.primitives.Ints;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
import org.apache.hadoop.fs.FileStatus;
@@ -983,7 +985,8 @@ public class JavaActionExecutor extends ActionExecutor {
// Set the launcher Main Class
LauncherMapperHelper.setupMainClass(launcherJobConf, getLauncherMain(launcherJobConf, actionXml));
LauncherMapperHelper.setupLauncherURIHandlerConf(launcherJobConf);
- LauncherMapperHelper.setupMaxOutputData(launcherJobConf, maxActionOutputLen);
+
+ LauncherMapperHelper.setupMaxOutputData(launcherJobConf, getMaxOutputData(actionConf));
LauncherMapperHelper.setupMaxExternalStatsSize(launcherJobConf, maxExternalStatsSize);
LauncherMapperHelper.setupMaxFSGlob(launcherJobConf, maxFSGlobMax);
@@ -1034,6 +1037,16 @@ public class JavaActionExecutor extends ActionExecutor {
}
}
+ @VisibleForTesting
+ protected static int getMaxOutputData(Configuration actionConf) {
+ String userMaxActionOutputLen = actionConf.get("oozie.action.max.output.data");
+ if (userMaxActionOutputLen != null) {
+ Integer i = Ints.tryParse(userMaxActionOutputLen);
+ return i != null ? i : maxActionOutputLen;
+ }
+ return maxActionOutputLen;
+ }
+
private boolean checkPropertiesToDisableUber(Configuration launcherConf) {
boolean disable = false;
if (launcherConf.getBoolean(HADOOP_JOB_CLASSLOADER, false)) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/fbe5e491/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 1c4b429..d1f53fe 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2967,4 +2967,18 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
ae.end(context, context.getAction());
assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
}
+
+ public void testMaxOutputDataSetByUser() {
+ Configuration conf = new Configuration(false);
+ conf.set("oozie.action.max.output.data", "10000");
+
+ assertEquals(10000, JavaActionExecutor.getMaxOutputData(conf));
+ }
+
+ public void testInvalidMaxOutputDataSetByUser() {
+ Configuration conf = new Configuration(false);
+ conf.set("oozie.action.max.output.data", "sdasda");
+
+ assertEquals(2048, JavaActionExecutor.getMaxOutputData(conf));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/fbe5e491/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 71af781..11ae948 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.4.0 release (trunk - unreleased)
+OOZIE-2818 Can't overwrite oozie.action.max.output.data on a per-workflow basis (asasvari via pbacsko)
OOZIE-2827 More directly view of the coordinator\u2019s history from perspective of workflow action. (Alonzo Zhou via pbacsko)
OOZIE-2864 Maven artifacts for package com.codahale.metrics have inconsistent groupId (andras.piros via pbacsko)
OOZIE-2862 Coord change command doesn't change job to running if job was killed without creating any actions (puru)