You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by an...@apache.org on 2018/09/25 09:59:12 UTC
oozie git commit: OOZIE-3307 [core] Limit heap usage of LauncherAM
(andras.piros)
Repository: oozie
Updated Branches:
refs/heads/master 68bcd3d38 -> 57c2a2f55
OOZIE-3307 [core] Limit heap usage of LauncherAM (andras.piros)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/57c2a2f5
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/57c2a2f5
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/57c2a2f5
Branch: refs/heads/master
Commit: 57c2a2f556b047dbd40fde028593c66a84d644e7
Parents: 68bcd3d
Author: Andras Piros <an...@cloudera.com>
Authored: Tue Sep 25 11:55:59 2018 +0200
Committer: Andras Piros <an...@cloudera.com>
Committed: Tue Sep 25 11:59:03 2018 +0200
----------------------------------------------------------------------
.../action/hadoop/BytesAndUOMConverter.java | 78 ++++++++++++++
.../oozie/action/hadoop/JavaActionExecutor.java | 73 +++++++++++--
.../oozie/action/hadoop/LauncherMainTester.java | 50 +++++++--
.../action/hadoop/TestBytesAndUOMConverter.java | 102 +++++++++++++++++++
.../action/hadoop/TestHeapModifiersPattern.java | 54 ++++++++++
.../action/hadoop/TestJavaActionExecutor.java | 28 +++++
release-log.txt | 1 +
7 files changed, 371 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/main/java/org/apache/oozie/action/hadoop/BytesAndUOMConverter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/BytesAndUOMConverter.java b/core/src/main/java/org/apache/oozie/action/hadoop/BytesAndUOMConverter.java
new file mode 100644
index 0000000..95b72bd
--- /dev/null
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/BytesAndUOMConverter.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import org.apache.oozie.util.XLog;
+
+import static org.apache.commons.io.FileUtils.ONE_GB;
+import static org.apache.commons.io.FileUtils.ONE_KB;
+import static org.apache.commons.io.FileUtils.ONE_MB;
+
+/**
+ * Converts {@code String}s that contain byte counts and Units of Measure (K, M, or G) to a {@code long bytesCount} in the
+ * desired Unit of Measure.
+ */
+class BytesAndUOMConverter {
+ private static final XLog LOG = XLog.getLog(BytesAndUOMConverter.class);
+
+ /**
+ * Convert {@code unitAndUOM} to {@code long bytesCount} in megabytes.
+ * @param unitAndUOM a {@code String} consisting of count and Unit of Measure (K, M, or G)
+ * @return {@code long bytesCount} in megabytes
+ */
+ long toMegabytes(final String unitAndUOM) {
+ Preconditions.checkArgument(!Strings.isNullOrEmpty(unitAndUOM), "unitAndUOM should not be empty");
+ Preconditions.checkArgument(unitAndUOM.toUpperCase().endsWith("K")
+ || unitAndUOM.toUpperCase().endsWith("M")
+ || unitAndUOM.toUpperCase().endsWith("G")
+ || Character.isDigit(unitAndUOM.charAt(unitAndUOM.length() -1)),
+ "unitAndUOM should end with a proper UoM or with a digit");
+
+ try {
+ final long bytesCount;
+
+ if (unitAndUOM.toUpperCase().endsWith("K")) {
+ bytesCount = getUnit(unitAndUOM) * ONE_KB;
+ }
+ else if (unitAndUOM.toUpperCase().endsWith("M")) {
+ bytesCount = getUnit(unitAndUOM) * ONE_MB;
+ }
+ else if (unitAndUOM.toUpperCase().endsWith("G")) {
+ bytesCount = getUnit(unitAndUOM) * ONE_GB;
+ }
+ else {
+ bytesCount = Long.parseLong(unitAndUOM);
+ }
+
+ Preconditions.checkArgument(bytesCount > 0L, "unit should be positive");
+
+ return bytesCount / ONE_MB;
+ }
+ catch (final NumberFormatException e) {
+ LOG.error("Cannot parse bytes and UoM {0}, cannot convert to megabytes.");
+ throw e;
+ }
+ }
+
+ private long getUnit(final String unitAndUOM) {
+ return Long.parseLong(unitAndUOM.substring(0, unitAndUOM.length() - 1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 0385c77..7305a12 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -45,6 +45,7 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -113,6 +114,7 @@ import org.jdom.Namespace;
import java.util.Objects;
import java.util.Properties;
import java.util.Set;
+import java.util.regex.Pattern;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@@ -169,6 +171,31 @@ public class JavaActionExecutor extends ActionExecutor {
private final static String ACTION_SHARELIB_FOR = "oozie.action.sharelib.for.";
public static final String OOZIE_ACTION_DEPENDENCY_DEDUPLICATE = "oozie.action.dependency.deduplicate";
+ /**
+ * Heap to physical memory ration for {@link LauncherAM}, in order its YARN container doesn't get killed before physical memory
+ * gets exhausted.
+ */
+ private static final double LAUNCHER_HEAP_PMEM_RATIO = 0.8;
+
+ /**
+ * Matches one or more occurrence of {@code Xmx}, {@code Xms}, {@code mx}, {@code ms}, {@code XX:MaxHeapSize}, or
+ * {@code XX:MinHeapSize} JVM parameters, mixed with any other content.
+ * <p>
+ * Examples:
+ * <ul>
+ * <li>{@code -Xms384m}</li>
+ * <li>{@code -Xmx:789k}</li>
+ * <li>{@code -XX:MaxHeapSize=123g}</li>
+ * <li>{@code -ms:384m}</li>
+ * <li>{@code -mx789k}</li>
+ * <li>{@code -XX:MinHeapSize=123g}</li>
+ * </ul>
+ */
+ @VisibleForTesting
+ @SuppressFBWarnings(value = {"REDOS"}, justification = "Complex regular expression")
+ static final Pattern HEAP_MODIFIERS_PATTERN =
+ Pattern.compile(".*((\\-X?m[s|x][\\:]?)|(\\-XX\\:(Min|Max)HeapSize\\=))([0-9]+[kKmMgG]?).*");
+
private static int maxActionOutputLen;
private static int maxExternalStatsSize;
private static int maxFSGlobMax;
@@ -1265,6 +1292,8 @@ public class JavaActionExecutor extends ActionExecutor {
vargs.add(oozieLauncherJavaOpts);
}
}
+
+ checkAndSetMaxHeap(launcherJobConf, vargs);
}
private boolean handleJavaOpts(Element actionXml, StringBuilder javaOpts) {
@@ -1289,6 +1318,29 @@ public class JavaActionExecutor extends ActionExecutor {
return oldJavaOpts;
}
+ private void checkAndSetMaxHeap(final Configuration launcherJobConf, final List<String> vargs) {
+ LOG.debug("Checking and setting max heap for the LauncherAM");
+
+ final int launcherMemoryMb = readMemoryMb(launcherJobConf);
+ final int calculatedHeapMaxMb = (int) (launcherMemoryMb * LAUNCHER_HEAP_PMEM_RATIO);
+
+ boolean heapModifiersPresent = false;
+ for (final String varg : vargs) {
+ if (HEAP_MODIFIERS_PATTERN.matcher(varg).matches()) {
+ heapModifiersPresent = true;
+ }
+ }
+ if (heapModifiersPresent) {
+ LOG.trace("Some heap modifier JVM options are configured by the user, leaving LauncherAM's maximum heap option");
+ }
+ else {
+ LOG.trace("No heap modifier JVM options are configured by the user, overriding LauncherAM's maximum heap option");
+
+ LOG.debug("Calculated maximum heap option {0} MB set for the LauncherAM", calculatedHeapMaxMb);
+ vargs.add(String.format("-Xmx%sm", calculatedHeapMaxMb));
+ }
+ }
+
private void setApplicationName(final Context context,
final WorkflowAction action,
final ApplicationSubmissionContext appContext) {
@@ -1340,28 +1392,35 @@ public class JavaActionExecutor extends ActionExecutor {
appContext.setPriority(pri);
}
- private void setResources(Configuration launcherJobConf, ApplicationSubmissionContext appContext) {
- int memory;
+ private void setResources(final Configuration launcherJobConf, final ApplicationSubmissionContext appContext) {
+ final Resource resource = Resource.newInstance(readMemoryMb(launcherJobConf), readVCores(launcherJobConf));
+ appContext.setResource(resource);
+ }
+
+ private int readMemoryMb(final Configuration launcherJobConf) {
+ final int memory;
if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY) != null) {
memory = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_MEMORY_MB_PROPERTY, -1);
Preconditions.checkArgument(memory > 0, "Launcher memory is 0 or negative");
} else {
- int defaultMemory = ConfigurationService.getInt(DEFAULT_LAUNCHER_MEMORY_MB, -1);
+ final int defaultMemory = ConfigurationService.getInt(DEFAULT_LAUNCHER_MEMORY_MB, -1);
Preconditions.checkArgument(defaultMemory > 0, "Default launcher memory is 0 or negative");
memory = defaultMemory;
}
+ return memory;
+ }
- int vcores;
+ private int readVCores(final Configuration launcherJobConf) {
+ final int vcores;
if (launcherJobConf.get(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY) != null) {
vcores = launcherJobConf.getInt(LauncherAM.OOZIE_LAUNCHER_VCORES_PROPERTY, -1);
Preconditions.checkArgument(vcores > 0, "Launcher vcores is 0 or negative");
} else {
- int defaultVcores = ConfigurationService.getInt(DEFAULT_LAUNCHER_VCORES);
+ final int defaultVcores = ConfigurationService.getInt(DEFAULT_LAUNCHER_VCORES);
Preconditions.checkArgument(defaultVcores > 0, "Default launcher vcores is 0 or negative");
vcores = defaultVcores;
}
- Resource resource = Resource.newInstance(memory, vcores);
- appContext.setResource(resource);
+ return vcores;
}
private Map<String, String> extractEnvVarsFromOozieLauncherProps(String oozieLauncherEnvProperty) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
index ada7005..bcab80c 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/LauncherMainTester.java
@@ -18,6 +18,7 @@
package org.apache.oozie.action.hadoop;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
@@ -37,8 +38,14 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.Writer;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Properties;
+import static org.apache.commons.io.FileUtils.ONE_KB;
+import static org.apache.commons.io.FileUtils.ONE_MB;
+
public class LauncherMainTester {
public static final String JOB_ID_FILE_NAME = "jobID.txt";
@@ -57,20 +64,21 @@ public class LauncherMainTester {
throw new RuntimeException("Failing on purpose");
}
+ final String firstArgument = args[0];
if (args.length == 1) {
- if (args[0].equals("throwable")) {
+ if (firstArgument.equals("throwable")) {
throw new Throwable("throwing throwable");
}
- if (args[0].equals("exception")) {
+ if (firstArgument.equals("exception")) {
throw new IOException("throwing exception");
}
- if (args[0].equals("exit0")) {
+ if (firstArgument.equals("exit0")) {
System.exit(0);
}
- if (args[0].equals("exit1")) {
+ if (firstArgument.equals("exit1")) {
System.exit(1);
}
- if (args[0].equals("out")) {
+ if (firstArgument.equals("out")) {
File file = new File(System.getProperty("oozie.action.output.properties"));
Properties props = new Properties();
props.setProperty("a", "A");
@@ -79,7 +87,7 @@ public class LauncherMainTester {
os.close();
System.out.println(file.getAbsolutePath());
}
- if (args[0].equals("id")) {
+ if (firstArgument.equals("id")) {
File file = new File(System.getProperty("oozie.action.newId"));
Properties props = new Properties();
props.setProperty("id", "IDSWAP");
@@ -88,7 +96,7 @@ public class LauncherMainTester {
os.close();
System.out.println(file.getAbsolutePath());
}
- if (args[0].equals("securityManager")) {
+ if (firstArgument.equals("securityManager")) {
SecurityManager sm = System.getSecurityManager();
if (sm == null) {
throw new Throwable("no security manager");
@@ -100,9 +108,12 @@ public class LauncherMainTester {
sm.checkPermission(null);
sm.checkPermission(null, sm.getSecurityContext());
}
+ if (firstArgument.startsWith("-Xmx")) {
+ tryAllocate(firstArgument);
+ }
}
if(args.length == 3) {
- if(args[0].equals("javamapreduce")) {
+ if(firstArgument.equals("javamapreduce")) {
executeJavaMapReduce(args);
}
}
@@ -150,6 +161,29 @@ public class LauncherMainTester {
System.out.println("Job Id written to file");
}
+ private static void tryAllocate(final String xmxParameter) {
+ Preconditions.checkArgument(JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher(xmxParameter).matches(),
+ String.format("malformed heap modifier pattern [%s]", xmxParameter));
+
+ final String xmxParameterKey = "-Xmx";
+ final String configuredHeapMaxUnitAndUOM =
+ xmxParameter.substring(xmxParameter.indexOf(xmxParameterKey) + xmxParameterKey.length());
+ final long configuredHeapMaxMb = new BytesAndUOMConverter().toMegabytes(
+ configuredHeapMaxUnitAndUOM);
+
+ System.out.println(String.format("Trying to allocate in total [%s] megabytes", configuredHeapMaxMb));
+
+ final List<ByteBuffer> megabytes = new ArrayList<>();
+ for (int ixMB = 0; ixMB < configuredHeapMaxMb; ixMB++) {
+ megabytes.add(ByteBuffer.allocate((int) ONE_MB));
+ if (ixMB % (ONE_KB / 8) == 0) {
+ System.out.println(String.format("Allocated [%s] megabytes", ixMB));
+ }
+ }
+
+ System.out.println(String.format("All [%s] megabytes allocated successfully", configuredHeapMaxMb));
+ }
+
private static void checkAndSleep(String args[]) throws InterruptedException {
if (args.length == 2 && args[0].equals("sleep")) {
long sleepTime = Long.parseLong(args[1]);
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/test/java/org/apache/oozie/action/hadoop/TestBytesAndUOMConverter.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestBytesAndUOMConverter.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestBytesAndUOMConverter.java
new file mode 100644
index 0000000..8ad820b
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestBytesAndUOMConverter.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestBytesAndUOMConverter {
+ @Rule
+ public final ExpectedException expectedException = ExpectedException.none();
+
+ @Test
+ public void whenEmptyInputIsGivenException() {
+ expectedException.expect(IllegalArgumentException.class);
+
+ new BytesAndUOMConverter().toMegabytes(null);
+ }
+
+ @Test
+ public void whenUOMIsIncorrectException() {
+ expectedException.expect(IllegalArgumentException.class);
+
+ new BytesAndUOMConverter().toMegabytes("123T");
+ }
+
+ @Test
+ public void whenNoUnitIsGivenException() {
+ expectedException.expect(NumberFormatException.class);
+
+ new BytesAndUOMConverter().toMegabytes("K");
+ }
+
+ @Test
+ public void whenIncorrectUnitIsGivenException() {
+ expectedException.expect(NumberFormatException.class);
+
+ new BytesAndUOMConverter().toMegabytes("1aa1K");
+ }
+
+ @Test
+ public void whenNotPositiveUnitIsGivenException() {
+ expectedException.expect(IllegalArgumentException.class);
+
+ new BytesAndUOMConverter().toMegabytes("0K");
+ }
+
+ @Test
+ public void whenUnitIsGivenAndNoUOMIsPresentConvertedCorrectly() {
+ assertEquals("bytes count should be converted correctly",
+ 1L,
+ new BytesAndUOMConverter().toMegabytes(
+ Integer.toString(new Double(Math.pow(2, 20)).intValue())));
+ }
+
+ @Test
+ public void whenMegabytesAreGivenSameReturned() {
+ assertEquals("megabytes count should remain unchanged",
+ 1L,
+ new BytesAndUOMConverter().toMegabytes("1M"));
+ }
+
+ @Test
+ public void whenKilobytesAreGivenConvertedCorrectly() {
+ assertEquals("kilobytes count should be converted correctly",
+ 1L,
+ new BytesAndUOMConverter().toMegabytes("1024K"));
+
+ assertEquals("kilobytes count should be converted correctly",
+ 0L,
+ new BytesAndUOMConverter().toMegabytes("1023K"));
+
+ assertEquals("kilobytes count should be converted correctly",
+ 10L,
+ new BytesAndUOMConverter().toMegabytes("10240K"));
+ }
+
+ @Test
+ public void whenGigabytesAreGivenConvertedCorrectly() {
+ assertEquals("gigabytes count should be converted correctly",
+ 1024L,
+ new BytesAndUOMConverter().toMegabytes("1G"));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/test/java/org/apache/oozie/action/hadoop/TestHeapModifiersPattern.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestHeapModifiersPattern.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestHeapModifiersPattern.java
new file mode 100644
index 0000000..64afc94
--- /dev/null
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestHeapModifiersPattern.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+public class TestHeapModifiersPattern {
+
+ @Test
+ public void whenMatchingParameterIsGivenValueIsExtracted() {
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-Xms1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-Xmx1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-ms1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-mx1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-Xms1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-Xms1G").matches());
+
+ assertTrue("matching parameter value should be found",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-Xms1G -Xmx:123K -XX:+UnlockExperimentalVMOptions").matches());
+
+ assertFalse("not matching parameter value should not be extracted",
+ JavaActionExecutor.HEAP_MODIFIERS_PATTERN.matcher("-XX:+UnlockExperimentalVMOptions").matches());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
index 784dc96..6383e81 100644
--- a/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
+++ b/core/src/test/java/org/apache/oozie/action/hadoop/TestJavaActionExecutor.java
@@ -2697,4 +2697,32 @@ public class TestJavaActionExecutor extends ActionExecutorTestCase {
" </configuration>" +
"</global>";
}
+
+ public void testSubmitOKWithLauncherJavaOptsExhaustingHeap() throws Exception {
+ final String actionXml = "<java>" +
+ " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" +
+ " <name-node>" + getNameNodeUri() + "</name-node>" +
+ " <configuration>" +
+ " <property>" +
+ " <name>oozie.launcher.javaopts</name>" +
+ " <value>-Xms512m -Xmx1536m -XX:-DisableExplicitGC</value>" +
+ " </property>" +
+ " </configuration>" +
+ " <main-class>" + LauncherMainTester.class.getName() + "</main-class>" +
+ " <arg>-Xmx3072m</arg>" +
+ "</java>";
+ final Context context = createContext(actionXml, null);
+ submitAction(context);
+ waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId());
+ ActionExecutor ae = new JavaActionExecutor();
+ ae.check(context, context.getAction());
+ assertEquals("FAILED/KILLED", context.getAction().getExternalStatus());
+ assertNull(context.getAction().getData());
+
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus());
+
+ assertTrue("error message should contain: \"Java heap space\"",
+ context.getAction().getErrorMessage().contains("Java heap space"));
+ }
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/57c2a2f5/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index a99c399..0a97e34 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.1.0 release (trunk - unreleased)
+OOZIE-3307 [core] Limit heap usage of LauncherAM (andras.piros)
OOZIE-3352 [tests] TestCallableQueueService#testPriorityExecutionOrder() is flaky (pbacsko)
OOZIE-3351 [tests] Flaky test TestMemoryLocks#testWriteLockSameThreadNoWait() (pbacsko)
OOZIE-3229 [client] [ui] Improved SLA filtering options (asalamon74, andras.piros)