You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2015/10/06 19:17:28 UTC
spark git commit: [SPARK-10916] [YARN] Set perm gen size when
launching containers on YARN.
Repository: spark
Updated Branches:
refs/heads/master 27ecfe61f -> 744f03e70
[SPARK-10916] [YARN] Set perm gen size when launching containers on YARN.
This makes YARN containers behave like all other processes launched by
Spark, which launch with a default perm gen size of 256m unless
overridden by the user (or not needed by the vm).
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #8970 from vanzin/SPARK-10916.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/744f03e7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/744f03e7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/744f03e7
Branch: refs/heads/master
Commit: 744f03e700b0e3a7c2a92e92edc79d2374c19023
Parents: 27ecfe6
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Tue Oct 6 10:17:12 2015 -0700
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Tue Oct 6 10:17:12 2015 -0700
----------------------------------------------------------------------
.../spark/launcher/WorkerCommandBuilder.scala | 2 +-
.../spark/launcher/AbstractCommandBuilder.java | 23 --------------------
.../spark/launcher/CommandBuilderUtils.java | 23 ++++++++++++++++++++
.../org/apache/spark/deploy/yarn/Client.scala | 4 +++-
.../spark/deploy/yarn/ExecutorRunnable.scala | 2 ++
.../launcher/YarnCommandBuilderUtils.scala | 21 ++++++++++++++++--
6 files changed, 48 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
index 0c09665..a2add61 100644
--- a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
@@ -40,7 +40,7 @@ private[spark] class WorkerCommandBuilder(sparkHome: String, memoryMb: Int, comm
cmd.add(s"-Xms${memoryMb}M")
cmd.add(s"-Xmx${memoryMb}M")
command.javaOpts.foreach(cmd.add)
- addPermGenSizeOpt(cmd)
+ CommandBuilderUtils.addPermGenSizeOpt(cmd)
addOptionString(cmd, getenv("SPARK_JAVA_OPTS"))
cmd
}
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 0a237ee..610e8bd 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -116,29 +116,6 @@ abstract class AbstractCommandBuilder {
return cmd;
}
- /**
- * Adds the default perm gen size option for Spark if the VM requires it and the user hasn't
- * set it.
- */
- void addPermGenSizeOpt(List<String> cmd) {
- // Don't set MaxPermSize for IBM Java, or Oracle Java 8 and later.
- if (getJavaVendor() == JavaVendor.IBM) {
- return;
- }
- String[] version = System.getProperty("java.version").split("\\.");
- if (Integer.parseInt(version[0]) > 1 || Integer.parseInt(version[1]) > 7) {
- return;
- }
-
- for (String arg : cmd) {
- if (arg.startsWith("-XX:MaxPermSize=")) {
- return;
- }
- }
-
- cmd.add("-XX:MaxPermSize=256m");
- }
-
void addOptionString(List<String> cmd, String options) {
if (!isEmpty(options)) {
for (String opt : parseOptionString(options)) {
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
----------------------------------------------------------------------
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index a16c0d2..d30c2ec 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -313,4 +313,27 @@ class CommandBuilderUtils {
return quoted.append('"').toString();
}
+ /**
+ * Adds the default perm gen size option for Spark if the VM requires it and the user hasn't
+ * set it.
+ */
+ static void addPermGenSizeOpt(List<String> cmd) {
+ // Don't set MaxPermSize for IBM Java, or Oracle Java 8 and later.
+ if (getJavaVendor() == JavaVendor.IBM) {
+ return;
+ }
+ String[] version = System.getProperty("java.version").split("\\.");
+ if (Integer.parseInt(version[0]) > 1 || Integer.parseInt(version[1]) > 7) {
+ return;
+ }
+
+ for (String arg : cmd) {
+ if (arg.startsWith("-XX:MaxPermSize=")) {
+ return;
+ }
+ }
+
+ cmd.add("-XX:MaxPermSize=256m");
+ }
+
}
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 8c53c24..f8748ef 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -54,8 +54,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException
import org.apache.hadoop.yarn.util.Records
-import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkException}
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.launcher.YarnCommandBuilderUtils
import org.apache.spark.util.Utils
private[spark] class Client(
@@ -730,6 +731,7 @@ private[spark] class Client(
// For log4j configuration to reference
javaOpts += ("-Dspark.yarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR)
+ YarnCommandBuilderUtils.addPermGenSizeOpt(javaOpts)
val userClass =
if (isClusterMode) {
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 9abd09b..2232ffb 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkException}
+import org.apache.spark.launcher.YarnCommandBuilderUtils
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.Utils
@@ -199,6 +200,7 @@ class ExecutorRunnable(
// For log4j configuration to reference
javaOpts += ("-Dspark.yarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR)
+ YarnCommandBuilderUtils.addPermGenSizeOpt(javaOpts)
val userClassPath = Client.getUserClasspath(sparkConf).flatMap { uri =>
val absPath =
http://git-wip-us.apache.org/repos/asf/spark/blob/744f03e7/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
----------------------------------------------------------------------
diff --git a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
index 3ac36ef..7d246bf 100644
--- a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
+++ b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
@@ -17,11 +17,28 @@
package org.apache.spark.launcher
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
+
/**
- * Exposes needed methods
+ * Exposes methods from the launcher library that are used by the YARN backend.
*/
private[spark] object YarnCommandBuilderUtils {
- def quoteForBatchScript(arg: String) : String = {
+
+ def quoteForBatchScript(arg: String): String = {
CommandBuilderUtils.quoteForBatchScript(arg)
}
+
+ /**
+ * Adds the perm gen configuration to the list of java options if needed and not yet added.
+ *
+ * Note that this method adds the option based on the local JVM version; if the node where
+ * the container is running has a different Java version, there's a risk that the option will
+ * not be added (e.g. if the AM is running Java 8 but the container's node is set up to use
+ * Java 7).
+ */
+ def addPermGenSizeOpt(args: ListBuffer[String]): Unit = {
+ CommandBuilderUtils.addPermGenSizeOpt(args.asJava)
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org