You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by tg...@apache.org on 2014/09/11 15:13:21 UTC
git commit: [SPARK-2140] Updating heap memory calculation for YARN
stable and alpha.
Repository: spark
Updated Branches:
refs/heads/master c27718f37 -> ed1980ffa
[SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Updated pull request, reflecting YARN stable and alpha states. I am getting intermittent test failures on my own test infrastructure. Is that tracked anywhere yet?
Author: Chris Cope <cc...@resilientscience.com>
Closes #2253 from copester/master and squashes the following commits:
5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since they are no longer needed.
52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ed1980ff
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ed1980ff
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ed1980ff
Branch: refs/heads/master
Commit: ed1980ffa9ccb87d76694ba910ef22df034bca49
Parents: c27718f
Author: Chris Cope <cc...@resilientscience.com>
Authored: Thu Sep 11 08:13:07 2014 -0500
Committer: Thomas Graves <tg...@apache.org>
Committed: Thu Sep 11 08:13:07 2014 -0500
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 8 --------
.../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +---
.../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 ---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 9 ---------
4 files changed, 1 insertion(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 10fc39b..aff9ab7 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -103,14 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
appContext
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int = {
- val minResMemory = newApp.getMinimumResourceCapability().getMemory()
- val amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- memoryOverhead)
- amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 8075b7a..c96f731 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -300,8 +300,6 @@ trait ClientBase extends Logging {
retval.toString
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int
-
def setupSecurityToken(amContainer: ContainerLaunchContext)
def createContainerLaunchContext(
@@ -346,7 +344,7 @@ trait ClientBase extends Logging {
}
amContainer.setEnvironment(env)
- val amMemory = calculateAMMemory(newApp)
+ val amMemory = args.amMemory
val javaOpts = ListBuffer[String]()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
----------------------------------------------------------------------
diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
index 68cc289..5480eca 100644
--- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
+++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
@@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers {
val sparkConf: SparkConf,
val yarnConf: YarnConfiguration) extends ClientBase {
- override def calculateAMMemory(newApp: GetNewApplicationResponse): Int =
- throw new UnsupportedOperationException()
-
override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit =
throw new UnsupportedOperationException()
http://git-wip-us.apache.org/repos/asf/spark/blob/ed1980ff/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 313a0d2..82e45e3 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -103,15 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
clusterMetrics.getNumNodeManagers)
}
- def calculateAMMemory(newApp: GetNewApplicationResponse) :Int = {
- // TODO: Need a replacement for the following code to fix -Xmx?
- // val minResMemory: Int = newApp.getMinimumResourceCapability().getMemory()
- // var amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- // ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- // memoryOverhead )
- args.amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org