You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/01/22 23:10:17 UTC
[1/3] git commit: SPARK-1033. Ask for cores in Yarn container requests
Updated Branches:
refs/heads/master 5bcfd7981 -> 576c4a4c5
SPARK-1033. Ask for cores in Yarn container requests
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/3e85b87d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/3e85b87d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/3e85b87d
Branch: refs/heads/master
Commit: 3e85b87d9033e6d9a2634f7598abc3acee77486f
Parents: 792d908
Author: Sandy Ryza <sa...@cloudera.com>
Authored: Sun Jan 19 10:16:25 2014 -0800
Committer: Sandy Ryza <sa...@cloudera.com>
Committed: Mon Jan 20 14:42:32 2014 -0800
----------------------------------------------------------------------
docs/running-on-yarn.md | 2 +-
.../apache/spark/deploy/yarn/YarnAllocationHandler.scala | 9 +++++----
2 files changed, 6 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/3e85b87d/docs/running-on-yarn.md
----------------------------------------------------------------------
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index 3bd6264..fb8a043 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -133,7 +133,7 @@ See [Building Spark with Maven](building-with-maven.html) for instructions on ho
# Important Notes
-- We do not requesting container resources based on the number of cores. Thus the numbers of cores given via command line arguments cannot be guaranteed.
+- Before Hadoop 2.2, YARN does not support cores in container resource requests. Thus, when running against an earlier version, the numbers of cores given via command line arguments cannot be guaranteed.
- The local directories used for spark will be the local directories configured for YARN (Hadoop Yarn config yarn.nodemanager.local-dirs). If the user specifies spark.local.dir, it will be ignored.
- The --files and --archives options support specifying file names with the # similar to Hadoop. For example you can specify: --files localtest.txt#appSees.txt and this will upload the file you have locally named localtest.txt into HDFS but this will be linked to by the name appSees.txt and your application should use the name as appSees.txt to reference it when running on YARN.
- The --addJars option allows the SparkContext.addJar function to work if you are using it with local files. It does not need to be used if you are using it with HDFS, HTTP, HTTPS, or FTP files.
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/3e85b87d/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index 738ff98..f53c130 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -102,7 +102,8 @@ private[yarn] class YarnAllocationHandler(
def getNumWorkersFailed: Int = numWorkersFailed.intValue
def isResourceConstraintSatisfied(container: Container): Boolean = {
- container.getResource.getMemory >= (workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD)
+ (container.getResource.getMemory >= (workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD)
+ && container.getResource.getVirtualCores >= workerCores)
}
def releaseContainer(container: Container) {
@@ -532,15 +533,15 @@ private[yarn] class YarnAllocationHandler(
priority: Int
): ArrayBuffer[ContainerRequest] = {
- val memoryResource = Records.newRecord(classOf[Resource])
- memoryResource.setMemory(workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD)
+ val memoryRequest = workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD
+ val resource = Resource.newInstance(memoryRequest, workerCores)
val prioritySetting = Records.newRecord(classOf[Priority])
prioritySetting.setPriority(priority)
val requests = new ArrayBuffer[ContainerRequest]()
for (i <- 0 until numWorkers) {
- requests += new ContainerRequest(memoryResource, hosts, racks, prioritySetting)
+ requests += new ContainerRequest(resource, hosts, racks, prioritySetting)
}
requests
}
[2/3] git commit: Incorporate Tom's comments - update doc and code to
reflect that core requests may not always be honored
Posted by pw...@apache.org.
Incorporate Tom's comments - update doc and code to reflect that core requests may not always be honored
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/adf42611
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/adf42611
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/adf42611
Branch: refs/heads/master
Commit: adf42611f16daec091af10f2c9b39bd154dc67b3
Parents: 3e85b87
Author: Sandy Ryza <sa...@cloudera.com>
Authored: Tue Jan 21 00:38:02 2014 -0800
Committer: Sandy Ryza <sa...@cloudera.com>
Committed: Tue Jan 21 00:38:02 2014 -0800
----------------------------------------------------------------------
docs/running-on-yarn.md | 2 +-
.../org/apache/spark/deploy/yarn/YarnAllocationHandler.scala | 3 +--
2 files changed, 2 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/adf42611/docs/running-on-yarn.md
----------------------------------------------------------------------
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index fb8a043..5dadd54 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -133,7 +133,7 @@ See [Building Spark with Maven](building-with-maven.html) for instructions on ho
# Important Notes
-- Before Hadoop 2.2, YARN does not support cores in container resource requests. Thus, when running against an earlier version, the numbers of cores given via command line arguments cannot be guaranteed.
+- Before Hadoop 2.2, YARN does not support cores in container resource requests. Thus, when running against an earlier version, the numbers of cores given via command line arguments cannot be passed to YARN. Whether core requests are honored in scheduling decisions depends on which scheduler is in use and how it is configured.
- The local directories used for spark will be the local directories configured for YARN (Hadoop Yarn config yarn.nodemanager.local-dirs). If the user specifies spark.local.dir, it will be ignored.
- The --files and --archives options support specifying file names with the # similar to Hadoop. For example you can specify: --files localtest.txt#appSees.txt and this will upload the file you have locally named localtest.txt into HDFS but this will be linked to by the name appSees.txt and your application should use the name as appSees.txt to reference it when running on YARN.
- The --addJars option allows the SparkContext.addJar function to work if you are using it with local files. It does not need to be used if you are using it with HDFS, HTTP, HTTPS, or FTP files.
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/adf42611/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
----------------------------------------------------------------------
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
index f53c130..1ac6112 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocationHandler.scala
@@ -102,8 +102,7 @@ private[yarn] class YarnAllocationHandler(
def getNumWorkersFailed: Int = numWorkersFailed.intValue
def isResourceConstraintSatisfied(container: Container): Boolean = {
- (container.getResource.getMemory >= (workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD)
- && container.getResource.getVirtualCores >= workerCores)
+ container.getResource.getMemory >= (workerMemory + YarnAllocationHandler.MEMORY_OVERHEAD)
}
def releaseContainer(container: Container) {
[3/3] git commit: Merge pull request #478 from sryza/sandy-spark-1033
Posted by pw...@apache.org.
Merge pull request #478 from sryza/sandy-spark-1033
SPARK-1033. Ask for cores in Yarn container requests
Tested on a pseudo-distributed cluster against the Fair Scheduler and observed a worker taking more than a single core.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/576c4a4c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/576c4a4c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/576c4a4c
Branch: refs/heads/master
Commit: 576c4a4c502ccca5fcd6b3552dd93cc2f3c50666
Parents: 5bcfd79 adf4261
Author: Patrick Wendell <pw...@gmail.com>
Authored: Wed Jan 22 14:10:07 2014 -0800
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Jan 22 14:10:07 2014 -0800
----------------------------------------------------------------------
docs/running-on-yarn.md | 2 +-
.../org/apache/spark/deploy/yarn/YarnAllocationHandler.scala | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------