You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by va...@apache.org on 2017/01/25 22:23:06 UTC
spark git commit: [SPARK-18750][YARN] Follow up: move test to correct
directory in 2.1 branch.
Repository: spark
Updated Branches:
refs/heads/branch-2.1 c9f075abb -> 97d3353ef
[SPARK-18750][YARN] Follow up: move test to correct directory in 2.1 branch.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #16704 from vanzin/SPARK-18750_2.1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/97d3353e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/97d3353e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/97d3353e
Branch: refs/heads/branch-2.1
Commit: 97d3353ef16a6e6edc93d8177b08442a03e19eee
Parents: c9f075a
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Jan 25 14:22:58 2017 -0800
Committer: Marcelo Vanzin <va...@cloudera.com>
Committed: Wed Jan 25 14:22:58 2017 -0800
----------------------------------------------------------------------
.../yarn/LocalityPlacementStrategySuite.scala | 87 --------------------
.../yarn/LocalityPlacementStrategySuite.scala | 87 ++++++++++++++++++++
2 files changed, 87 insertions(+), 87 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/97d3353e/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
----------------------------------------------------------------------
diff --git a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
deleted file mode 100644
index fb80ff9..0000000
--- a/resource-managers/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
+++ /dev/null
@@ -1,87 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.deploy.yarn
-
-import scala.collection.mutable.{HashMap, HashSet, Set}
-
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic
-import org.apache.hadoop.net.DNSToSwitchMapping
-import org.apache.hadoop.yarn.api.records._
-import org.apache.hadoop.yarn.conf.YarnConfiguration
-import org.mockito.Mockito._
-
-import org.apache.spark.{SparkConf, SparkFunSuite}
-
-class LocalityPlacementStrategySuite extends SparkFunSuite {
-
- test("handle large number of containers and tasks (SPARK-18750)") {
- // Run the test in a thread with a small stack size, since the original issue
- // surfaced as a StackOverflowError.
- var error: Throwable = null
-
- val runnable = new Runnable() {
- override def run(): Unit = try {
- runTest()
- } catch {
- case e: Throwable => error = e
- }
- }
-
- val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 32 * 1024)
- thread.start()
- thread.join()
-
- assert(error === null)
- }
-
- private def runTest(): Unit = {
- val yarnConf = new YarnConfiguration()
- yarnConf.setClass(
- CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
- classOf[MockResolver], classOf[DNSToSwitchMapping])
-
- // The numbers below have been chosen to balance being large enough to replicate the
- // original issue while not taking too long to run when the issue is fixed. The main
- // goal is to create enough requests for localized containers (so there should be many
- // tasks on several hosts that have no allocated containers).
-
- val resource = Resource.newInstance(8 * 1024, 4)
- val strategy = new LocalityPreferredContainerPlacementStrategy(new SparkConf(),
- yarnConf, resource)
-
- val totalTasks = 32 * 1024
- val totalContainers = totalTasks / 16
- val totalHosts = totalContainers / 16
-
- val mockId = mock(classOf[ContainerId])
- val hosts = (1 to totalHosts).map { i => (s"host_$i", totalTasks % i) }.toMap
- val containers = (1 to totalContainers).map { i => mockId }
- val count = containers.size / hosts.size / 2
-
- val hostToContainerMap = new HashMap[String, Set[ContainerId]]()
- hosts.keys.take(hosts.size / 2).zipWithIndex.foreach { case (host, i) =>
- val hostContainers = new HashSet[ContainerId]()
- containers.drop(count * i).take(i).foreach { c => hostContainers += c }
- hostToContainerMap(host) = hostContainers
- }
-
- strategy.localityOfRequestedContainers(containers.size * 2, totalTasks, hosts,
- hostToContainerMap, Nil)
- }
-
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/97d3353e/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
----------------------------------------------------------------------
diff --git a/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala b/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
new file mode 100644
index 0000000..fb80ff9
--- /dev/null
+++ b/yarn/src/test/scala/org/apache/spark/deploy/yarn/LocalityPlacementStrategySuite.scala
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.deploy.yarn
+
+import scala.collection.mutable.{HashMap, HashSet, Set}
+
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic
+import org.apache.hadoop.net.DNSToSwitchMapping
+import org.apache.hadoop.yarn.api.records._
+import org.apache.hadoop.yarn.conf.YarnConfiguration
+import org.mockito.Mockito._
+
+import org.apache.spark.{SparkConf, SparkFunSuite}
+
+class LocalityPlacementStrategySuite extends SparkFunSuite {
+
+ test("handle large number of containers and tasks (SPARK-18750)") {
+ // Run the test in a thread with a small stack size, since the original issue
+ // surfaced as a StackOverflowError.
+ var error: Throwable = null
+
+ val runnable = new Runnable() {
+ override def run(): Unit = try {
+ runTest()
+ } catch {
+ case e: Throwable => error = e
+ }
+ }
+
+ val thread = new Thread(new ThreadGroup("test"), runnable, "test-thread", 32 * 1024)
+ thread.start()
+ thread.join()
+
+ assert(error === null)
+ }
+
+ private def runTest(): Unit = {
+ val yarnConf = new YarnConfiguration()
+ yarnConf.setClass(
+ CommonConfigurationKeysPublic.NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY,
+ classOf[MockResolver], classOf[DNSToSwitchMapping])
+
+ // The numbers below have been chosen to balance being large enough to replicate the
+ // original issue while not taking too long to run when the issue is fixed. The main
+ // goal is to create enough requests for localized containers (so there should be many
+ // tasks on several hosts that have no allocated containers).
+
+ val resource = Resource.newInstance(8 * 1024, 4)
+ val strategy = new LocalityPreferredContainerPlacementStrategy(new SparkConf(),
+ yarnConf, resource)
+
+ val totalTasks = 32 * 1024
+ val totalContainers = totalTasks / 16
+ val totalHosts = totalContainers / 16
+
+ val mockId = mock(classOf[ContainerId])
+ val hosts = (1 to totalHosts).map { i => (s"host_$i", totalTasks % i) }.toMap
+ val containers = (1 to totalContainers).map { i => mockId }
+ val count = containers.size / hosts.size / 2
+
+ val hostToContainerMap = new HashMap[String, Set[ContainerId]]()
+ hosts.keys.take(hosts.size / 2).zipWithIndex.foreach { case (host, i) =>
+ val hostContainers = new HashSet[ContainerId]()
+ containers.drop(count * i).take(i).foreach { c => hostContainers += c }
+ hostToContainerMap(host) = hostContainers
+ }
+
+ strategy.localityOfRequestedContainers(containers.size * 2, totalTasks, hosts,
+ hostToContainerMap, Nil)
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org