You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2015/10/22 12:59:37 UTC
spark git commit: [SPARK-11121][CORE] Correct the TaskLocation type
Repository: spark
Updated Branches:
refs/heads/master 1d9733271 -> c03b6d115
[SPARK-11121][CORE] Correct the TaskLocation type
Correct the logic to return `HDFSCacheTaskLocation` instance when the input `str` is a in memory location.
Author: zhichao.li <zh...@intel.com>
Closes #9096 from zhichao-li/uselessBranch.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c03b6d11
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c03b6d11
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c03b6d11
Branch: refs/heads/master
Commit: c03b6d11589102b91f08728519e8520025db91e1
Parents: 1d97332
Author: zhichao.li <zh...@intel.com>
Authored: Thu Oct 22 03:59:26 2015 -0700
Committer: Sean Owen <so...@cloudera.com>
Committed: Thu Oct 22 03:59:26 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/scheduler/TaskLocation.scala | 2 +-
.../org/apache/spark/scheduler/TaskSetManagerSuite.scala | 11 ++++++++---
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c03b6d11/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala b/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
index da07ce2..1b65926 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/TaskLocation.scala
@@ -67,7 +67,7 @@ private[spark] object TaskLocation {
if (hstr.equals(str)) {
new HostTaskLocation(str)
} else {
- new HostTaskLocation(hstr)
+ new HDFSCacheTaskLocation(hstr)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c03b6d11/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
index f0eadf2..695523c 100644
--- a/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/scheduler/TaskSetManagerSuite.scala
@@ -759,9 +759,9 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg
val sched = new FakeTaskScheduler(sc,
("execA", "host1"), ("execB", "host2"), ("execC", "host3"))
val taskSet = FakeTask.createTaskSet(3,
- Seq(HostTaskLocation("host1")),
- Seq(HostTaskLocation("host2")),
- Seq(HDFSCacheTaskLocation("host3")))
+ Seq(TaskLocation("host1")),
+ Seq(TaskLocation("host2")),
+ Seq(TaskLocation("hdfs_cache_host3")))
val clock = new ManualClock
val manager = new TaskSetManager(sched, taskSet, MAX_TASK_FAILURES, clock)
assert(manager.myLocalityLevels.sameElements(Array(PROCESS_LOCAL, NODE_LOCAL, ANY)))
@@ -776,6 +776,11 @@ class TaskSetManagerSuite extends SparkFunSuite with LocalSparkContext with Logg
assert(manager.myLocalityLevels.sameElements(Array(ANY)))
}
+ test("Test TaskLocation for different host type.") {
+ assert(TaskLocation("host1") === HostTaskLocation("host1"))
+ assert(TaskLocation("hdfs_cache_host1") === HDFSCacheTaskLocation("host1"))
+ }
+
def createTaskResult(id: Int): DirectTaskResult[Int] = {
val valueSer = SparkEnv.get.serializer.newInstance()
new DirectTaskResult[Int](valueSer.serialize(id), mutable.Map.empty, new TaskMetrics)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org