You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2013/11/07 20:08:32 UTC
[1/4] git commit: include the appid in the cmd line arguments to
Executors
Updated Branches:
refs/heads/master be7e8da98 -> 3d4ad84b6
include the appid in the cmd line arguments to Executors
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/36e832bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/36e832bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/36e832bf
Branch: refs/heads/master
Commit: 36e832bff06ba053ce1cc69776e0d9fb0c61e7e1
Parents: aadeda5
Author: Imran Rashid <im...@quantifind.com>
Authored: Thu Nov 7 01:11:49 2013 -0600
Committer: Imran Rashid <im...@quantifind.com>
Committed: Thu Nov 7 01:11:49 2013 -0600
----------------------------------------------------------------------
.../main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala | 2 +-
.../org/apache/spark/executor/CoarseGrainedExecutorBackend.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/36e832bf/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
index 8fabc95..fff9cb6 100644
--- a/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/ExecutorRunner.scala
@@ -104,7 +104,7 @@ private[spark] class ExecutorRunner(
// SPARK-698: do not call the run.cmd script, as process.destroy()
// fails to kill a process tree on Windows
Seq(runner) ++ buildJavaOpts() ++ Seq(command.mainClass) ++
- command.arguments.map(substituteVariables)
+ (command.arguments ++ Seq(appId)).map(substituteVariables)
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/36e832bf/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
index 80ff4c5..caee6b0 100644
--- a/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
+++ b/core/src/main/scala/org/apache/spark/executor/CoarseGrainedExecutorBackend.scala
@@ -111,7 +111,7 @@ private[spark] object CoarseGrainedExecutorBackend {
def main(args: Array[String]) {
if (args.length < 4) {
- //the reason we allow the last frameworkId argument is to make it easy to kill rogue executors
+ //the reason we allow the last appid argument is to make it easy to kill rogue executors
System.err.println(
"Usage: CoarseGrainedExecutorBackend <driverUrl> <executorId> <hostname> <cores> " +
"[<appid>]")
[2/4] git commit: very basic regression test to make sure appId
doesnt get dropped in future
Posted by rx...@apache.org.
very basic regression test to make sure appId doesnt get dropped in future
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/8d3cdda9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/8d3cdda9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/8d3cdda9
Branch: refs/heads/master
Commit: 8d3cdda9a22a23794c0cd12b15279a80a31952e3
Parents: 36e832b
Author: Imran Rashid <im...@quantifind.com>
Authored: Thu Nov 7 01:35:48 2013 -0600
Committer: Imran Rashid <im...@quantifind.com>
Committed: Thu Nov 7 01:35:48 2013 -0600
----------------------------------------------------------------------
.../spark/deploy/worker/ExecutorRunnerTest.scala | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/8d3cdda9/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
new file mode 100644
index 0000000..a3111e2
--- /dev/null
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -0,0 +1,18 @@
+package org.apache.spark.deploy.worker
+
+import org.scalatest.FunSuite
+import org.apache.spark.deploy.{ExecutorState, Command, ApplicationDescription}
+import java.io.File
+
+class ExecutorRunnerTest extends FunSuite {
+
+ test("command includes appId") {
+ def f(s:String) = new File(s)
+ val sparkHome = sys.props("user.dir")
+ val appDesc = new ApplicationDescription("app name", 8, 500, Command("foo", Seq(),Map()), sparkHome, "appUiUrl")
+ val appId = "12345-worker321-9876"
+ val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", f(sparkHome), f("ooga"), ExecutorState.RUNNING)
+
+ assert(er.buildCommandSeq().last === appId)
+ }
+}
[4/4] git commit: Merge pull request #148 from squito/include_appId
Posted by rx...@apache.org.
Merge pull request #148 from squito/include_appId
Include appId in executor cmd line args
add the appId back into the executor cmd line args.
I also made a pretty lame regression test, just to make sure it doesn't get dropped in the future. not sure it will run on the build server, though, b/c `ExecutorRunner.buildCommandSeq()` expects to be abel to run the scripts in `bin`.
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/3d4ad84b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/3d4ad84b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/3d4ad84b
Branch: refs/heads/master
Commit: 3d4ad84b63e440fd3f4b3edb1b120ff7c14a42d1
Parents: be7e8da ca66f5d
Author: Reynold Xin <rx...@apache.org>
Authored: Thu Nov 7 11:08:27 2013 -0800
Committer: Reynold Xin <rx...@apache.org>
Committed: Thu Nov 7 11:08:27 2013 -0800
----------------------------------------------------------------------
.../spark/deploy/worker/ExecutorRunner.scala | 2 +-
.../executor/CoarseGrainedExecutorBackend.scala | 2 +-
.../deploy/worker/ExecutorRunnerTest.scala | 20 ++++++++++++++++++++
3 files changed, 22 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[3/4] git commit: fix formatting
Posted by rx...@apache.org.
fix formatting
Project: http://git-wip-us.apache.org/repos/asf/incubator-spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-spark/commit/ca66f5d5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-spark/tree/ca66f5d5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-spark/diff/ca66f5d5
Branch: refs/heads/master
Commit: ca66f5d5a287873d930a3bdc36d40d8e776ca25a
Parents: 8d3cdda
Author: Imran Rashid <im...@quantifind.com>
Authored: Thu Nov 7 07:23:59 2013 -0600
Committer: Imran Rashid <im...@quantifind.com>
Committed: Thu Nov 7 07:23:59 2013 -0600
----------------------------------------------------------------------
.../org/apache/spark/deploy/worker/ExecutorRunnerTest.scala | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-spark/blob/ca66f5d5/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
index a3111e2..d433806 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -1,17 +1,19 @@
package org.apache.spark.deploy.worker
+import java.io.File
import org.scalatest.FunSuite
import org.apache.spark.deploy.{ExecutorState, Command, ApplicationDescription}
-import java.io.File
class ExecutorRunnerTest extends FunSuite {
test("command includes appId") {
def f(s:String) = new File(s)
val sparkHome = sys.props("user.dir")
- val appDesc = new ApplicationDescription("app name", 8, 500, Command("foo", Seq(),Map()), sparkHome, "appUiUrl")
+ val appDesc = new ApplicationDescription("app name", 8, 500, Command("foo", Seq(),Map()),
+ sparkHome, "appUiUrl")
val appId = "12345-worker321-9876"
- val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", f(sparkHome), f("ooga"), ExecutorState.RUNNING)
+ val er = new ExecutorRunner(appId, 1, appDesc, 8, 500, null, "blah", "worker321", f(sparkHome),
+ f("ooga"), ExecutorState.RUNNING)
assert(er.buildCommandSeq().last === appId)
}