You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/06/05 07:56:56 UTC
git commit: Fix issue in ReplSuite with hadoop-provided profile.
Repository: spark
Updated Branches:
refs/heads/master abea2d4ff -> b77c19be0
Fix issue in ReplSuite with hadoop-provided profile.
When building the assembly with the maven "hadoop-provided"
profile, the executors were failing to come up because Hadoop classes
were not found in the classpath anymore; so add them explicitly to
the classpath using spark.executor.extraClassPath. This is only
needed for the local-cluster mode, but doesn't affect other tests,
so it's added for all of them to keep the code simpler.
Author: Marcelo Vanzin <va...@cloudera.com>
Closes #781 from vanzin/repl-test-fix and squashes the following commits:
4f0a3b0 [Marcelo Vanzin] Fix issue in ReplSuite with hadoop-provided profile.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b77c19be
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b77c19be
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b77c19be
Branch: refs/heads/master
Commit: b77c19be053125fde99b098ec1e1162f25b5433c
Parents: abea2d4
Author: Marcelo Vanzin <va...@cloudera.com>
Authored: Wed Jun 4 22:56:49 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Jun 4 22:56:49 2014 -0700
----------------------------------------------------------------------
.../test/scala/org/apache/spark/repl/ReplSuite.scala | 14 +++++++++++++-
1 file changed, 13 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b77c19be/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
----------------------------------------------------------------------
diff --git a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 98cdfd0..7c765ed 100644
--- a/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -32,6 +32,8 @@ import org.apache.spark.util.Utils
class ReplSuite extends FunSuite {
def runInterpreter(master: String, input: String): String = {
+ val CONF_EXECUTOR_CLASSPATH = "spark.executor.extraClassPath"
+
val in = new BufferedReader(new StringReader(input + "\n"))
val out = new StringWriter()
val cl = getClass.getClassLoader
@@ -44,13 +46,23 @@ class ReplSuite extends FunSuite {
}
}
}
+ val classpath = paths.mkString(File.pathSeparator)
+
+ val oldExecutorClasspath = System.getProperty(CONF_EXECUTOR_CLASSPATH)
+ System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
+
val interp = new SparkILoop(in, new PrintWriter(out), master)
org.apache.spark.repl.Main.interp = interp
- interp.process(Array("-classpath", paths.mkString(File.pathSeparator)))
+ interp.process(Array("-classpath", classpath))
org.apache.spark.repl.Main.interp = null
if (interp.sparkContext != null) {
interp.sparkContext.stop()
}
+ if (oldExecutorClasspath != null) {
+ System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
+ } else {
+ System.clearProperty(CONF_EXECUTOR_CLASSPATH)
+ }
return out.toString
}