You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@livy.apache.org by js...@apache.org on 2017/06/27 06:38:57 UTC
[16/50] [abbrv] incubator-livy git commit: LIVY-315. Fixed
SparkRInterpreter to respect config enableHiveContext. (#293)
LIVY-315. Fixed SparkRInterpreter to respect config enableHiveContext. (#293)
Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/ed1c86e8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/ed1c86e8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/ed1c86e8
Branch: refs/heads/master
Commit: ed1c86e874bee2d4323d6267cdf2a8d14b7db571
Parents: f7c434c
Author: Jeff Zhang <zj...@gmail.com>
Authored: Sat Feb 18 07:50:05 2017 +0800
Committer: Alex Man <tc...@gmail.com>
Committed: Fri Feb 17 15:50:05 2017 -0800
----------------------------------------------------------------------
.../cloudera/livy/repl/ProcessInterpreter.scala | 1 +
.../cloudera/livy/repl/SparkRInterpreter.scala | 19 ++++++++++++++-----
2 files changed, 15 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ed1c86e8/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
index 78826e3..25a8654 100644
--- a/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
+++ b/repl/src/main/scala/com/cloudera/livy/repl/ProcessInterpreter.scala
@@ -130,6 +130,7 @@ abstract class ProcessInterpreter(process: Process)
val exitCode = process.waitFor()
if (exitCode != 0) {
error(f"Process has died with $exitCode")
+ error(stderrLines.mkString("\n"))
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/ed1c86e8/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
----------------------------------------------------------------------
diff --git a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
index b3d3b7e..6c42fe3 100644
--- a/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
+++ b/repl/src/main/scala/com/cloudera/livy/repl/SparkRInterpreter.scala
@@ -116,7 +116,8 @@ object SparkRInterpreter {
builder.redirectErrorStream(true)
val process = builder.start()
new SparkRInterpreter(process, backendInstance, backendThread,
- conf.get("spark.livy.spark_major_version", "1"))
+ conf.get("spark.livy.spark_major_version", "1"),
+ conf.getBoolean("spark.repl.enableHiveContext", false))
} catch {
case e: Exception =>
if (backendThread != null) {
@@ -130,7 +131,8 @@ object SparkRInterpreter {
class SparkRInterpreter(process: Process,
backendInstance: Any,
backendThread: Thread,
- val sparkMajorVersion: String)
+ val sparkMajorVersion: String,
+ hiveEnabled: Boolean)
extends ProcessInterpreter(process) {
import SparkRInterpreter._
@@ -145,15 +147,22 @@ class SparkRInterpreter(process: Process,
sendRequest("options(error = dump.frames)")
if (!ClientConf.TEST_MODE) {
sendRequest("library(SparkR)")
-
if (sparkMajorVersion >= "2") {
- sendRequest("spark <- SparkR::sparkR.session()")
+ if (hiveEnabled) {
+ sendRequest("spark <- SparkR::sparkR.session()")
+ } else {
+ sendRequest("spark <- SparkR::sparkR.session(enableHiveSupport=FALSE)")
+ }
sendRequest(
"""sc <- SparkR:::callJStatic("org.apache.spark.sql.api.r.SQLUtils",
"getJavaSparkContext", spark)""")
} else {
sendRequest("sc <- sparkR.init()")
- sendRequest("sqlContext <- sparkRSQL.init(sc)")
+ if (hiveEnabled) {
+ sendRequest("sqlContext <- sparkRHive.init(sc)")
+ } else {
+ sendRequest("sqlContext <- sparkRSQL.init(sc)")
+ }
}
}