You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/04/14 03:26:25 UTC
spark git commit: [SPARK-5794] [SQL] fix add jar
Repository: spark
Updated Branches:
refs/heads/master 3782e1f2b -> b45059d0d
[SPARK-5794] [SQL] fix add jar
Author: Daoyuan Wang <da...@intel.com>
Closes #4586 from adrian-wang/addjar and squashes the following commits:
efdd602 [Daoyuan Wang] move jar to another place
6c707e8 [Daoyuan Wang] restrict hive version for test
32c4fb8 [Daoyuan Wang] fix style and add a test
9957d87 [Daoyuan Wang] use sessionstate classloader in makeRDDforTable
0810e71 [Daoyuan Wang] remove variable substitution
1898309 [Daoyuan Wang] fix classnotfound
95a40da [Daoyuan Wang] support env argus in add jar, and set add jar ret to 0
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b45059d0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b45059d0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b45059d0
Branch: refs/heads/master
Commit: b45059d0d7809a986ba07a447deb71f11ec6afe4
Parents: 3782e1f
Author: Daoyuan Wang <da...@intel.com>
Authored: Mon Apr 13 18:26:00 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Apr 13 18:26:00 2015 -0700
----------------------------------------------------------------------
.../sql/hive/thriftserver/SparkSQLCLIDriver.scala | 5 +++--
.../org/apache/spark/sql/hive/TableReader.scala | 5 ++++-
.../spark/sql/hive/execution/commands.scala | 2 +-
.../test/resources/hive-hcatalog-core-0.13.1.jar | Bin 0 -> 468533 bytes
.../spark/sql/hive/execution/HiveQuerySuite.scala | 15 +++++++++++++++
5 files changed, 23 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b45059d0/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
----------------------------------------------------------------------
diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
index 6272cde..62c061b 100644
--- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
+++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala
@@ -33,7 +33,7 @@ import org.apache.hadoop.hive.common.{HiveInterruptCallback, HiveInterruptUtils,
import org.apache.hadoop.hive.conf.HiveConf
import org.apache.hadoop.hive.ql.Driver
import org.apache.hadoop.hive.ql.exec.Utilities
-import org.apache.hadoop.hive.ql.processors.{SetProcessor, CommandProcessor, CommandProcessorFactory}
+import org.apache.hadoop.hive.ql.processors.{AddResourceProcessor, SetProcessor, CommandProcessor, CommandProcessorFactory}
import org.apache.hadoop.hive.ql.session.SessionState
import org.apache.hadoop.hive.shims.ShimLoader
import org.apache.thrift.transport.TSocket
@@ -264,7 +264,8 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging {
val proc: CommandProcessor = HiveShim.getCommandProcessor(Array(tokens(0)), hconf)
if (proc != null) {
- if (proc.isInstanceOf[Driver] || proc.isInstanceOf[SetProcessor]) {
+ if (proc.isInstanceOf[Driver] || proc.isInstanceOf[SetProcessor] ||
+ proc.isInstanceOf[AddResourceProcessor]) {
val driver = new SparkSQLDriver
driver.init()
http://git-wip-us.apache.org/repos/asf/spark/blob/b45059d0/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
index d352915..e556c74 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/TableReader.scala
@@ -35,6 +35,7 @@ import org.apache.spark.broadcast.Broadcast
import org.apache.spark.rdd.{EmptyRDD, HadoopRDD, RDD, UnionRDD}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.types.DateUtils
+import org.apache.spark.util.Utils
/**
* A trait for subclasses that handle table scans.
@@ -76,7 +77,9 @@ class HadoopTableReader(
override def makeRDDForTable(hiveTable: HiveTable): RDD[Row] =
makeRDDForTable(
hiveTable,
- relation.tableDesc.getDeserializerClass.asInstanceOf[Class[Deserializer]],
+ Class.forName(
+ relation.tableDesc.getSerdeClassName, true, sc.sessionState.getConf.getClassLoader)
+ .asInstanceOf[Class[Deserializer]],
filterOpt = None)
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/b45059d0/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
index 99dc586..902a127 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala
@@ -80,7 +80,7 @@ case class AddJar(path: String) extends RunnableCommand {
val hiveContext = sqlContext.asInstanceOf[HiveContext]
hiveContext.runSqlHive(s"ADD JAR $path")
hiveContext.sparkContext.addJar(path)
- Seq.empty[Row]
+ Seq(Row(0))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/b45059d0/sql/hive/src/test/resources/hive-hcatalog-core-0.13.1.jar
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/resources/hive-hcatalog-core-0.13.1.jar b/sql/hive/src/test/resources/hive-hcatalog-core-0.13.1.jar
new file mode 100644
index 0000000..37af9aa
Binary files /dev/null and b/sql/hive/src/test/resources/hive-hcatalog-core-0.13.1.jar differ
http://git-wip-us.apache.org/repos/asf/spark/blob/b45059d0/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 1222fba..300b1f7 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -813,6 +813,21 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter {
sql("DROP TABLE alter1")
}
+ test("ADD JAR command 2") {
+ // this is a test case from mapjoin_addjar.q
+ val testJar = TestHive.getHiveFile("hive-hcatalog-core-0.13.1.jar").getCanonicalPath
+ val testData = TestHive.getHiveFile("data/files/sample.json").getCanonicalPath
+ if (HiveShim.version == "0.13.1") {
+ sql(s"ADD JAR $testJar")
+ sql(
+ """CREATE TABLE t1(a string, b string)
+ |ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'""".stripMargin)
+ sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t1""")
+ sql("select * from src join t1 on src.key = t1.a")
+ sql("DROP TABLE t1")
+ }
+ }
+
test("ADD FILE command") {
val testFile = TestHive.getHiveFile("data/files/v1.txt").getCanonicalFile
sql(s"ADD FILE $testFile")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org