You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/01/10 23:25:51 UTC
spark git commit: [SPARK-5187][SQL] Fix caching of tables with
HiveUDFs in the WHERE clause
Repository: spark
Updated Branches:
refs/heads/master 77106df69 -> 3684fd21e
[SPARK-5187][SQL] Fix caching of tables with HiveUDFs in the WHERE clause
Author: Michael Armbrust <mi...@databricks.com>
Closes #3987 from marmbrus/hiveUdfCaching and squashes the following commits:
8bca2fa [Michael Armbrust] [SPARK-5187][SQL] Fix caching of tables with HiveUDFs in the WHERE clause
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3684fd21
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3684fd21
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3684fd21
Branch: refs/heads/master
Commit: 3684fd21e1ffdc0adaad8ff6b31394b637e866ce
Parents: 77106df
Author: Michael Armbrust <mi...@databricks.com>
Authored: Sat Jan 10 14:25:45 2015 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Sat Jan 10 14:25:45 2015 -0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/hive/CachedTableSuite.scala | 6 ++++++
.../src/main/scala/org/apache/spark/sql/hive/Shim12.scala | 2 +-
.../src/main/scala/org/apache/spark/sql/hive/Shim13.scala | 2 +-
3 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3684fd21/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
index 2060e1f..f95a6b4 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/CachedTableSuite.scala
@@ -158,4 +158,10 @@ class CachedTableSuite extends QueryTest {
uncacheTable("src")
assert(!isMaterialized(rddId), "Uncached in-memory table should have been unpersisted")
}
+
+ test("CACHE TABLE with Hive UDF") {
+ sql("CACHE TABLE udfTest AS SELECT * FROM src WHERE floor(key) = 1")
+ assertCached(table("udfTest"))
+ uncacheTable("udfTest")
+ }
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3684fd21/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
----------------------------------------------------------------------
diff --git a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
index 2d01a85..25fdf5c 100644
--- a/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
+++ b/sql/hive/v0.12.0/src/main/scala/org/apache/spark/sql/hive/Shim12.scala
@@ -44,7 +44,7 @@ import scala.language.implicitConversions
import org.apache.spark.sql.catalyst.types.DecimalType
-class HiveFunctionWrapper(var functionClassName: String) extends java.io.Serializable {
+case class HiveFunctionWrapper(functionClassName: String) extends java.io.Serializable {
// for Serialization
def this() = this(null)
http://git-wip-us.apache.org/repos/asf/spark/blob/3684fd21/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
----------------------------------------------------------------------
diff --git a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
index b78c757..e47002c 100644
--- a/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
+++ b/sql/hive/v0.13.1/src/main/scala/org/apache/spark/sql/hive/Shim13.scala
@@ -53,7 +53,7 @@ import scala.language.implicitConversions
*
* @param functionClassName UDF class name
*/
-class HiveFunctionWrapper(var functionClassName: String) extends java.io.Externalizable {
+case class HiveFunctionWrapper(var functionClassName: String) extends java.io.Externalizable {
// for Serialization
def this() = this(null)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org