You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/07/05 01:36:13 UTC
spark git commit: [SPARK-22384][SQL][FOLLOWUP] Refine partition
pruning when attribute is wrapped in Cast
Repository: spark
Updated Branches:
refs/heads/master ca8243f30 -> bf764a33b
[SPARK-22384][SQL][FOLLOWUP] Refine partition pruning when attribute is wrapped in Cast
## What changes were proposed in this pull request?
As mentioned in https://github.com/apache/spark/pull/21586 , `Cast.mayTruncate` is not 100% safe, string to boolean is allowed. Since changing `Cast.mayTruncate` also changes the behavior of Dataset, here I propose to add a new `Cast.canSafeCast` for partition pruning.
## How was this patch tested?
new test cases
Author: Wenchen Fan <we...@databricks.com>
Closes #21712 from cloud-fan/safeCast.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/bf764a33
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/bf764a33
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/bf764a33
Branch: refs/heads/master
Commit: bf764a33bef617aa9bae535a5ea73d6a3e278d42
Parents: ca8243f
Author: Wenchen Fan <we...@databricks.com>
Authored: Wed Jul 4 18:36:09 2018 -0700
Committer: Xiao Li <ga...@gmail.com>
Committed: Wed Jul 4 18:36:09 2018 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/Cast.scala | 20 ++++++++++++++++++++
.../apache/spark/sql/hive/client/HiveShim.scala | 5 +++--
.../spark/sql/hive/client/HiveClientSuite.scala | 20 ++++++++++++++++++--
3 files changed, 41 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index 699ea53..7971ae6 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -134,6 +134,26 @@ object Cast {
toPrecedence > 0 && fromPrecedence > toPrecedence
}
+ /**
+ * Returns true iff we can safely cast the `from` type to `to` type without any truncating or
+ * precision lose, e.g. int -> long, date -> timestamp.
+ */
+ def canSafeCast(from: AtomicType, to: AtomicType): Boolean = (from, to) match {
+ case _ if from == to => true
+ case (from: NumericType, to: DecimalType) if to.isWiderThan(from) => true
+ case (from: DecimalType, to: NumericType) if from.isTighterThan(to) => true
+ case (from, to) if legalNumericPrecedence(from, to) => true
+ case (DateType, TimestampType) => true
+ case (_, StringType) => true
+ case _ => false
+ }
+
+ private def legalNumericPrecedence(from: DataType, to: DataType): Boolean = {
+ val fromPrecedence = TypeCoercion.numericPrecedence.indexOf(from)
+ val toPrecedence = TypeCoercion.numericPrecedence.indexOf(to)
+ fromPrecedence >= 0 && fromPrecedence < toPrecedence
+ }
+
def forceNullable(from: DataType, to: DataType): Boolean = (from, to) match {
case (NullType, _) => true
case (_, _) if from == to => false
http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
index 8620f3f..933384e 100644
--- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
+++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveShim.scala
@@ -45,7 +45,7 @@ import org.apache.spark.sql.catalyst.analysis.NoSuchPermanentFunctionException
import org.apache.spark.sql.catalyst.catalog.{CatalogFunction, CatalogTablePartition, CatalogUtils, FunctionResource, FunctionResourceType}
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.internal.SQLConf
-import org.apache.spark.sql.types.{IntegralType, StringType}
+import org.apache.spark.sql.types.{AtomicType, IntegralType, StringType}
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
@@ -660,7 +660,8 @@ private[client] class Shim_v0_13 extends Shim_v0_12 {
def unapply(expr: Expression): Option[Attribute] = {
expr match {
case attr: Attribute => Some(attr)
- case Cast(child, dt, _) if !Cast.mayTruncate(child.dataType, dt) => unapply(child)
+ case Cast(child @ AtomicType(), dt: AtomicType, _)
+ if Cast.canSafeCast(child.dataType.asInstanceOf[AtomicType], dt) => unapply(child)
case _ => None
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/bf764a33/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
index 55275f6..fa9f753 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
@@ -24,7 +24,7 @@ import org.scalatest.BeforeAndAfterAll
import org.apache.spark.sql.catalyst.catalog._
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions._
-import org.apache.spark.sql.types.LongType
+import org.apache.spark.sql.types.{BooleanType, IntegerType, LongType}
// TODO: Refactor this to `HivePartitionFilteringSuite`
class HiveClientSuite(version: String)
@@ -122,6 +122,22 @@ class HiveClientSuite(version: String)
"aa" :: Nil)
}
+ test("getPartitionsByFilter: cast(chunk as int)=1 (not a valid partition predicate)") {
+ testMetastorePartitionFiltering(
+ attr("chunk").cast(IntegerType) === 1,
+ 20170101 to 20170103,
+ 0 to 23,
+ "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ }
+
+ test("getPartitionsByFilter: cast(chunk as boolean)=true (not a valid partition predicate)") {
+ testMetastorePartitionFiltering(
+ attr("chunk").cast(BooleanType) === true,
+ 20170101 to 20170103,
+ 0 to 23,
+ "aa" :: "ab" :: "ba" :: "bb" :: Nil)
+ }
+
test("getPartitionsByFilter: 20170101=ds") {
testMetastorePartitionFiltering(
Literal(20170101) === attr("ds"),
@@ -138,7 +154,7 @@ class HiveClientSuite(version: String)
"aa" :: "ab" :: "ba" :: "bb" :: Nil)
}
- test("getPartitionsByFilter: chunk in cast(ds as long)=20170101L") {
+ test("getPartitionsByFilter: cast(ds as long)=20170101L and h=10") {
testMetastorePartitionFiltering(
attr("ds").cast(LongType) === 20170101L && attr("h") === 10,
20170101 to 20170101,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org