You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/01/08 03:39:52 UTC
spark git commit: [SPARK-22985] Fix argument escaping bug in
from_utc_timestamp / to_utc_timestamp codegen
Repository: spark
Updated Branches:
refs/heads/master 18e941499 -> 71d65a321
[SPARK-22985] Fix argument escaping bug in from_utc_timestamp / to_utc_timestamp codegen
## What changes were proposed in this pull request?
This patch adds additional escaping in `from_utc_timestamp` / `to_utc_timestamp` expression codegen in order to a bug where invalid timezones which contain special characters could cause generated code to fail to compile.
## How was this patch tested?
New regression tests in `DateExpressionsSuite`.
Author: Josh Rosen <jo...@databricks.com>
Closes #20182 from JoshRosen/SPARK-22985-fix-utc-timezone-function-escaping-bugs.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/71d65a32
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/71d65a32
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/71d65a32
Branch: refs/heads/master
Commit: 71d65a32158a55285be197bec4e41fedc9225b94
Parents: 18e9414
Author: Josh Rosen <jo...@databricks.com>
Authored: Mon Jan 8 11:39:45 2018 +0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Mon Jan 8 11:39:45 2018 +0800
----------------------------------------------------------------------
.../sql/catalyst/expressions/datetimeExpressions.scala | 12 ++++++++----
.../sql/catalyst/expressions/DateExpressionsSuite.scala | 6 ++++++
2 files changed, 14 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/71d65a32/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
index 7a674ea..424871f 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
@@ -23,6 +23,8 @@ import java.util.{Calendar, TimeZone}
import scala.util.control.NonFatal
+import org.apache.commons.lang3.StringEscapeUtils
+
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, CodegenFallback, ExprCode}
import org.apache.spark.sql.catalyst.util.DateTimeUtils
@@ -1008,7 +1010,7 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
if (right.foldable) {
- val tz = right.eval()
+ val tz = right.eval().asInstanceOf[UTF8String]
if (tz == null) {
ev.copy(code = s"""
|boolean ${ev.isNull} = true;
@@ -1017,8 +1019,9 @@ case class FromUTCTimestamp(left: Expression, right: Expression)
} else {
val tzClass = classOf[TimeZone].getName
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+ val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
val tzTerm = ctx.addMutableState(tzClass, "tz",
- v => s"""$v = $dtu.getTimeZone("$tz");""")
+ v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
val utcTerm = "tzUTC"
ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
v => s"""$v = $dtu.getTimeZone("UTC");""")
@@ -1185,7 +1188,7 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
if (right.foldable) {
- val tz = right.eval()
+ val tz = right.eval().asInstanceOf[UTF8String]
if (tz == null) {
ev.copy(code = s"""
|boolean ${ev.isNull} = true;
@@ -1194,8 +1197,9 @@ case class ToUTCTimestamp(left: Expression, right: Expression)
} else {
val tzClass = classOf[TimeZone].getName
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
+ val escapedTz = StringEscapeUtils.escapeJava(tz.toString)
val tzTerm = ctx.addMutableState(tzClass, "tz",
- v => s"""$v = $dtu.getTimeZone("$tz");""")
+ v => s"""$v = $dtu.getTimeZone("$escapedTz");""")
val utcTerm = "tzUTC"
ctx.addImmutableStateIfNotExists(tzClass, utcTerm,
v => s"""$v = $dtu.getTimeZone("UTC");""")
http://git-wip-us.apache.org/repos/asf/spark/blob/71d65a32/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
index 63f6cee..786266a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
@@ -22,6 +22,7 @@ import java.text.SimpleDateFormat
import java.util.{Calendar, Locale, TimeZone}
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.catalyst.expressions.codegen.GenerateUnsafeProjection
import org.apache.spark.sql.catalyst.util.DateTimeTestUtils._
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import org.apache.spark.sql.catalyst.util.DateTimeUtils.TimeZoneGMT
@@ -791,6 +792,9 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test(null, "UTC", null)
test("2015-07-24 00:00:00", null, null)
test(null, null, null)
+ // Test escaping of timezone
+ GenerateUnsafeProjection.generate(
+ ToUTCTimestamp(Literal(Timestamp.valueOf("2015-07-24 00:00:00")), Literal("\"quote")) :: Nil)
}
test("from_utc_timestamp") {
@@ -811,5 +815,7 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
test(null, "UTC", null)
test("2015-07-24 00:00:00", null, null)
test(null, null, null)
+ // Test escaping of timezone
+ GenerateUnsafeProjection.generate(FromUTCTimestamp(Literal(0), Literal("\"quote")) :: Nil)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org