You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2015/07/21 20:18:51 UTC
spark git commit: Revert "[SPARK-9154] [SQL] codegen StringFormat"
Repository: spark
Updated Branches:
refs/heads/master 89db3c0b6 -> 87d890cc1
Revert "[SPARK-9154] [SQL] codegen StringFormat"
This reverts commit 7f072c3d5ec50c65d76bd9f28fac124fce96a89e.
Revert #7546
Author: Michael Armbrust <mi...@databricks.com>
Closes #7570 from marmbrus/revert9154 and squashes the following commits:
ed2c32a [Michael Armbrust] Revert "[SPARK-9154] [SQL] codegen StringFormat"
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/87d890cc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/87d890cc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/87d890cc
Branch: refs/heads/master
Commit: 87d890cc105a7f41478433b28f53c9aa431db211
Parents: 89db3c0
Author: Michael Armbrust <mi...@databricks.com>
Authored: Tue Jul 21 11:18:39 2015 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Jul 21 11:18:39 2015 -0700
----------------------------------------------------------------------
.../catalyst/expressions/stringOperations.scala | 42 +-------------------
.../expressions/StringExpressionsSuite.scala | 18 +++++----
.../apache/spark/sql/StringFunctionsSuite.scala | 10 -----
3 files changed, 11 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/87d890cc/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 280ae0e..fe57d17 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -526,7 +526,7 @@ case class StringRPad(str: Expression, len: Expression, pad: Expression)
/**
* Returns the input formatted according do printf-style format strings
*/
-case class StringFormat(children: Expression*) extends Expression with ImplicitCastInputTypes {
+case class StringFormat(children: Expression*) extends Expression with CodegenFallback {
require(children.nonEmpty, "printf() should take at least 1 argument")
@@ -536,10 +536,6 @@ case class StringFormat(children: Expression*) extends Expression with ImplicitC
private def format: Expression = children(0)
private def args: Seq[Expression] = children.tail
- override def inputTypes: Seq[AbstractDataType] =
- children.zipWithIndex.map(x => if (x._2 == 0) StringType else AnyDataType)
-
-
override def eval(input: InternalRow): Any = {
val pattern = format.eval(input)
if (pattern == null) {
@@ -555,42 +551,6 @@ case class StringFormat(children: Expression*) extends Expression with ImplicitC
}
}
- override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
- val pattern = children.head.gen(ctx)
-
- val argListGen = children.tail.map(x => (x.dataType, x.gen(ctx)))
- val argListCode = argListGen.map(_._2.code + "\n")
-
- val argListString = argListGen.foldLeft("")((s, v) => {
- val nullSafeString =
- if (ctx.boxedType(v._1) != ctx.javaType(v._1)) {
- // Java primitives get boxed in order to allow null values.
- s"(${v._2.isNull}) ? (${ctx.boxedType(v._1)}) null : " +
- s"new ${ctx.boxedType(v._1)}(${v._2.primitive})"
- } else {
- s"(${v._2.isNull}) ? null : ${v._2.primitive}"
- }
- s + "," + nullSafeString
- })
-
- val form = ctx.freshName("formatter")
- val formatter = classOf[java.util.Formatter].getName
- val sb = ctx.freshName("sb")
- val stringBuffer = classOf[StringBuffer].getName
- s"""
- ${pattern.code}
- boolean ${ev.isNull} = ${pattern.isNull};
- ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)};
- if (!${ev.isNull}) {
- ${argListCode.mkString}
- $stringBuffer $sb = new $stringBuffer();
- $formatter $form = new $formatter($sb, ${classOf[Locale].getName}.US);
- $form.format(${pattern.primitive}.toString() $argListString);
- ${ev.primitive} = UTF8String.fromString($sb.toString());
- }
- """
- }
-
override def prettyName: String = "printf"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/87d890cc/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
index 3c2d887..96c540a 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala
@@ -351,16 +351,18 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}
test("FORMAT") {
- checkEvaluation(StringFormat(Literal("aa%d%s"), Literal(123), Literal("a")), "aa123a")
+ val f = 'f.string.at(0)
+ val d1 = 'd.int.at(1)
+ val s1 = 's.int.at(2)
+
+ val row1 = create_row("aa%d%s", 12, "cc")
+ val row2 = create_row(null, 12, "cc")
+ checkEvaluation(StringFormat(Literal("aa%d%s"), Literal(123), Literal("a")), "aa123a", row1)
checkEvaluation(StringFormat(Literal("aa")), "aa", create_row(null))
- checkEvaluation(StringFormat(Literal("aa%d%s"), Literal(123), Literal("a")), "aa123a")
- checkEvaluation(StringFormat(Literal("aa%d%s"), 12, "cc"), "aa12cc")
+ checkEvaluation(StringFormat(Literal("aa%d%s"), Literal(123), Literal("a")), "aa123a", row1)
- checkEvaluation(StringFormat(Literal.create(null, StringType), 12, "cc"), null)
- checkEvaluation(
- StringFormat(Literal("aa%d%s"), Literal.create(null, IntegerType), "cc"), "aanullcc")
- checkEvaluation(
- StringFormat(Literal("aa%d%s"), 12, Literal.create(null, StringType)), "aa12null")
+ checkEvaluation(StringFormat(f, d1, s1), "aa12cc", row1)
+ checkEvaluation(StringFormat(f, d1, s1), null, row2)
}
test("INSTR") {
http://git-wip-us.apache.org/repos/asf/spark/blob/87d890cc/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
index 3702e73..d1f8559 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala
@@ -132,16 +132,6 @@ class StringFunctionsSuite extends QueryTest {
checkAnswer(
df.selectExpr("printf(a, b, c)"),
Row("aa123cc"))
-
- val df2 = Seq(("aa%d%s".getBytes, 123, "cc")).toDF("a", "b", "c")
-
- checkAnswer(
- df2.select(formatString($"a", $"b", $"c"), formatString("aa%d%s", "b", "c")),
- Row("aa123cc", "aa123cc"))
-
- checkAnswer(
- df2.selectExpr("printf(a, b, c)"),
- Row("aa123cc"))
}
test("string instr function") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org