You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2016/12/27 14:22:28 UTC
spark git commit: [SPARK-18999][SQL][MINOR] simplify Literal codegen
Repository: spark
Updated Branches:
refs/heads/master dd724c84c -> 6ddbf467b
[SPARK-18999][SQL][MINOR] simplify Literal codegen
## What changes were proposed in this pull request?
`Literal` can use `CodegenContex.addReferenceObj` to implement codegen, instead of `CodegenFallback`. This can also simplify the generated code a little bit, before we will generate: `((Expression) references[1]).eval(null)`, now it's just `references[1]`.
## How was this patch tested?
N/A
Author: Wenchen Fan <we...@databricks.com>
Closes #16402 from cloud-fan/minor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6ddbf467
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6ddbf467
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6ddbf467
Branch: refs/heads/master
Commit: 6ddbf467b41126c894e2a725f2460ba0a1e9292b
Parents: dd724c8
Author: Wenchen Fan <we...@databricks.com>
Authored: Tue Dec 27 06:22:12 2016 -0800
Committer: gatorsmile <ga...@gmail.com>
Committed: Tue Dec 27 06:22:12 2016 -0800
----------------------------------------------------------------------
.../expressions/codegen/CodeGenerator.scala | 4 +-
.../sql/catalyst/expressions/literals.scala | 41 ++++++--------------
.../spark/sql/catalyst/expressions/misc.scala | 2 +-
.../catalyst/expressions/objects/objects.scala | 6 +--
4 files changed, 18 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6ddbf467/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index 09007b7..d7746ca 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -92,10 +92,10 @@ class CodegenContext {
* This is for minor objects not to store the object into field but refer it from the references
* field at the time of use because number of fields in class is limited so we should reduce it.
*/
- def addReferenceObj(obj: Any): String = {
+ def addReferenceMinorObj(obj: Any, className: String = null): String = {
val idx = references.length
references += obj
- val clsName = obj.getClass.getName
+ val clsName = Option(className).getOrElse(obj.getClass.getName)
s"(($clsName) references[$idx])"
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ddbf467/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
index 1985e68..ab45c41 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala
@@ -220,7 +220,7 @@ object DecimalLiteral {
/**
* In order to do type checking, use Literal.create() instead of constructor
*/
-case class Literal (value: Any, dataType: DataType) extends LeafExpression with CodegenFallback {
+case class Literal (value: Any, dataType: DataType) extends LeafExpression {
override def foldable: Boolean = true
override def nullable: Boolean = value == null
@@ -271,45 +271,28 @@ case class Literal (value: Any, dataType: DataType) extends LeafExpression with
ev.isNull = "true"
ev.copy(s"final ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)};")
} else {
- dataType match {
- case BooleanType =>
- ev.isNull = "false"
- ev.value = value.toString
- ev.copy("")
+ ev.isNull = "false"
+ ev.value = dataType match {
+ case BooleanType | IntegerType | DateType => value.toString
case FloatType =>
val v = value.asInstanceOf[Float]
if (v.isNaN || v.isInfinite) {
- super[CodegenFallback].doGenCode(ctx, ev)
+ ctx.addReferenceMinorObj(v)
} else {
- ev.isNull = "false"
- ev.value = s"${value}f"
- ev.copy("")
+ s"${value}f"
}
case DoubleType =>
val v = value.asInstanceOf[Double]
if (v.isNaN || v.isInfinite) {
- super[CodegenFallback].doGenCode(ctx, ev)
+ ctx.addReferenceMinorObj(v)
} else {
- ev.isNull = "false"
- ev.value = s"${value}D"
- ev.copy("")
+ s"${value}D"
}
- case ByteType | ShortType =>
- ev.isNull = "false"
- ev.value = s"(${ctx.javaType(dataType)})$value"
- ev.copy("")
- case IntegerType | DateType =>
- ev.isNull = "false"
- ev.value = value.toString
- ev.copy("")
- case TimestampType | LongType =>
- ev.isNull = "false"
- ev.value = s"${value}L"
- ev.copy("")
- // eval() version may be faster for non-primitive types
- case other =>
- super[CodegenFallback].doGenCode(ctx, ev)
+ case ByteType | ShortType => s"(${ctx.javaType(dataType)})$value"
+ case TimestampType | LongType => s"${value}L"
+ case other => ctx.addReferenceMinorObj(value, ctx.javaType(dataType))
}
+ ev.copy("")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6ddbf467/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index a874a1c..bb9368c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -78,7 +78,7 @@ case class AssertTrue(child: Expression) extends UnaryExpression with ImplicitCa
// Use unnamed reference that doesn't create a local field here to reduce the number of fields
// because errMsgField is used only when the value is null or false.
- val errMsgField = ctx.addReferenceObj(errMsg)
+ val errMsgField = ctx.addReferenceMinorObj(errMsg)
ExprCode(code = s"""${eval.code}
|if (${eval.isNull} || !${eval.value}) {
| throw new RuntimeException($errMsgField);
http://git-wip-us.apache.org/repos/asf/spark/blob/6ddbf467/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index fc32369..36bf301 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -961,7 +961,7 @@ case class AssertNotNull(child: Expression, walkedTypePath: Seq[String])
// Use unnamed reference that doesn't create a local field here to reduce the number of fields
// because errMsgField is used only when the value is null.
- val errMsgField = ctx.addReferenceObj(errMsg)
+ val errMsgField = ctx.addReferenceMinorObj(errMsg)
val code = s"""
${childGen.code}
@@ -998,7 +998,7 @@ case class GetExternalRowField(
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
// Use unnamed reference that doesn't create a local field here to reduce the number of fields
// because errMsgField is used only when the field is null.
- val errMsgField = ctx.addReferenceObj(errMsg)
+ val errMsgField = ctx.addReferenceMinorObj(errMsg)
val row = child.genCode(ctx)
val code = s"""
${row.code}
@@ -1038,7 +1038,7 @@ case class ValidateExternalType(child: Expression, expected: DataType)
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
// Use unnamed reference that doesn't create a local field here to reduce the number of fields
// because errMsgField is used only when the type doesn't match.
- val errMsgField = ctx.addReferenceObj(errMsg)
+ val errMsgField = ctx.addReferenceMinorObj(errMsg)
val input = child.genCode(ctx)
val obj = input.value
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org