You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2018/03/08 21:02:34 UTC
spark git commit: [SPARK-23602][SQL] PrintToStderr prints value also
in interpreted mode
Repository: spark
Updated Branches:
refs/heads/master ea480990e -> e7bbca889
[SPARK-23602][SQL] PrintToStderr prints value also in interpreted mode
## What changes were proposed in this pull request?
`PrintToStderr` was doing what is it supposed to only when code generation is enabled.
The PR adds the same behavior in interpreted mode too.
## How was this patch tested?
added UT
Author: Marco Gaido <ma...@gmail.com>
Closes #20773 from mgaido91/SPARK-23602.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e7bbca88
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e7bbca88
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e7bbca88
Branch: refs/heads/master
Commit: e7bbca88964d95593fa15eb94643ba519801e352
Parents: ea48099
Author: Marco Gaido <ma...@gmail.com>
Authored: Thu Mar 8 22:02:28 2018 +0100
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Thu Mar 8 22:02:28 2018 +0100
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/misc.scala | 7 +++++-
.../expressions/MiscExpressionsSuite.scala | 25 ++++++++++++++++++++
2 files changed, 31 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e7bbca88/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 4b9006a..38e4fe4 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -31,7 +31,12 @@ case class PrintToStderr(child: Expression) extends UnaryExpression {
override def dataType: DataType = child.dataType
- protected override def nullSafeEval(input: Any): Any = input
+ protected override def nullSafeEval(input: Any): Any = {
+ // scalastyle:off println
+ System.err.println(outputPrefix + input)
+ // scalastyle:on println
+ input
+ }
private val outputPrefix = s"Result of ${child.simpleString} is "
http://git-wip-us.apache.org/repos/asf/spark/blob/e7bbca88/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
index a21c139..c3d08bf 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/MiscExpressionsSuite.scala
@@ -17,6 +17,8 @@
package org.apache.spark.sql.catalyst.expressions
+import java.io.PrintStream
+
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.types._
@@ -43,4 +45,27 @@ class MiscExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(Length(Uuid()), 36)
assert(evaluateWithoutCodegen(Uuid()) !== evaluateWithoutCodegen(Uuid()))
}
+
+ test("PrintToStderr") {
+ val inputExpr = Literal(1)
+ val systemErr = System.err
+
+ val (outputEval, outputCodegen) = try {
+ val errorStream = new java.io.ByteArrayOutputStream()
+ System.setErr(new PrintStream(errorStream))
+ // check without codegen
+ checkEvaluationWithoutCodegen(PrintToStderr(inputExpr), 1)
+ val outputEval = errorStream.toString
+ errorStream.reset()
+ // check with codegen
+ checkEvaluationWithGeneratedMutableProjection(PrintToStderr(inputExpr), 1)
+ val outputCodegen = errorStream.toString
+ (outputEval, outputCodegen)
+ } finally {
+ System.setErr(systemErr)
+ }
+
+ assert(outputCodegen.contains(s"Result of $inputExpr is 1"))
+ assert(outputEval.contains(s"Result of $inputExpr is 1"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org