You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/02/11 08:52:21 UTC
spark git commit: [SPARK-13270][SQL] Remove extra new lines in whole
stage codegen and include pipeline plan in comments.
Repository: spark
Updated Branches:
refs/heads/master e88bff127 -> 18bcbbdd8
[SPARK-13270][SQL] Remove extra new lines in whole stage codegen and include pipeline plan in comments.
Author: Nong Li <no...@databricks.com>
Closes #11155 from nongli/spark-13270.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/18bcbbdd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/18bcbbdd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/18bcbbdd
Branch: refs/heads/master
Commit: 18bcbbdd84e80222d1d29530831c6d68d02e7593
Parents: e88bff1
Author: Nong Li <no...@databricks.com>
Authored: Wed Feb 10 23:52:19 2016 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Feb 10 23:52:19 2016 -0800
----------------------------------------------------------------------
.../catalyst/expressions/codegen/CodeFormatter.scala | 14 ++++++++++++++
.../spark/sql/execution/WholeStageCodegen.scala | 8 ++++++--
2 files changed, 20 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/18bcbbdd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala
index 9b8b638..9d99bbf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeFormatter.scala
@@ -25,6 +25,20 @@ package org.apache.spark.sql.catalyst.expressions.codegen
*/
object CodeFormatter {
def format(code: String): String = new CodeFormatter().addLines(code).result()
+ def stripExtraNewLines(input: String): String = {
+ val code = new StringBuilder
+ var lastLine: String = "dummy"
+ input.split('\n').foreach { l =>
+ val line = l.trim()
+ val skip = line == "" && (lastLine == "" || lastLine.endsWith("{"))
+ if (!skip) {
+ code.append(line)
+ code.append("\n")
+ }
+ lastLine = line
+ }
+ code.result()
+ }
}
private class CodeFormatter {
http://git-wip-us.apache.org/repos/asf/spark/blob/18bcbbdd/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
index b200239..30f74fc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegen.scala
@@ -237,6 +237,9 @@ case class WholeStageCodegen(plan: CodegenSupport, children: Seq[SparkPlan])
return new GeneratedIterator(references);
}
+ /** Codegened pipeline for:
+ * ${plan.treeString.trim}
+ */
class GeneratedIterator extends org.apache.spark.sql.execution.BufferedRowIterator {
private Object[] references;
@@ -256,8 +259,9 @@ case class WholeStageCodegen(plan: CodegenSupport, children: Seq[SparkPlan])
"""
// try to compile, helpful for debug
- // println(s"${CodeFormatter.format(source)}")
- CodeGenerator.compile(source)
+ val cleanedSource = CodeFormatter.stripExtraNewLines(source)
+ // println(s"${CodeFormatter.format(cleanedSource)}")
+ CodeGenerator.compile(cleanedSource)
plan.upstream().mapPartitions { iter =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org