You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2017/07/09 23:30:38 UTC
spark git commit: [SPARK-18016][SQL][FOLLOWUP] merge
declareAddedFunctions, initNestedClasses and declareNestedClasses
Repository: spark
Updated Branches:
refs/heads/master 08e0d033b -> 680b33f16
[SPARK-18016][SQL][FOLLOWUP] merge declareAddedFunctions, initNestedClasses and declareNestedClasses
## What changes were proposed in this pull request?
These 3 methods have to be used together, so it makes more sense to merge them into one method and then the caller side only need to call one method.
## How was this patch tested?
existing tests.
Author: Wenchen Fan <we...@databricks.com>
Closes #18579 from cloud-fan/minor.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/680b33f1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/680b33f1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/680b33f1
Branch: refs/heads/master
Commit: 680b33f16694b7c460235b11b8c265bc304f795a
Parents: 08e0d03
Author: Wenchen Fan <we...@databricks.com>
Authored: Sun Jul 9 16:30:35 2017 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Sun Jul 9 16:30:35 2017 -0700
----------------------------------------------------------------------
.../expressions/codegen/CodeGenerator.scala | 29 ++++++++------------
.../codegen/GenerateMutableProjection.scala | 5 +---
.../expressions/codegen/GenerateOrdering.scala | 5 +---
.../expressions/codegen/GeneratePredicate.scala | 5 +---
.../codegen/GenerateSafeProjection.scala | 5 +---
.../codegen/GenerateUnsafeProjection.scala | 5 +---
.../sql/execution/WholeStageCodegenExec.scala | 5 +---
.../columnar/GenerateColumnAccessor.scala | 5 +---
8 files changed, 18 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index b15bf2c..7cf9daf 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -302,29 +302,20 @@ class CodegenContext {
}
/**
- * Instantiates all nested, private sub-classes as objects to the `OuterClass`
+ * Declares all function code. If the added functions are too many, split them into nested
+ * sub-classes to avoid hitting Java compiler constant pool limitation.
*/
- private[sql] def initNestedClasses(): String = {
+ def declareAddedFunctions(): String = {
+ val inlinedFunctions = classFunctions(outerClassName).values
+
// Nested, private sub-classes have no mutable state (though they do reference the outer class'
// mutable state), so we declare and initialize them inline to the OuterClass.
- classes.filter(_._1 != outerClassName).map {
+ val initNestedClasses = classes.filter(_._1 != outerClassName).map {
case (className, classInstance) =>
s"private $className $classInstance = new $className();"
- }.mkString("\n")
- }
-
- /**
- * Declares all function code that should be inlined to the `OuterClass`.
- */
- private[sql] def declareAddedFunctions(): String = {
- classFunctions(outerClassName).values.mkString("\n")
- }
+ }
- /**
- * Declares all nested, private sub-classes and the function code that should be inlined to them.
- */
- private[sql] def declareNestedClasses(): String = {
- classFunctions.filterKeys(_ != outerClassName).map {
+ val declareNestedClasses = classFunctions.filterKeys(_ != outerClassName).map {
case (className, functions) =>
s"""
|private class $className {
@@ -332,7 +323,9 @@ class CodegenContext {
|}
""".stripMargin
}
- }.mkString("\n")
+
+ (inlinedFunctions ++ initNestedClasses ++ declareNestedClasses).mkString("\n")
+ }
final val JAVA_BOOLEAN = "boolean"
final val JAVA_BYTE = "byte"
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
index 6357668..3768dcd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateMutableProjection.scala
@@ -115,8 +115,6 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP
${ctx.initPartition()}
}
- ${ctx.declareAddedFunctions()}
-
public ${classOf[BaseMutableProjection].getName} target(InternalRow row) {
mutableRow = row;
return this;
@@ -136,8 +134,7 @@ object GenerateMutableProjection extends CodeGenerator[Seq[Expression], MutableP
return mutableRow;
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}
"""
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
index a319432..4e47895 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateOrdering.scala
@@ -173,15 +173,12 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
${ctx.initMutableStates()}
}
- ${ctx.declareAddedFunctions()}
-
public int compare(InternalRow a, InternalRow b) {
$comparisons
return 0;
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
index b400783..e35b9dd 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GeneratePredicate.scala
@@ -66,15 +66,12 @@ object GeneratePredicate extends CodeGenerator[Expression, Predicate] {
${ctx.initPartition()}
}
- ${ctx.declareAddedFunctions()}
-
public boolean eval(InternalRow ${ctx.INPUT_ROW}) {
${eval.code}
return !${eval.isNull} && ${eval.value};
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
index dd0419d..192701a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateSafeProjection.scala
@@ -175,16 +175,13 @@ object GenerateSafeProjection extends CodeGenerator[Seq[Expression], Projection]
${ctx.initPartition()}
}
- ${ctx.declareAddedFunctions()}
-
public java.lang.Object apply(java.lang.Object _i) {
InternalRow ${ctx.INPUT_ROW} = (InternalRow) _i;
$allExpressions
return mutableRow;
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}
"""
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
index 6be69d1..f2a66ef 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateUnsafeProjection.scala
@@ -391,8 +391,6 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
${ctx.initPartition()}
}
- ${ctx.declareAddedFunctions()}
-
// Scala.Function1 need this
public java.lang.Object apply(java.lang.Object row) {
return apply((InternalRow) row);
@@ -403,8 +401,7 @@ object GenerateUnsafeProjection extends CodeGenerator[Seq[Expression], UnsafePro
return ${eval.value};
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}
"""
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
index 0bd28e3..1007a7d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala
@@ -352,14 +352,11 @@ case class WholeStageCodegenExec(child: SparkPlan) extends UnaryExecNode with Co
${ctx.initPartition()}
}
- ${ctx.declareAddedFunctions()}
-
protected void processNext() throws java.io.IOException {
${code.trim}
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}
""".trim
http://git-wip-us.apache.org/repos/asf/spark/blob/680b33f1/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
index fc977f2..da34643 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/GenerateColumnAccessor.scala
@@ -192,8 +192,6 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
this.columnIndexes = columnIndexes;
}
- ${ctx.declareAddedFunctions()}
-
public boolean hasNext() {
if (currentRow < numRowsInBatch) {
return true;
@@ -222,8 +220,7 @@ object GenerateColumnAccessor extends CodeGenerator[Seq[DataType], ColumnarItera
return unsafeRow;
}
- ${ctx.initNestedClasses()}
- ${ctx.declareNestedClasses()}
+ ${ctx.declareAddedFunctions()}
}"""
val code = CodeFormatter.stripOverlappingComments(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org