You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/04/11 13:53:04 UTC
spark git commit: [SPARK-23960][SQL][MINOR] Mark
HashAggregateExec.bufVars as transient
Repository: spark
Updated Branches:
refs/heads/master c604d659e -> 271c891b9
[SPARK-23960][SQL][MINOR] Mark HashAggregateExec.bufVars as transient
## What changes were proposed in this pull request?
Mark `HashAggregateExec.bufVars` as transient to avoid it from being serialized.
Also manually null out this field at the end of `doProduceWithoutKeys()` to shorten its lifecycle, because it'll no longer be used after that.
## How was this patch tested?
Existing tests.
Author: Kris Mok <kr...@databricks.com>
Closes #21039 from rednaxelafx/codegen-improve.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/271c891b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/271c891b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/271c891b
Branch: refs/heads/master
Commit: 271c891b91917d660d1f6b995de397c47c7a6058
Parents: c604d65
Author: Kris Mok <kr...@databricks.com>
Authored: Wed Apr 11 21:52:48 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Wed Apr 11 21:52:48 2018 +0800
----------------------------------------------------------------------
.../spark/sql/execution/aggregate/HashAggregateExec.scala | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/271c891b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
index a5dc6eb..965950e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/aggregate/HashAggregateExec.scala
@@ -174,8 +174,8 @@ case class HashAggregateExec(
}
}
- // The variables used as aggregation buffer. Only used for aggregation without keys.
- private var bufVars: Seq[ExprCode] = _
+ // The variables used as aggregation buffer. Only used in codegen for aggregation without keys.
+ @transient private var bufVars: Seq[ExprCode] = _
private def doProduceWithoutKeys(ctx: CodegenContext): String = {
val initAgg = ctx.addMutableState(CodeGenerator.JAVA_BOOLEAN, "initAgg")
@@ -238,6 +238,8 @@ case class HashAggregateExec(
| }
""".stripMargin)
+ bufVars = null // explicitly null this field out to allow the referent to be GC'd sooner
+
val numOutput = metricTerm(ctx, "numOutputRows")
val aggTime = metricTerm(ctx, "aggTime")
val beforeAgg = ctx.freshName("beforeAgg")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org