You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by an...@apache.org on 2015/09/16 01:53:29 UTC
spark git commit: [SPARK-10612] [SQL] Add prepare to LocalNode.
Repository: spark
Updated Branches:
refs/heads/master b6e998634 -> a63cdc769
[SPARK-10612] [SQL] Add prepare to LocalNode.
The idea is that we should separate the function call that does memory reservation (i.e. prepare) from the function call that consumes the input (e.g. open()), so all operators can be a chance to reserve memory before they are all consumed.
Author: Reynold Xin <rx...@databricks.com>
Closes #8761 from rxin/SPARK-10612.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a63cdc76
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a63cdc76
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a63cdc76
Branch: refs/heads/master
Commit: a63cdc769f511e98b38c3318bcc732c9a6c76c22
Parents: b6e9986
Author: Reynold Xin <rx...@databricks.com>
Authored: Tue Sep 15 16:53:27 2015 -0700
Committer: Andrew Or <an...@databricks.com>
Committed: Tue Sep 15 16:53:27 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/execution/local/LocalNode.scala | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/a63cdc76/sql/core/src/main/scala/org/apache/spark/sql/execution/local/LocalNode.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/local/LocalNode.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/local/LocalNode.scala
index 9840080..569cff5 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/local/LocalNode.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/local/LocalNode.scala
@@ -46,6 +46,14 @@ abstract class LocalNode(conf: SQLConf) extends TreeNode[LocalNode] with Logging
def output: Seq[Attribute]
/**
+ * Called before open(). Prepare can be used to reserve memory needed. It must NOT consume
+ * any input data.
+ *
+ * Implementations of this must also call the `prepare()` function of its children.
+ */
+ def prepare(): Unit = children.foreach(_.prepare())
+
+ /**
* Initializes the iterator state. Must be called before calling `next()`.
*
* Implementations of this must also call the `open()` function of its children.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org