You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2015/08/16 08:52:32 UTC
spark git commit: [SPARK-9973] [SQL] Correct in-memory columnar
buffer size
Repository: spark
Updated Branches:
refs/heads/master 1db7179fa -> 182f9b7a6
[SPARK-9973] [SQL] Correct in-memory columnar buffer size
The `initialSize` argument of `ColumnBuilder.initialize()` should be the
number of rows rather than bytes. However `InMemoryColumnarTableScan`
passes in a byte size, which makes Spark SQL allocate more memory than
necessary when building in-memory columnar buffers.
Author: Kun Xu <vi...@163.com>
Closes #8189 from viper-kun/errorSize.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/182f9b7a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/182f9b7a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/182f9b7a
Branch: refs/heads/master
Commit: 182f9b7a6d3a3ee7ec7de6abc24e296aa794e4e8
Parents: 1db7179
Author: Kun Xu <vi...@163.com>
Authored: Sun Aug 16 14:44:23 2015 +0800
Committer: Cheng Lian <li...@databricks.com>
Committed: Sun Aug 16 14:44:45 2015 +0800
----------------------------------------------------------------------
.../org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/182f9b7a/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
index d553bb61..45f15fd 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/InMemoryColumnarTableScan.scala
@@ -121,8 +121,7 @@ private[sql] case class InMemoryRelation(
def next(): CachedBatch = {
val columnBuilders = output.map { attribute =>
val columnType = ColumnType(attribute.dataType)
- val initialBufferSize = columnType.defaultSize * batchSize
- ColumnBuilder(attribute.dataType, initialBufferSize, attribute.name, useCompression)
+ ColumnBuilder(attribute.dataType, batchSize, attribute.name, useCompression)
}.toArray
var rowCount = 0
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org