You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ya...@apache.org on 2019/02/22 06:23:29 UTC
[spark] branch master updated: [SPARK-26851][SQL][FOLLOWUP] Fix
cachedColumnBuffers field for Scala 2.11 build
This is an automated email from the ASF dual-hosted git repository.
yamamuro pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 95bb012 [SPARK-26851][SQL][FOLLOWUP] Fix cachedColumnBuffers field for Scala 2.11 build
95bb012 is described below
commit 95bb01282cc94f95bbc69aafcbc1550b137238be
Author: Sean Owen <se...@databricks.com>
AuthorDate: Fri Feb 22 15:22:52 2019 +0900
[SPARK-26851][SQL][FOLLOWUP] Fix cachedColumnBuffers field for Scala 2.11 build
## What changes were proposed in this pull request?
Per https://github.com/apache/spark/pull/23768/files#r259083019 the last change to this line here caused the 2.11 build to fail. It's worked around by making `_cachedColumnBuffers` a field, as it was never set by callers to anything other than its default of null.
## How was this patch tested?
Existing tests.
Closes #23864 from srowen/SPARK-26851.2.
Authored-by: Sean Owen <se...@databricks.com>
Signed-off-by: Takeshi Yamamuro <ya...@apache.org>
---
.../main/scala/org/apache/spark/sql/execution/CacheManager.scala | 3 +--
.../org/apache/spark/sql/execution/columnar/InMemoryRelation.scala | 7 ++++---
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
index c6ee735..f7a78ea 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/CacheManager.scala
@@ -213,8 +213,7 @@ class CacheManager extends Logging {
cd.cachedRepresentation.cacheBuilder.clearCache()
val plan = spark.sessionState.executePlan(cd.plan).executedPlan
val newCache = InMemoryRelation(
- cacheBuilder = cd.cachedRepresentation
- .cacheBuilder.copy(cachedPlan = plan)(_cachedColumnBuffers = null),
+ cacheBuilder = cd.cachedRepresentation.cacheBuilder.copy(cachedPlan = plan),
logicalPlan = cd.plan)
val recomputedPlan = cd.copy(cachedRepresentation = newCache)
writeLock {
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
index bc6e958..7180853 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/columnar/InMemoryRelation.scala
@@ -48,8 +48,9 @@ case class CachedRDDBuilder(
batchSize: Int,
storageLevel: StorageLevel,
@transient cachedPlan: SparkPlan,
- tableName: Option[String])(
- @transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null) {
+ tableName: Option[String]) {
+
+ @transient @volatile private var _cachedColumnBuffers: RDD[CachedBatch] = null
val sizeInBytesStats: LongAccumulator = cachedPlan.sqlContext.sparkContext.longAccumulator
@@ -143,7 +144,7 @@ object InMemoryRelation {
child: SparkPlan,
tableName: Option[String],
logicalPlan: LogicalPlan): InMemoryRelation = {
- val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)()
+ val cacheBuilder = CachedRDDBuilder(useCompression, batchSize, storageLevel, child, tableName)
new InMemoryRelation(child.output, cacheBuilder, logicalPlan.outputOrdering)(
statsOfPlanToCache = logicalPlan.stats)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org