You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2016/10/14 04:36:11 UTC
spark git commit: [SPARK-17927][SQL] Remove dead code in
WriterContainer.
Repository: spark
Updated Branches:
refs/heads/master 44cbb61b3 -> 8543996c3
[SPARK-17927][SQL] Remove dead code in WriterContainer.
## What changes were proposed in this pull request?
speculationEnabled and DATASOURCE_OUTPUTPATH seem like just dead code.
## How was this patch tested?
Tests should fail if they are not dead code.
Author: Reynold Xin <rx...@databricks.com>
Closes #15477 from rxin/SPARK-17927.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8543996c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8543996c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8543996c
Branch: refs/heads/master
Commit: 8543996c3f44098a521fc6b90ca0bb575f606e2a
Parents: 44cbb61
Author: Reynold Xin <rx...@databricks.com>
Authored: Fri Oct 14 12:35:59 2016 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Fri Oct 14 12:35:59 2016 +0800
----------------------------------------------------------------------
.../sql/execution/datasources/WriterContainer.scala | 11 -----------
1 file changed, 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8543996c/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
index 7880c7c..253aa44 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/WriterContainer.scala
@@ -49,7 +49,6 @@ private[datasources] case class WriteRelation(
object WriterContainer {
val DATASOURCE_WRITEJOBUUID = "spark.sql.sources.writeJobUUID"
- val DATASOURCE_OUTPUTPATH = "spark.sql.sources.output.path"
}
private[datasources] abstract class BaseWriterContainer(
@@ -73,9 +72,6 @@ private[datasources] abstract class BaseWriterContainer(
// This is only used on driver side.
@transient private val jobContext: JobContext = job
- private val speculationEnabled: Boolean =
- relation.sparkSession.sparkContext.conf.getBoolean("spark.speculation", defaultValue = false)
-
// The following fields are initialized and used on both driver and executor side.
@transient protected var outputCommitter: OutputCommitter = _
@transient private var jobId: JobID = _
@@ -247,8 +243,6 @@ private[datasources] class DefaultWriterContainer(
def writeRows(taskContext: TaskContext, iterator: Iterator[InternalRow]): Unit = {
executorSideSetup(taskContext)
- val configuration = taskAttemptContext.getConfiguration
- configuration.set(WriterContainer.DATASOURCE_OUTPUTPATH, outputPath)
var writer = newOutputWriter(getWorkPath)
writer.initConverter(dataSchema)
@@ -353,15 +347,10 @@ private[datasources] class DynamicPartitionWriterContainer(
private def newOutputWriter(
key: InternalRow,
getPartitionString: UnsafeProjection): OutputWriter = {
- val configuration = taskAttemptContext.getConfiguration
val path = if (partitionColumns.nonEmpty) {
val partitionPath = getPartitionString(key).getString(0)
- configuration.set(
- WriterContainer.DATASOURCE_OUTPUTPATH,
- new Path(outputPath, partitionPath).toString)
new Path(getWorkPath, partitionPath).toString
} else {
- configuration.set(WriterContainer.DATASOURCE_OUTPUTPATH, outputPath)
getWorkPath
}
val bucketId = getBucketIdFromKey(key)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org