You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/07/23 13:33:07 UTC
spark git commit: [SPARK-10063] Follow-up: remove a useless test
related to an old output committer
Repository: spark
Updated Branches:
refs/heads/master cc00e99d5 -> ccaee5b54
[SPARK-10063] Follow-up: remove a useless test related to an old output committer
## What changes were proposed in this pull request?
It's a follow-up of https://github.com/apache/spark/pull/18689 , which forgot to remove a useless test.
## How was this patch tested?
N/A
Author: Wenchen Fan <we...@databricks.com>
Closes #18716 from cloud-fan/test.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ccaee5b5
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ccaee5b5
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ccaee5b5
Branch: refs/heads/master
Commit: ccaee5b54d119155fdf020ba4c8c084c4cd37e77
Parents: cc00e99
Author: Wenchen Fan <we...@databricks.com>
Authored: Sun Jul 23 21:32:59 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Sun Jul 23 21:32:59 2017 +0800
----------------------------------------------------------------------
.../sql/sources/HadoopFsRelationTest.scala | 73 --------------------
1 file changed, 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ccaee5b5/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
----------------------------------------------------------------------
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
index d23b66a..80aff44 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/sources/HadoopFsRelationTest.scala
@@ -22,9 +22,6 @@ import java.io.File
import scala.util.Random
import org.apache.hadoop.fs.Path
-import org.apache.hadoop.mapreduce.{JobContext, TaskAttemptContext}
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter
-import org.apache.parquet.hadoop.ParquetOutputCommitter
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.sql._
@@ -783,52 +780,6 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
}
}
- test("SPARK-8578 specified custom output committer will not be used to append data") {
- withSQLConf(SQLConf.FILE_COMMIT_PROTOCOL_CLASS.key ->
- classOf[SQLHadoopMapReduceCommitProtocol].getCanonicalName) {
- val extraOptions = Map[String, String](
- SQLConf.OUTPUT_COMMITTER_CLASS.key -> classOf[AlwaysFailOutputCommitter].getName,
- // Since Parquet has its own output committer setting, also set it
- // to AlwaysFailParquetOutputCommitter at here.
- "spark.sql.parquet.output.committer.class" ->
- classOf[AlwaysFailParquetOutputCommitter].getName
- )
-
- val df = spark.range(1, 10).toDF("i")
- withTempPath { dir =>
- df.write.mode("append").format(dataSourceName).save(dir.getCanonicalPath)
- // Because there data already exists,
- // this append should succeed because we will use the output committer associated
- // with file format and AlwaysFailOutputCommitter will not be used.
- df.write.mode("append").format(dataSourceName).save(dir.getCanonicalPath)
- checkAnswer(
- spark.read
- .format(dataSourceName)
- .option("dataSchema", df.schema.json)
- .options(extraOptions)
- .load(dir.getCanonicalPath),
- df.union(df))
-
- // This will fail because AlwaysFailOutputCommitter is used when we do append.
- intercept[Exception] {
- df.write.mode("overwrite")
- .options(extraOptions).format(dataSourceName).save(dir.getCanonicalPath)
- }
- }
- withTempPath { dir =>
- // Because there is no existing data,
- // this append will fail because AlwaysFailOutputCommitter is used when we do append
- // and there is no existing data.
- intercept[Exception] {
- df.write.mode("append")
- .options(extraOptions)
- .format(dataSourceName)
- .save(dir.getCanonicalPath)
- }
- }
- }
- }
-
test("SPARK-8887: Explicitly define which data types can be used as dynamic partition columns") {
val df = Seq(
(1, "v1", Array(1, 2, 3), Map("k1" -> "v1"), Tuple2(1, "4")),
@@ -898,27 +849,3 @@ abstract class HadoopFsRelationTest extends QueryTest with SQLTestUtils with Tes
}
}
}
-
-// This class is used to test SPARK-8578. We should not use any custom output committer when
-// we actually append data to an existing dir.
-class AlwaysFailOutputCommitter(
- outputPath: Path,
- context: TaskAttemptContext)
- extends FileOutputCommitter(outputPath, context) {
-
- override def commitJob(context: JobContext): Unit = {
- sys.error("Intentional job commitment failure for testing purpose.")
- }
-}
-
-// This class is used to test SPARK-8578. We should not use any custom output committer when
-// we actually append data to an existing dir.
-class AlwaysFailParquetOutputCommitter(
- outputPath: Path,
- context: TaskAttemptContext)
- extends ParquetOutputCommitter(outputPath, context) {
-
- override def commitJob(context: JobContext): Unit = {
- sys.error("Intentional job commitment failure for testing purpose.")
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org