You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by cloud-fan <gi...@git.apache.org> on 2018/08/23 04:18:36 UTC
[GitHub] spark pull request #22157: [SPARK-25126][SQL] Avoid creating Reader for all ...
Github user cloud-fan commented on a diff in the pull request:
https://github.com/apache/spark/pull/22157#discussion_r212178321
--- Diff: sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala ---
@@ -562,20 +562,57 @@ abstract class OrcQueryTest extends OrcTest {
}
}
+ def testAllCorruptFiles(): Unit = {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(1).toDF("a").write.json(new Path(basePath, "first").toString)
+ spark.range(1, 2).toDF("a").write.json(new Path(basePath, "second").toString)
+ val df = spark.read.orc(
+ new Path(basePath, "first").toString,
+ new Path(basePath, "second").toString)
+ assert(df.count() == 0)
+ }
+ }
+
+ def testAllCorruptFilesWithoutSchemaInfer(): Unit = {
+ withTempDir { dir =>
+ val basePath = dir.getCanonicalPath
+ spark.range(1).toDF("a").write.json(new Path(basePath, "first").toString)
+ spark.range(1, 2).toDF("a").write.json(new Path(basePath, "second").toString)
+ val df = spark.read.schema("a long").orc(
+ new Path(basePath, "first").toString,
+ new Path(basePath, "second").toString)
+ assert(df.count() == 0)
+ }
+ }
+
withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "true") {
testIgnoreCorruptFiles()
testIgnoreCorruptFilesWithoutSchemaInfer()
+ val m1 = intercept[AnalysisException] {
+ testAllCorruptFiles()
+ }.getMessage
+ assert(m1.contains("Unable to infer schema for ORC"))
+ testAllCorruptFilesWithoutSchemaInfer()
}
withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
val m1 = intercept[SparkException] {
testIgnoreCorruptFiles()
}.getMessage
- assert(m1.contains("Could not read footer for file"))
+ assert(m1.contains("Malformed ORC file"))
--- End diff --
why the error message changed?
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org