You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by cloud-fan <gi...@git.apache.org> on 2018/08/23 04:18:36 UTC

[GitHub] spark pull request #22157: [SPARK-25126][SQL] Avoid creating Reader for all ...

Github user cloud-fan commented on a diff in the pull request:

    https://github.com/apache/spark/pull/22157#discussion_r212178321
  
    --- Diff: sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/orc/OrcQuerySuite.scala ---
    @@ -562,20 +562,57 @@ abstract class OrcQueryTest extends OrcTest {
           }
         }
     
    +    def testAllCorruptFiles(): Unit = {
    +      withTempDir { dir =>
    +        val basePath = dir.getCanonicalPath
    +        spark.range(1).toDF("a").write.json(new Path(basePath, "first").toString)
    +        spark.range(1, 2).toDF("a").write.json(new Path(basePath, "second").toString)
    +        val df = spark.read.orc(
    +          new Path(basePath, "first").toString,
    +          new Path(basePath, "second").toString)
    +        assert(df.count() == 0)
    +      }
    +    }
    +
    +    def testAllCorruptFilesWithoutSchemaInfer(): Unit = {
    +      withTempDir { dir =>
    +        val basePath = dir.getCanonicalPath
    +        spark.range(1).toDF("a").write.json(new Path(basePath, "first").toString)
    +        spark.range(1, 2).toDF("a").write.json(new Path(basePath, "second").toString)
    +        val df = spark.read.schema("a long").orc(
    +          new Path(basePath, "first").toString,
    +          new Path(basePath, "second").toString)
    +        assert(df.count() == 0)
    +      }
    +    }
    +
         withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "true") {
           testIgnoreCorruptFiles()
           testIgnoreCorruptFilesWithoutSchemaInfer()
    +      val m1 = intercept[AnalysisException] {
    +        testAllCorruptFiles()
    +      }.getMessage
    +      assert(m1.contains("Unable to infer schema for ORC"))
    +      testAllCorruptFilesWithoutSchemaInfer()
         }
     
         withSQLConf(SQLConf.IGNORE_CORRUPT_FILES.key -> "false") {
           val m1 = intercept[SparkException] {
             testIgnoreCorruptFiles()
           }.getMessage
    -      assert(m1.contains("Could not read footer for file"))
    +      assert(m1.contains("Malformed ORC file"))
    --- End diff --
    
    why the error message changed?


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org