You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2021/06/22 20:57:47 UTC

[spark] branch branch-3.1 updated: [SPARK-35700][SQL][FOLLOWUP] Read schema from ORC files should strip CHAR/VARCHAR types

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.1 by this push:
     new fd7df35  [SPARK-35700][SQL][FOLLOWUP] Read schema from ORC files should strip CHAR/VARCHAR types
fd7df35 is described below

commit fd7df35f7df4cfe55fecfa44fed70fe967791d9b
Author: Wenchen Fan <we...@databricks.com>
AuthorDate: Tue Jun 22 13:50:49 2021 -0700

    [SPARK-35700][SQL][FOLLOWUP] Read schema from ORC files should strip CHAR/VARCHAR types
    
    This is a followup of https://github.com/apache/spark/pull/33001 , to provide a more direct fix.
    
    The regression in 3.1 was caused by the fact that we changed the parser and allow the parser to return CHAR/VARCHAR type. We should have replaced CHAR/VARCHAR with STRING before the data type flows into the query engine, however, `OrcUtils` is missed.
    
    When reading ORC files, at the task side we will read the real schema from ORC file metadata, then apply filter pushdown. For some reason, the implementation turns ORC schema to Spark schema before filter pushdown, and this step does not strip CHAR/VARCHAR. Note, for Parquet we use the Parquet schema directly in filter pushdown, and do not this have problem.
    
    This PR proposes to replace the CHAR/VARCHAR with STRING when turning ORC schema to Spark schema.
    
    a more directly bug fix
    
    no
    
    existing tests
    
    Closes #33030 from cloud-fan/help.
    
    Authored-by: Wenchen Fan <we...@databricks.com>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit a2c1a55b1fed5d552f6bc355ba3c542dfeee5a91)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 .../sql/execution/datasources/orc/OrcFilters.scala      |  2 +-
 .../spark/sql/execution/datasources/orc/OrcUtils.scala  | 17 +++++++++++------
 .../org/apache/spark/sql/HiveCharVarcharTestSuite.scala |  1 -
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
index f54cff2..9511fc3 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcFilters.scala
@@ -143,7 +143,7 @@ private[sql] object OrcFilters extends OrcFiltersBase {
     case BooleanType => PredicateLeaf.Type.BOOLEAN
     case ByteType | ShortType | IntegerType | LongType => PredicateLeaf.Type.LONG
     case FloatType | DoubleType => PredicateLeaf.Type.FLOAT
-    case StringType | _: CharType | _: VarcharType => PredicateLeaf.Type.STRING
+    case StringType => PredicateLeaf.Type.STRING
     case DateType => PredicateLeaf.Type.DATE
     case TimestampType => PredicateLeaf.Type.TIMESTAMP
     case _: DecimalType => PredicateLeaf.Type.DECIMAL
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
index 623f4f7..2190f7e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/orc/OrcUtils.scala
@@ -32,7 +32,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.{SPARK_VERSION_METADATA_KEY, SparkSession}
 import org.apache.spark.sql.catalyst.analysis.caseSensitiveResolution
 import org.apache.spark.sql.catalyst.parser.CatalystSqlParser
-import org.apache.spark.sql.catalyst.util.quoteIdentifier
+import org.apache.spark.sql.catalyst.util.{quoteIdentifier, CharVarcharUtils}
 import org.apache.spark.sql.execution.datasources.SchemaMergeUtils
 import org.apache.spark.sql.types._
 import org.apache.spark.util.{ThreadUtils, Utils}
@@ -81,6 +81,13 @@ object OrcUtils extends Logging {
     }
   }
 
+  private def toCatalystSchema(schema: TypeDescription): StructType = {
+    // The Spark query engine has not completely supported CHAR/VARCHAR type yet, and here we
+    // replace the orc CHAR/VARCHAR with STRING type.
+    CharVarcharUtils.replaceCharVarcharWithStringInSchema(
+      CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType])
+  }
+
   def readSchema(sparkSession: SparkSession, files: Seq[FileStatus], options: Map[String, String])
       : Option[StructType] = {
     val ignoreCorruptFiles = sparkSession.sessionState.conf.ignoreCorruptFiles
@@ -88,7 +95,7 @@ object OrcUtils extends Logging {
     files.toIterator.map(file => readSchema(file.getPath, conf, ignoreCorruptFiles)).collectFirst {
       case Some(schema) =>
         logDebug(s"Reading schema from file $files, got Hive schema string: $schema")
-        CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType]
+        toCatalystSchema(schema)
     }
   }
 
@@ -97,8 +104,7 @@ object OrcUtils extends Logging {
       conf: Configuration,
       ignoreCorruptFiles: Boolean): Option[StructType] = {
     readSchema(file, conf, ignoreCorruptFiles) match {
-      case Some(schema) =>
-        Some(CatalystSqlParser.parseDataType(schema.toString).asInstanceOf[StructType])
+      case Some(schema) => Some(toCatalystSchema(schema))
 
       case None =>
         // Field names is empty or `FileFormatException` was thrown but ignoreCorruptFiles is true.
@@ -113,8 +119,7 @@ object OrcUtils extends Logging {
   def readOrcSchemasInParallel(
     files: Seq[FileStatus], conf: Configuration, ignoreCorruptFiles: Boolean): Seq[StructType] = {
     ThreadUtils.parmap(files, "readingOrcSchemas", 8) { currentFile =>
-      OrcUtils.readSchema(currentFile.getPath, conf, ignoreCorruptFiles)
-        .map(s => CatalystSqlParser.parseDataType(s.toString).asInstanceOf[StructType])
+      OrcUtils.readSchema(currentFile.getPath, conf, ignoreCorruptFiles).map(toCatalystSchema)
     }.flatten
   }
 
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala
index 3ab2522..48406d9 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/HiveCharVarcharTestSuite.scala
@@ -52,7 +52,6 @@ class HiveCharVarcharTestSuite extends CharVarcharTestSuite with TestHiveSinglet
   }
 
   test("SPARK-35700: Read char/varchar orc table with created and written by external systems") {
-
     withTable("t") {
       hiveClient.runSqlHive("CREATE TABLE t(c CHAR(5), v VARCHAR(7)) STORED AS ORC")
       hiveClient.runSqlHive("INSERT INTO t VALUES('Spark', 'kyuubi')")

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org