You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by su...@apache.org on 2023/04/20 04:03:02 UTC

[spark] branch master updated: [SPARK-43195][CORE] Remove unnecessary serializable wrapper in HadoopFSUtils

This is an automated email from the ASF dual-hosted git repository.

sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 29b40f6fd0c [SPARK-43195][CORE] Remove unnecessary serializable wrapper in HadoopFSUtils
29b40f6fd0c is described below

commit 29b40f6fd0c42e594e24b12c8b828913139e8f54
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Wed Apr 19 21:02:46 2023 -0700

    [SPARK-43195][CORE] Remove unnecessary serializable wrapper in HadoopFSUtils
    
    ### What changes were proposed in this pull request?
    
    Remove unnecessary serializable wrapper in `HadoopFSUtils`.
    
    ### Why are the changes needed?
    
    `Path`, `FileStatus` become serializable in Hadoop3, since SPARK-42452 removed support for Hadoop2, we can remove those wrapper now.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass CI.
    
    Closes #40854 from pan3793/SPARK-43195.
    
    Authored-by: Cheng Pan <ch...@apache.org>
    Signed-off-by: Chao Sun <su...@apple.com>
---
 .../apache/spark/serializer/KryoSerializer.scala   |  4 --
 .../org/apache/spark/util/HadoopFSUtils.scala      | 73 ++--------------------
 2 files changed, 5 insertions(+), 72 deletions(-)

diff --git a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
index d79f6453bc5..1736088b498 100644
--- a/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
+++ b/core/src/main/scala/org/apache/spark/serializer/KryoSerializer.scala
@@ -510,10 +510,6 @@ private[serializer] object KryoSerializer {
   // SQL / ML / MLlib classes once and then re-use that filtered list in newInstance() calls.
   private lazy val loadableSparkClasses: Seq[Class[_]] = {
     Seq(
-      "org.apache.spark.util.HadoopFSUtils$SerializableBlockLocation",
-      "[Lorg.apache.spark.util.HadoopFSUtils$SerializableBlockLocation;",
-      "org.apache.spark.util.HadoopFSUtils$SerializableFileStatus",
-
       "org.apache.spark.sql.catalyst.expressions.BoundReference",
       "org.apache.spark.sql.catalyst.expressions.SortOrder",
       "[Lorg.apache.spark.sql.catalyst.expressions.SortOrder;",
diff --git a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
index 01dc3ba68cc..bb03cac8f1f 100644
--- a/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
+++ b/core/src/main/scala/org/apache/spark/util/HadoopFSUtils.scala
@@ -102,14 +102,13 @@ private[spark] object HadoopFSUtils extends Logging {
     HiveCatalogMetrics.incrementParallelListingJobCount(1)
 
     val serializableConfiguration = new SerializableConfiguration(hadoopConf)
-    val serializedPaths = paths.map(_.toString)
 
     // Set the number of parallelism to prevent following file listing from generating many tasks
     // in case of large #defaultParallelism.
     val numParallelism = Math.min(paths.size, parallelismMax)
 
     val previousJobDescription = sc.getLocalProperty(SparkContext.SPARK_JOB_DESCRIPTION)
-    val statusMap = try {
+    try {
       val description = paths.size match {
         case 0 =>
           "Listing leaf files and directories 0 paths"
@@ -119,11 +118,10 @@ private[spark] object HadoopFSUtils extends Logging {
           s"Listing leaf files and directories for $s paths:<br/>${paths(0)}, ..."
       }
       sc.setJobDescription(description)
-      sc
-        .parallelize(serializedPaths, numParallelism)
-        .mapPartitions { pathStrings =>
+      sc.parallelize(paths, numParallelism)
+        .mapPartitions { pathsEachPartition =>
           val hadoopConf = serializableConfiguration.value
-          pathStrings.map(new Path(_)).toSeq.map { path =>
+          pathsEachPartition.map { path =>
             val leafFiles = listLeafFiles(
               path = path,
               hadoopConf = hadoopConf,
@@ -135,54 +133,11 @@ private[spark] object HadoopFSUtils extends Logging {
               parallelismThreshold = Int.MaxValue,
               parallelismMax = 0)
             (path, leafFiles)
-          }.iterator
-        }.map { case (path, statuses) =>
-            val serializableStatuses = statuses.map { status =>
-              // Turn FileStatus into SerializableFileStatus so we can send it back to the driver
-              val blockLocations = status match {
-                case f: LocatedFileStatus =>
-                  f.getBlockLocations.map { loc =>
-                    SerializableBlockLocation(
-                      loc.getNames,
-                      loc.getHosts,
-                      loc.getOffset,
-                      loc.getLength)
-                  }
-
-                case _ =>
-                  Array.empty[SerializableBlockLocation]
-              }
-
-              SerializableFileStatus(
-                status.getPath.toString,
-                status.getLen,
-                status.isDirectory,
-                status.getReplication,
-                status.getBlockSize,
-                status.getModificationTime,
-                status.getAccessTime,
-                blockLocations)
-            }
-            (path.toString, serializableStatuses)
+          }
         }.collect()
     } finally {
       sc.setJobDescription(previousJobDescription)
     }
-
-    // turn SerializableFileStatus back to Status
-    statusMap.map { case (path, serializableStatuses) =>
-      val statuses = serializableStatuses.map { f =>
-        val blockLocations = f.blockLocations.map { loc =>
-          new BlockLocation(loc.names, loc.hosts, loc.offset, loc.length)
-        }
-        new LocatedFileStatus(
-          new FileStatus(
-            f.length, f.isDir, f.blockReplication, f.blockSize, f.modificationTime,
-            new Path(f.path)),
-          blockLocations)
-      }
-      (new Path(path), statuses)
-    }
   }
 
   // scalastyle:off argcount
@@ -340,24 +295,6 @@ private[spark] object HadoopFSUtils extends Logging {
   }
   // scalastyle:on argcount
 
-  /** A serializable variant of HDFS's BlockLocation. This is required by Hadoop 2.7. */
-  private case class SerializableBlockLocation(
-    names: Array[String],
-    hosts: Array[String],
-    offset: Long,
-    length: Long)
-
-  /** A serializable variant of HDFS's FileStatus. This is required by Hadoop 2.7. */
-  private case class SerializableFileStatus(
-    path: String,
-    length: Long,
-    isDir: Boolean,
-    blockReplication: Short,
-    blockSize: Long,
-    modificationTime: Long,
-    accessTime: Long,
-    blockLocations: Array[SerializableBlockLocation])
-
   /** Checks if we should filter out this path name. */
   def shouldFilterOutPathName(pathName: String): Boolean = {
     // We filter follow paths:


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org