[hudi] branch master updated: [HUDI-4944] Use mor solution to uri filenotfound problem (#8809)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 4193f40c4d9 [HUDI-4944] Use mor solution to uri filenotfound problem (#8809)
4193f40c4d9 is described below

commit 4193f40c4d9a94750b66a5d1bc9193d8803d92a4
Author: Jon Vexler <jb...@gmail.com>
AuthorDate: Tue May 30 16:29:18 2023 -0400

    [HUDI-4944] Use mor solution to uri filenotfound problem (#8809)
    
    Co-authored-by: Jonathan Vexler <=>
---
 .../org/apache/hudi/HoodieBootstrapRelation.scala  | 31 +++++++---------------
 1 file changed, 10 insertions(+), 21 deletions(-)

diff --git a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
index 34db2850501..70ebd21d7e9 100644
--- a/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
+++ b/hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieBootstrapRelation.scala
@@ -18,13 +18,11 @@
 
 package org.apache.hudi
 
-import org.apache.hadoop.fs.{FileStatus, Path}
+import org.apache.hadoop.fs.Path
 import org.apache.hudi.HoodieBaseRelation.{BaseFileReader, convertToAvroSchema, projectReader}
-import org.apache.hudi.HoodieBootstrapRelation.validate
+import org.apache.hudi.HoodieBootstrapRelation.{getFilePath, validate}
 import org.apache.hudi.common.table.HoodieTableMetaClient
-import org.apache.hudi.common.util.PartitionPathEncodeUtils
 import org.apache.hudi.common.util.ValidationUtils.checkState
-import org.apache.hudi.hadoop.CachingPath
 import org.apache.spark.rdd.RDD
 import org.apache.spark.sql.SQLContext
 import org.apache.spark.sql.catalyst.InternalRow
@@ -34,8 +32,6 @@ import org.apache.spark.sql.hudi.HoodieSqlCommonUtils.isMetaField
 import org.apache.spark.sql.sources.Filter
 import org.apache.spark.sql.types.StructType
 
-import java.net.URI
-
 case class HoodieBootstrapSplit(dataFile: PartitionedFile, skeletonFile: Option[PartitionedFile] = None) extends HoodieFileSplit
 
 /**
@@ -77,16 +73,11 @@ case class HoodieBootstrapRelation(override val sqlContext: SQLContext,
     val isPartitioned = metaClient.getTableConfig.isTablePartitioned
     fileSlices.map { fileSlice =>
       val baseFile = fileSlice.getBaseFile.get()
-
       if (baseFile.getBootstrapBaseFile.isPresent) {
         val partitionValues = getPartitionColumnsAsInternalRowInternal(baseFile.getBootstrapBaseFile.get.getFileStatus,
             bootstrapBasePath, extractPartitionValuesFromPartitionPath = isPartitioned)
-        val filePath = if (isPartitioned) {
-          encodePartitionPath(baseFile.getBootstrapBaseFile.get.getFileStatus)
-        } else {
-          baseFile.getBootstrapBaseFile.get.getPath
-        }
-        val dataFile = PartitionedFile(partitionValues, filePath, 0, baseFile.getBootstrapBaseFile.get().getFileLen)
+        val dataFile = PartitionedFile(partitionValues, getFilePath(baseFile.getBootstrapBaseFile.get.getFileStatus.getPath),
+          0, baseFile.getBootstrapBaseFile.get().getFileLen)
         val skeletonFile = Option(PartitionedFile(InternalRow.empty, baseFile.getPath, 0, baseFile.getFileLen))
 
         HoodieBootstrapSplit(dataFile, skeletonFile)
@@ -210,14 +201,6 @@ case class HoodieBootstrapRelation(override val sqlContext: SQLContext,
         optParams)(sparkSession)
   }
 
-  //TODO: This should be unnecessary with spark 3.4 [SPARK-41970]
-  private def encodePartitionPath(file: FileStatus): String = {
-    val tablePathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(bootstrapBasePath)
-    val partitionPathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(file.getPath.getParent)
-    val filePathWithoutScheme = CachingPath.getPathWithoutSchemeAndAuthority(file.getPath)
-    val relativePath = new URI(tablePathWithoutScheme.toString).relativize(new URI(partitionPathWithoutScheme.toString)).toString
-    CachingPath.concatPathUnsafe(CachingPath.concatPathUnsafe(tablePathWithoutScheme, PartitionPathEncodeUtils.escapePathName(relativePath)), filePathWithoutScheme.getName).toString
-  }
 }
 
 
@@ -231,4 +214,10 @@ object HoodieBootstrapRelation {
     checkState(combinedColumns.sorted == requiredDataColumns.sorted)
   }
 
+  def getFilePath(path: Path): String = {
+
+    //see MergeOnReadSnapshotRelation.getFilePath()
+    path.toUri.toString
+  }
+
 }