(sedona) branch SEDONA-495 created (now 8d195b4b7)

[ji...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jiayu pushed a change to branch SEDONA-495
in repository https://gitbox.apache.org/repos/asf/sedona.git


      at 8d195b4b7 Fix

This branch includes the following new commits:

     new 8d195b4b7 Fix

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.

(sedona) 01/01: Fix

[ji...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jiayu pushed a commit to branch SEDONA-495
in repository https://gitbox.apache.org/repos/asf/sedona.git

commit 8d195b4b7e0f23d35334b7b131b33bf07681a9c7
Author: Jia Yu <ji...@apache.org>
AuthorDate: Wed Feb 14 17:07:43 2024 -0800

    Fix
---
 .../org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala | 5 ++---
 spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala | 2 +-
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala
index d7851b11d..eaa3d0295 100644
--- a/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala
+++ b/spark/common/src/main/scala/org/apache/spark/sql/sedona_sql/io/raster/RasterFileFormat.scala
@@ -20,7 +20,7 @@
 
 package org.apache.spark.sql.sedona_sql.io.raster
 
-import org.apache.hadoop.fs.{FileStatus, Path}
+import org.apache.hadoop.fs.{FileStatus, FileSystem, Path}
 import org.apache.hadoop.mapreduce.{Job, TaskAttemptContext}
 import org.apache.spark.sql.SparkSession
 import org.apache.spark.sql.catalyst.InternalRow
@@ -29,7 +29,6 @@ import org.apache.spark.sql.sources.DataSourceRegister
 import org.apache.spark.sql.types.StructType
 
 import java.io.IOException
-import java.nio.file.Paths
 import java.util.UUID
 
 private[spark] class RasterFileFormat extends FileFormat with DataSourceRegister {
@@ -82,7 +81,7 @@ private class RasterFileWriter(savePath: String,
                                 dataSchema: StructType,
                                 context: TaskAttemptContext) extends OutputWriter {
 
-  private val hfs = new Path(savePath).getFileSystem(context.getConfiguration)
+  private val hfs = FileSystem.newInstance(new Path(savePath).toUri, context.getConfiguration)
   private val rasterFieldIndex = if (rasterOptions.rasterField.isEmpty) getRasterFieldIndex else dataSchema.fieldIndex(rasterOptions.rasterField.get)
 
   private def getRasterFieldIndex: Int = {
diff --git a/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala b/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
index d5203e6a0..9cc755c48 100644
--- a/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
+++ b/spark/common/src/test/scala/org/apache/sedona/sql/rasterIOTest.scala
@@ -149,7 +149,7 @@ class rasterIOTest extends TestBaseScala with BeforeAndAfter with GivenWhenThen
     }
 
     it("should read geotiff using binary source and write geotiff back to hdfs using raster source") {
-      var rasterDf = sparkSession.read.format("binaryFile").load(rasterdatalocation)
+      var rasterDf = sparkSession.read.format("binaryFile").load(rasterdatalocation).repartition(3)
       val rasterCount = rasterDf.count()
       rasterDf.write.format("raster").mode(SaveMode.Overwrite).save(hdfsURI + "/raster-written")
       rasterDf = sparkSession.read.format("binaryFile").load(hdfsURI + "/raster-written/*")