[spark] branch branch-3.2 updated: [SPARK-36670][FOLLOWUP][TEST] Remove brotli-codec dependency

[do...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new affd7a4  [SPARK-36670][FOLLOWUP][TEST] Remove brotli-codec dependency
affd7a4 is described below

commit affd7a4d47576be863c13df2c6e8068fcea3ba7c
Author: Gengliang Wang <ge...@apache.org>
AuthorDate: Tue Sep 21 10:57:20 2021 -0700

    [SPARK-36670][FOLLOWUP][TEST] Remove brotli-codec dependency
    
    ### What changes were proposed in this pull request?
    
    Remove `com.github.rdblue:brotli-codec:0.1.1` dependency.
    
    ### Why are the changes needed?
    
    As Stephen Coy pointed out in the dev list, we should not have `com.github.rdblue:brotli-codec:0.1.1` dependency which is not available on Maven Central. This is to avoid possible artifact changes on `Jitpack.io`.
    Also, the dependency is for tests only. I suggest that we remove it now to unblock the 3.2.0 release ASAP.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No
    
    ### How was this patch tested?
    
    GA tests.
    
    Closes #34059 from gengliangwang/removeDeps.
    
    Authored-by: Gengliang Wang <ge...@apache.org>
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
    (cherry picked from commit ba5708d944c5e38db750ad480668e524672ee963)
    Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
 pom.xml                                                          | 6 ------
 project/SparkBuild.scala                                         | 4 +---
 sql/core/pom.xml                                                 | 6 ------
 .../spark/sql/execution/datasources/FileSourceCodecSuite.scala   | 9 +++------
 4 files changed, 4 insertions(+), 21 deletions(-)

diff --git a/pom.xml b/pom.xml
index 5adbe8a..4a3bd71 100644
--- a/pom.xml
+++ b/pom.xml
@@ -300,12 +300,6 @@
         <enabled>false</enabled>
       </snapshots>
     </repository>
-    <repository>
-      <id>jitpack.io</id>
-      <url>https://jitpack.io</url>
-      <name>Jitpack.io repository</name>
-      <!-- needed for brotli-codec -->
-    </repository>
   </repositories>
   <pluginRepositories>
     <pluginRepository>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index b9068cc..b1531a6 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -274,9 +274,7 @@ object SparkBuild extends PomBuild {
       "gcs-maven-central-mirror" at "https://maven-central.storage-download.googleapis.com/maven2/",
       DefaultMavenRepository,
       Resolver.mavenLocal,
-      Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns),
-      // needed for brotli-codec
-      "jitpack.io" at "https://jitpack.io"
+      Resolver.file("ivyLocal", file(Path.userHome.absolutePath + "/.ivy2/local"))(Resolver.ivyStylePatterns)
     ),
     externalResolvers := resolvers.value,
     otherResolvers := SbtPomKeys.mvnLocalRepository(dotM2 => Seq(Resolver.file("dotM2", dotM2))).value,
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 3cdd9a4..e023377 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -184,12 +184,6 @@
       <artifactId>htmlunit-driver</artifactId>
       <scope>test</scope>
     </dependency>
-    <dependency>
-      <groupId>com.github.rdblue</groupId>
-      <artifactId>brotli-codec</artifactId>
-      <version>0.1.1</version>
-      <scope>test</scope>
-    </dependency>
   </dependencies>
   <build>
     <outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCodecSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCodecSuite.scala
index 3c226d6..ac1fd1c 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCodecSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/FileSourceCodecSuite.scala
@@ -56,13 +56,10 @@ class ParquetCodecSuite extends FileSourceCodecSuite {
   override def format: String = "parquet"
   override val codecConfigName: String = SQLConf.PARQUET_COMPRESSION.key
   // Exclude "lzo" because it is GPL-licenced so not included in Hadoop.
+  // Exclude "brotli" because the com.github.rdblue:brotli-codec dependency is not available
+  // on Maven Central.
   override protected def availableCodecs: Seq[String] =
-    if (System.getProperty("os.arch") == "aarch64") {
-      // Exclude "brotli" due to PARQUET-1975.
-      Seq("none", "uncompressed", "snappy", "lz4", "gzip", "zstd")
-    } else {
-      Seq("none", "uncompressed", "snappy", "lz4", "gzip", "brotli", "zstd")
-    }
+    Seq("none", "uncompressed", "snappy", "lz4", "gzip", "zstd")
 }
 
 class OrcCodecSuite extends FileSourceCodecSuite {

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org