You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2025/08/04 01:34:39 UTC
(spark) branch master updated: [SPARK-53088][BUILD] Remove `scala-collection-compat` dependency
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new f3a08e7036a7 [SPARK-53088][BUILD] Remove `scala-collection-compat` dependency
f3a08e7036a7 is described below
commit f3a08e7036a7dd2a2ced08f88a7d56619a44fa27
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Sun Aug 3 18:34:27 2025 -0700
[SPARK-53088][BUILD] Remove `scala-collection-compat` dependency
### What changes were proposed in this pull request?
This PR aims to remove `scala-collection-compat` dependency at Apache Spark 4.1.0.
### Why are the changes needed?
Apache Spark 4.0 has a transitive dependency `scala-collection-compat` v2.7.0 (2022-03-22) which is a little old.
https://github.com/apache/spark/blob/21fd325362658e284218118803cad532f6d8cc53/dev/deps/spark-deps-hadoop-3-hive-2.3#L252
- https://github.com/scala/scala-collection-compat/releases/tag/v2.13.0 (2025-01-27)
- https://github.com/scala/scala-collection-compat/releases/tag/v2.12.0 (2024-04-15)
- https://github.com/scala/scala-collection-compat/releases/tag/v2.11.0 (2023-06-15)
- https://github.com/scala/scala-collection-compat/releases/tag/v2.10.0 (2024-04-24)
- https://github.com/scala/scala-collection-compat/releases/tag/v2.9.0 (2022-11-28)
- https://github.com/scala/scala-collection-compat/releases/tag/v2.8.0 (2022-07-08)
However, what we actually use is only the following wrapper. We had better simplify this dependency relationship.
- https://github.com/scala/scala-collection-compat
> The Scala 2.13 and Scala 3 versions consist only of an empty scala.collection.compat package object, so import scala.collection.compat._ won't cause an error in cross-compiled code.
```scala
package scala.collection
package object compat {
type Factory[-A, +C] = scala.collection.Factory[A, C]
val Factory = scala.collection.Factory
type BuildFrom[-From, -A, +C] = scala.collection.BuildFrom[From, A, C]
val BuildFrom = scala.collection.BuildFrom
type IterableOnce[+X] = scala.collection.IterableOnce[X]
val IterableOnce = scala.collection.IterableOnce
}
```
### Does this PR introduce _any_ user-facing change?
No behavior changes.
### How was this patch tested?
Pass the CIs.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #51802 from dongjoon-hyun/SPARK-53088-2.
Authored-by: Dongjoon Hyun <do...@apache.org>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
LICENSE | 1 +
LICENSE-binary | 1 -
dev/.rat-excludes | 1 +
dev/deps/spark-deps-hadoop-3-hive-2.3 | 1 -
.../scala/scala/collection/compat/package.scala | 26 ++++++++++++++++++++++
pom.xml | 4 ++++
scalastyle-config.xml | 2 +-
7 files changed, 33 insertions(+), 3 deletions(-)
diff --git a/LICENSE b/LICENSE
index 9b1e96a44a58..ef8192ec38d0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -217,6 +217,7 @@ core/src/main/resources/org/apache/spark/ui/static/vis*
connector/spark-ganglia-lgpl/src/main/java/com/codahale/metrics/ganglia/GangliaReporter.java
core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.min.js
core/src/main/resources/org/apache/spark/ui/static/d3-flamegraph.css
+mllib-local/src/main/scala/scala/collection/compat/package.scala
Python Software Foundation License
----------------------------------
diff --git a/LICENSE-binary b/LICENSE-binary
index 61a6a830dad3..6f6bf9db5448 100644
--- a/LICENSE-binary
+++ b/LICENSE-binary
@@ -398,7 +398,6 @@ org.rocksdb:rocksdbjni
org.scala-lang:scala-compiler
org.scala-lang:scala-library
org.scala-lang:scala-reflect
-org.scala-lang.modules:scala-collection-compat_2.13
org.scala-lang.modules:scala-parallel-collections_2.13
org.scala-lang.modules:scala-parser-combinators_2.13
org.scala-lang.modules:scala-xml_2.13
diff --git a/dev/.rat-excludes b/dev/.rat-excludes
index 5084d0b6905a..f976b03b49c9 100644
--- a/dev/.rat-excludes
+++ b/dev/.rat-excludes
@@ -132,6 +132,7 @@ SqlBaseLexer.tokens
.*\.proto.bin
LimitedInputStream.java
TimSort.java
+mllib-local/src/main/scala/scala/collection/compat/package.scala
xml-resources/*
loose_version.py
people.xml
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 020a2c4f6cfe..78025a4f934c 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -249,7 +249,6 @@ pickle/1.5//pickle-1.5.jar
py4j/0.10.9.9//py4j-0.10.9.9.jar
remotetea-oncrpc/1.1.2//remotetea-oncrpc-1.1.2.jar
rocksdbjni/9.8.4//rocksdbjni-9.8.4.jar
-scala-collection-compat_2.13/2.7.0//scala-collection-compat_2.13-2.7.0.jar
scala-compiler/2.13.16//scala-compiler-2.13.16.jar
scala-library/2.13.16//scala-library-2.13.16.jar
scala-parallel-collections_2.13/1.2.0//scala-parallel-collections_2.13-1.2.0.jar
diff --git a/mllib-local/src/main/scala/scala/collection/compat/package.scala b/mllib-local/src/main/scala/scala/collection/compat/package.scala
new file mode 100644
index 000000000000..8ce1cfff886f
--- /dev/null
+++ b/mllib-local/src/main/scala/scala/collection/compat/package.scala
@@ -0,0 +1,26 @@
+// scalastyle:off license
+/*
+ * Scala (https://www.scala-lang.org)
+ *
+ * Copyright EPFL and Lightbend, Inc. dba Akka
+ *
+ * Licensed under Apache License 2.0
+ * (http://www.apache.org/licenses/LICENSE-2.0).
+ *
+ * See the NOTICE file distributed with this work for
+ * additional information regarding copyright ownership.
+ */
+// scalastyle:on license
+
+package scala.collection
+
+package object compat {
+ type Factory[-A, +C] = scala.collection.Factory[A, C]
+ val Factory = scala.collection.Factory
+
+ type BuildFrom[-From, -A, +C] = scala.collection.BuildFrom[From, A, C]
+ val BuildFrom = scala.collection.BuildFrom
+
+ type IterableOnce[+X] = scala.collection.IterableOnce[X]
+ val IterableOnce = scala.collection.IterableOnce
+}
diff --git a/pom.xml b/pom.xml
index bd9e9650a6f1..4b5eeeeb6cd3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1150,6 +1150,10 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-math3</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.scala-lang.modules</groupId>
+ <artifactId>scala-collection-compat_${scala.binary.version}</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/scalastyle-config.xml b/scalastyle-config.xml
index d0e15998c36a..7b6b65a22810 100644
--- a/scalastyle-config.xml
+++ b/scalastyle-config.xml
@@ -46,7 +46,7 @@ This file is divided into 3 sections:
<check level="error" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
- <check level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
+ <check customId="license" level="error" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
<parameters>
<parameter name="header"><![CDATA[/*
* Licensed to the Apache Software Foundation (ASF) under one or more
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org