You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by fc...@apache.org on 2023/08/16 02:43:08 UTC
[incubator-celeborn] branch main updated: [CELEBORN-884][BUILD] Consolidate all dependencies into a global object `Dependencies`
This is an automated email from the ASF dual-hosted git repository.
fchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 6af3b5050 [CELEBORN-884][BUILD] Consolidate all dependencies into a global object `Dependencies`
6af3b5050 is described below
commit 6af3b50508cdd227e620ffe0eb3cc4164797b28c
Author: Fu Chen <cf...@gmail.com>
AuthorDate: Wed Aug 16 10:42:59 2023 +0800
[CELEBORN-884][BUILD] Consolidate all dependencies into a global object `Dependencies`
### What changes were proposed in this pull request?
As title
### Why are the changes needed?
consolidate all sbt dependencies into a global object `Dependencies`, similar to Maven's dependencyManagement, to improve dependency management.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Pass GA
Closes #1802 from cfmcgrady/sbt-dependencies.
Authored-by: Fu Chen <cf...@gmail.com>
Signed-off-by: Fu Chen <cf...@gmail.com>
---
project/CelebornBuild.scala | 205 +++++++++++++++++++++++++++-----------------
1 file changed, 126 insertions(+), 79 deletions(-)
diff --git a/project/CelebornBuild.scala b/project/CelebornBuild.scala
index c27f2293f..06e2b831f 100644
--- a/project/CelebornBuild.scala
+++ b/project/CelebornBuild.scala
@@ -30,19 +30,7 @@ import Utils._
import CelebornCommonSettings._
// import sbt.Keys.streams
-object CelebornCommonSettings {
-
- // Scala versions
- val SCALA_2_11_12 = "2.11.12"
- val SCALA_2_12_10 = "2.12.10"
- val SCALA_2_12_15 = "2.12.15"
- val SCALA_2_12_17 = "2.12.17"
- val scala213 = "2.13.5"
- val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15, SCALA_2_12_17, scala213)
-
- val DEFAULT_SCALA_VERSION = SCALA_2_12_15
-
- val projectScalaVersion = defaultScalaVersion()
+object Dependencies {
val zstdJniVersion = sparkClientProjects.map(_.zstdJniVersion).getOrElse("1.5.2-1")
val lz4JavaVersion = sparkClientProjects.map(_.lz4JavaVersion).getOrElse("1.8.0")
@@ -55,13 +43,18 @@ object CelebornCommonSettings {
val guavaVersion = "14.0.1"
val hadoopVersion = "3.2.4"
val javaxServletVersion = "3.1.0"
+ val junitInterfaceVersion = "0.13.3"
+ // don't forget update `junitInterfaceVersion` when we upgrade junit
+ val junitVersion = "4.13.2"
val leveldbJniVersion = "1.8"
val log4j2Version = "2.17.2"
val metricsVersion = "3.2.6"
- val scalatestMockitoVersion = "1.17.14"
+ val mockitoVersion = "4.11.0"
val nettyVersion = "4.1.93.Final"
val ratisVersion = "2.5.1"
val roaringBitmapVersion = "0.9.32"
+ val scalatestMockitoVersion = "1.17.14"
+ val scalatestVersion = "3.2.16"
val slf4jVersion = "1.7.36"
val snakeyamlVersion = "1.33"
@@ -69,6 +62,60 @@ object CelebornCommonSettings {
val protocVersion = "3.19.2"
val protoVersion = "3.19.2"
+ val commonsCrypto = "org.apache.commons" % "commons-crypto" % commonsCryptoVersion
+ val commonsIo = "commons-io" % "commons-io" % commonsIoVersion
+ val commonsLang3 = "org.apache.commons" % "commons-lang3" % commonsLang3Version
+ val findbugsJsr305 = "com.google.code.findbugs" % "jsr305" % findbugsVersion
+ val guava = "com.google.guava" % "guava" % guavaVersion
+ val hadoopClientApi = "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion
+ val hadoopClientRuntime = "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion
+ val ioDropwizardMetricsCore = "io.dropwizard.metrics" % "metrics-core" % metricsVersion
+ val ioDropwizardMetricsGraphite = "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion
+ val ioDropwizardMetricsJvm = "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion
+ val ioNetty = "io.netty" % "netty-all" % nettyVersion
+ val javaxServletApi = "javax.servlet" % "javax.servlet-api" % javaxServletVersion
+ val leveldbJniAll = "org.fusesource.leveldbjni" % "leveldbjni-all" % leveldbJniVersion
+ val log4j12Api = "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version
+ val log4jSlf4jImpl = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version
+ val lz4Java = "org.lz4" % "lz4-java" % lz4JavaVersion
+ val protobufJava = "com.google.protobuf" % "protobuf-java" % protoVersion
+ val ratisClient = "org.apache.ratis" % "ratis-client" % ratisVersion
+ val ratisCommon = "org.apache.ratis" % "ratis-common" % ratisVersion
+ val ratisGrpc = "org.apache.ratis" % "ratis-grpc" % ratisVersion
+ val ratisNetty = "org.apache.ratis" % "ratis-netty" % ratisVersion
+ val ratisServer = "org.apache.ratis" % "ratis-server" % ratisVersion
+ val ratisShell = "org.apache.ratis" % "ratis-shell" % ratisVersion
+ val roaringBitmap = "org.roaringbitmap" % "RoaringBitmap" % roaringBitmapVersion
+ val scalaReflect = "org.scala-lang" % "scala-reflect" % projectScalaVersion
+ val slf4jApi = "org.slf4j" % "slf4j-api" % slf4jVersion
+ val slf4jJulToSlf4j = "org.slf4j" % "jul-to-slf4j" % slf4jVersion
+ val slf4jJclOverSlf4j = "org.slf4j" % "jcl-over-slf4j" % slf4jVersion
+ val snakeyaml = "org.yaml" % "snakeyaml" % snakeyamlVersion
+ val zstdJni = "com.github.luben" % "zstd-jni" % zstdJniVersion
+
+ // Test dependencies
+ // https://www.scala-sbt.org/1.x/docs/Testing.html
+ val junitInterface = "com.github.sbt" % "junit-interface" % junitInterfaceVersion
+ val junit = "junit" % "junit" % junitVersion
+ val mockitoCore = "org.mockito" % "mockito-core" % mockitoVersion
+ val scalatestMockito = "org.mockito" %% "mockito-scala-scalatest" % scalatestMockitoVersion
+ val scalatest = "org.scalatest" %% "scalatest" % scalatestVersion
+}
+
+object CelebornCommonSettings {
+
+ // Scala versions
+ val SCALA_2_11_12 = "2.11.12"
+ val SCALA_2_12_10 = "2.12.10"
+ val SCALA_2_12_15 = "2.12.15"
+ val SCALA_2_12_17 = "2.12.17"
+ val scala213 = "2.13.5"
+ val ALL_SCALA_VERSIONS = Seq(SCALA_2_11_12, SCALA_2_12_10, SCALA_2_12_15, SCALA_2_12_17, scala213)
+
+ val DEFAULT_SCALA_VERSION = SCALA_2_12_15
+
+ val projectScalaVersion = defaultScalaVersion()
+
scalaVersion := projectScalaVersion
autoScalaLibrary := false
@@ -128,18 +175,18 @@ object CelebornCommonSettings {
lazy val protoSettings = Seq(
// Setting version for the protobuf compiler
- PB.protocVersion := protocVersion,
+ PB.protocVersion := Dependencies.protocVersion,
// set proto sources path
Compile / PB.protoSources := Seq(sourceDirectory.value / "main" / "proto"),
Compile / PB.targets := Seq(PB.gens.java -> (Compile / sourceManaged).value)
)
lazy val commonUnitTestDependencies = Seq(
- "org.mockito" % "mockito-core" % "4.11.0" % "test",
- "org.scalatest" %% "scalatest" % "3.2.16" % "test",
- "junit" % "junit" % "4.13.2" % "test",
+ Dependencies.mockitoCore % "test",
+ Dependencies.scalatest % "test",
+ Dependencies.junit % "test",
// https://www.scala-sbt.org/1.x/docs/Testing.html
- "com.github.sbt" % "junit-interface" % "0.13.3" % "test")
+ Dependencies.junitInterface % "test")
}
object CelebornBuild extends sbt.internal.BuildDef {
@@ -218,29 +265,29 @@ object CelebornCommon {
commonSettings,
protoSettings,
libraryDependencies ++= Seq(
- "com.google.protobuf" % "protobuf-java" % protoVersion % "protobuf",
- "com.google.code.findbugs" % "jsr305" % findbugsVersion,
- "com.google.guava" % "guava" % guavaVersion,
- "commons-io" % "commons-io" % commonsIoVersion,
- "io.dropwizard.metrics" % "metrics-core" % metricsVersion,
- "io.dropwizard.metrics" % "metrics-graphite" % metricsVersion,
- "io.dropwizard.metrics" % "metrics-jvm" % metricsVersion,
- "io.netty" % "netty-all" % nettyVersion,
- "org.apache.commons" % "commons-crypto" % commonsCryptoVersion,
- "org.apache.commons" % "commons-lang3" % commonsLang3Version,
- "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion,
- "org.apache.hadoop" % "hadoop-client-runtime" % hadoopVersion,
- "org.apache.ratis" % "ratis-client" % ratisVersion,
- "org.apache.ratis" % "ratis-common" % ratisVersion,
- "org.fusesource.leveldbjni" % "leveldbjni-all" % leveldbJniVersion,
- "org.roaringbitmap" % "RoaringBitmap" % roaringBitmapVersion,
- "org.scala-lang" % "scala-reflect" % scalaVersion.value,
- "org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
- "org.slf4j" % "jul-to-slf4j" % slf4jVersion,
- "org.slf4j" % "slf4j-api" % slf4jVersion,
- "org.yaml" % "snakeyaml" % snakeyamlVersion,
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version % "test",
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version % "test"
+ Dependencies.protobufJava % "protobuf",
+ Dependencies.findbugsJsr305,
+ Dependencies.guava,
+ Dependencies.commonsIo,
+ Dependencies.ioDropwizardMetricsCore,
+ Dependencies.ioDropwizardMetricsGraphite,
+ Dependencies.ioDropwizardMetricsJvm,
+ Dependencies.ioNetty,
+ Dependencies.commonsCrypto,
+ Dependencies.commonsLang3,
+ Dependencies.hadoopClientApi,
+ Dependencies.hadoopClientRuntime,
+ Dependencies.ratisClient,
+ Dependencies.ratisCommon,
+ Dependencies.leveldbJniAll,
+ Dependencies.roaringBitmap,
+ Dependencies.scalaReflect,
+ Dependencies.slf4jJclOverSlf4j,
+ Dependencies.slf4jJulToSlf4j,
+ Dependencies.slf4jApi,
+ Dependencies.snakeyaml,
+ Dependencies.log4jSlf4jImpl % "test",
+ Dependencies.log4j12Api % "test"
) ++ commonUnitTestDependencies,
Compile / sourceGenerators += Def.task {
@@ -274,13 +321,13 @@ object CelebornClient {
.settings (
commonSettings,
libraryDependencies ++= Seq(
- "io.netty" % "netty-all" % nettyVersion,
- "com.google.guava" % "guava" % guavaVersion,
- "org.lz4" % "lz4-java" % lz4JavaVersion,
- "com.github.luben" % "zstd-jni" % zstdJniVersion,
- "org.apache.commons" % "commons-lang3" % commonsLang3Version,
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version % "test",
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version % "test"
+ Dependencies.ioNetty,
+ Dependencies.guava,
+ Dependencies.lz4Java,
+ Dependencies.zstdJni,
+ Dependencies.commonsLang3,
+ Dependencies.log4jSlf4jImpl % "test",
+ Dependencies.log4j12Api % "test"
) ++ commonUnitTestDependencies
)
}
@@ -291,14 +338,14 @@ object CelebornService {
.settings (
commonSettings,
libraryDependencies ++= Seq(
- "com.google.code.findbugs" % "jsr305" % findbugsVersion,
- "commons-io" % "commons-io" % commonsIoVersion,
- "io.netty" % "netty-all" % nettyVersion,
- "javax.servlet" % "javax.servlet-api" % javaxServletVersion,
- "org.apache.commons" % "commons-crypto" % commonsCryptoVersion,
- "org.slf4j" % "slf4j-api" % slf4jVersion,
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version % "test",
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version % "test"
+ Dependencies.findbugsJsr305,
+ Dependencies.commonsIo,
+ Dependencies.ioNetty,
+ Dependencies.javaxServletApi,
+ Dependencies.commonsCrypto,
+ Dependencies.slf4jApi,
+ Dependencies.log4jSlf4jImpl % "test",
+ Dependencies.log4j12Api % "test"
) ++ commonUnitTestDependencies
)
}
@@ -310,18 +357,18 @@ object CelebornMaster {
commonSettings,
protoSettings,
libraryDependencies ++= Seq(
- "com.google.guava" % "guava" % guavaVersion,
- "com.google.protobuf" % "protobuf-java" % protoVersion,
- "io.netty" % "netty-all" % nettyVersion,
- "org.apache.hadoop" % "hadoop-client-api" % hadoopVersion,
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version,
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version,
- "org.apache.ratis" % "ratis-client" % ratisVersion,
- "org.apache.ratis" % "ratis-common" % ratisVersion,
- "org.apache.ratis" % "ratis-grpc" % ratisVersion,
- "org.apache.ratis" % "ratis-netty" % ratisVersion,
- "org.apache.ratis" % "ratis-server" % ratisVersion,
- "org.apache.ratis" % "ratis-shell" % ratisVersion,
+ Dependencies.guava,
+ Dependencies.protobufJava,
+ Dependencies.ioNetty,
+ Dependencies.hadoopClientApi,
+ Dependencies.log4j12Api,
+ Dependencies.log4jSlf4jImpl,
+ Dependencies.ratisClient,
+ Dependencies.ratisCommon,
+ Dependencies.ratisGrpc,
+ Dependencies.ratisNetty,
+ Dependencies.ratisServer,
+ Dependencies.ratisShell
) ++ commonUnitTestDependencies
)
}
@@ -334,14 +381,14 @@ object CelebornWorker {
.settings (
commonSettings,
libraryDependencies ++= Seq(
- "com.google.guava" % "guava" % guavaVersion,
- "commons-io" % "commons-io" % commonsIoVersion,
- "io.netty" % "netty-all" % nettyVersion,
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version,
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version,
- "org.fusesource.leveldbjni" % "leveldbjni-all" % leveldbJniVersion,
- "org.roaringbitmap" % "RoaringBitmap" % roaringBitmapVersion,
- "org.mockito" %% "mockito-scala-scalatest" % scalatestMockitoVersion % "test",
+ Dependencies.guava,
+ Dependencies.commonsIo,
+ Dependencies.ioNetty,
+ Dependencies.log4j12Api,
+ Dependencies.log4jSlf4jImpl,
+ Dependencies.leveldbJniAll,
+ Dependencies.roaringBitmap,
+ Dependencies.scalatestMockito % "test"
) ++ commonUnitTestDependencies
)
}
@@ -690,8 +737,8 @@ trait FlinkClientProjects {
libraryDependencies ++= Seq(
"org.apache.flink" % "flink-runtime" % flinkVersion % "provided",
- "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4j2Version % "test",
- "org.apache.logging.log4j" % "log4j-1.2-api" % log4j2Version % "test"
+ Dependencies.log4jSlf4jImpl % "test",
+ Dependencies.log4j12Api % "test"
) ++ commonUnitTestDependencies
)
}