You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by al...@apache.org on 2025/08/26 02:18:24 UTC
(kudu) 01/02: Upgrade to Spark3.5.5
This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch branch-1.18.x
in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 2bac7dd63039fb5a3ec3394b1cc7d7c150388085
Author: zchovan <zc...@cloudera.com>
AuthorDate: Thu Mar 6 17:26:17 2025 +0100
Upgrade to Spark3.5.5
Upgrading Spark dependency from 3.2.4 to 3.5.5, also as an addition
log4j was also updated from 2.17.1 to 2.20.0 for compatibility reasons.
The compareBinary method was copied from the Spark code as it's moved from
TypeUtils to ByteArray between Spark 3.2 and 3.3.
Change-Id: I341d84f7a7b32a26b8d9699b929cce1252d0ebad
Reviewed-on: http://gerrit.cloudera.org:8080/22604
Tested-by: Kudu Jenkins
Reviewed-by: Marton Greber <gr...@gmail.com>
Reviewed-by: Gabriella Lotz <lo...@gmail.com>
Reviewed-by: Attila Bukor <ab...@apache.org>
Reviewed-on: http://gerrit.cloudera.org:8080/23244
Reviewed-by: Zoltan Chovan <zc...@cloudera.com>
Tested-by: Abhishek Chennaka <ac...@cloudera.com>
Reviewed-by: Alexey Serbin <al...@apache.org>
---
java/gradle/dependencies.gradle | 2 +-
java/kudu-backup/build.gradle | 25 ++++++++++++++++------
java/kudu-spark-tools/build.gradle | 5 ++++-
java/kudu-spark/build.gradle | 5 ++++-
.../org/apache/kudu/spark/kudu/KuduContext.scala | 16 ++++++++++++--
5 files changed, 41 insertions(+), 12 deletions(-)
diff --git a/java/gradle/dependencies.gradle b/java/gradle/dependencies.gradle
index ade57f189..5bf766ad5 100755
--- a/java/gradle/dependencies.gradle
+++ b/java/gradle/dependencies.gradle
@@ -60,7 +60,7 @@ versions += [
scopt : "4.1.0",
slf4j : "1.7.36",
spark2 : "2.4.8",
- spark : "3.2.4",
+ spark : "3.5.5",
spotBugs : "4.1.1",
yetus : "0.13.0"
]
diff --git a/java/kudu-backup/build.gradle b/java/kudu-backup/build.gradle
index 12fe82495..6982370c5 100644
--- a/java/kudu-backup/build.gradle
+++ b/java/kudu-backup/build.gradle
@@ -59,25 +59,34 @@ dependencies {
// See: https://blog.gradle.org/introducing-compile-only-dependencies
// Note: compileUnshaded is extended from the compileOnly dependency scope.
testImplementation libs.scalaLibrary
- testImplementation libs.sparkAvro
- testImplementation libs.sparkCore
- testImplementation libs.sparkSql
+ testImplementation(libs.sparkAvro) {
+ exclude group: "org.apache.logging.log4j"
+ }
+ testImplementation( libs.sparkCore) {
+ exclude group: "org.apache.logging.log4j"
+ }
+ testImplementation(libs.sparkSql) {
+ exclude group: "org.apache.logging.log4j"
+ }
testImplementation libs.slf4jApi
}
shadowJar {
dependencies {
exclude(dependency(libs.jsr305))
- exclude(dependency("log4j:log4j:.*"))
+ exclude(dependency("log4j:log4j::.*"))
+ exclude(dependency("com.fasterxml.jackson.core::.*"))
exclude(dependency("org.apache.arrow::.*"))
exclude(dependency("org.apache.avro::.*"))
exclude(dependency("org.apache.curator::.*"))
exclude(dependency("org.apache.hadoop::.*"))
exclude(dependency("org.apache.ivy:ivy::.*"))
- exclude(dependency("org.apache.parquet:.*"))
+ exclude(dependency("org.apache.logging.log4j::.*"))
+ exclude(dependency("org.apache.parquet::.*"))
exclude(dependency("org.apache.spark::.*"))
exclude(dependency("org.codehaus.janino::.*"))
exclude(dependency("org.glassfish.jersey.core::.*"))
+ exclude(dependency("joda-time::.*"))
exclude(dependency("org.scala-lang::.*"))
exclude(dependency("org.xerial.snappy::.*"))
}
@@ -88,9 +97,11 @@ shadowJar {
exclude '**/*.html'
exclude '**/*.md'
exclude 'META-INF/services/**'
- exclude 'META-INF/versions/11/org/roaringbitmap/**'
exclude 'codegen/**'
exclude 'assets/**'
+ exclude 'org/threeten/**'
+ exclude 'org/apache/arrow/**'
+ exclude 'org/apache/orc/**'
exclude 'javax/**'
exclude 'org/jetbrains/**'
@@ -103,4 +114,4 @@ archivesBaseName = "kudu-backup${versions.sparkBase}_${versions.scalaBase}"
// kudu-backup has no public Javadoc.
javadoc {
enabled = false
-}
\ No newline at end of file
+}
diff --git a/java/kudu-spark-tools/build.gradle b/java/kudu-spark-tools/build.gradle
index 58bb21155..585c073fc 100644
--- a/java/kudu-spark-tools/build.gradle
+++ b/java/kudu-spark-tools/build.gradle
@@ -68,6 +68,7 @@ shadowJar {
exclude(dependency("org.apache.spark::.*"))
exclude(dependency("org.codehaus.janino::.*"))
exclude(dependency("org.glassfish.jersey.core::.*"))
+ exclude(dependency("joda-time::.*"))
exclude(dependency("org.scala-lang::.*"))
exclude(dependency("org.xerial.snappy::.*"))
}
@@ -78,9 +79,11 @@ shadowJar {
exclude '**/*.html'
exclude '**/*.md'
exclude 'META-INF/services/**'
- exclude 'META-INF/versions/11/org/roaringbitmap/**'
exclude 'codegen/**'
exclude 'javax/**'
+ exclude 'org/threeten/**'
+ exclude 'org/apache/arrow/**'
+ exclude 'org/apache/orc/**'
exclude 'org/jetbrains/**'
minimize()
diff --git a/java/kudu-spark/build.gradle b/java/kudu-spark/build.gradle
index e0f3d99d4..b190e185c 100644
--- a/java/kudu-spark/build.gradle
+++ b/java/kudu-spark/build.gradle
@@ -73,10 +73,13 @@ shadowJar {
exclude '**/*.html'
exclude '**/*.md'
exclude 'META-INF/services/**'
- exclude 'META-INF/versions/11/org/roaringbitmap/**'
exclude 'codegen/**'
+ exclude 'org/threeten/**'
+ exclude 'org/apache/arrow/**'
+ exclude 'org/apache/orc/**'
exclude 'javax/**'
exclude 'org/jetbrains/**'
+ exclude 'org/joda/time/tz/**'
minimize()
}
diff --git a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
index 67533fda5..4bc00d172 100644
--- a/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
+++ b/java/kudu-spark/src/main/scala/org/apache/kudu/spark/kudu/KuduContext.scala
@@ -29,7 +29,6 @@ import org.apache.hadoop.util.ShutdownHookManager
import org.apache.spark.Partitioner
import org.apache.spark.SparkContext
import org.apache.spark.rdd.RDD
-import org.apache.spark.sql.catalyst.util.TypeUtils
import org.apache.spark.sql.DataFrame
import org.apache.spark.sql.Row
import org.apache.spark.sql.types.DataType
@@ -445,6 +444,19 @@ class KuduContext(
log.info(s"completed $operation ops: duration histogram: $durationHistogram")
}
+ // This method is copied from Spark code as it's moved from TypeUtils to
+ // ByteArray between Spark 3.2 and 3.3
+ private def compareBinary(x: Array[Byte], y: Array[Byte]): Int = {
+ val limit = if (x.length <= y.length) x.length else y.length
+ var i = 0
+ while (i < limit) {
+ val res = (x(i) & 0xff) - (y(i) & 0xff)
+ if (res != 0) return res
+ i += 1
+ }
+ x.length - y.length
+ }
+
private[spark] def repartitionRows(
rdd: RDD[Row],
tableName: String,
@@ -475,7 +487,7 @@ class KuduContext(
// to enable rdd sorting functions below.
implicit val byteArrayOrdering: Ordering[Array[Byte]] = new Ordering[Array[Byte]] {
def compare(x: Array[Byte], y: Array[Byte]): Int = {
- TypeUtils.compareBinary(x, y)
+ compareBinary(x, y)
}
}