You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bigtop.apache.org by iw...@apache.org on 2021/12/24 05:37:08 UTC
[bigtop] branch master updated: BIGTOP-3602. Bump Spark to 3.1.2. (#833)
This is an automated email from the ASF dual-hosted git repository.
iwasakims pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/bigtop.git
The following commit(s) were added to refs/heads/master by this push:
new 3a54460 BIGTOP-3602. Bump Spark to 3.1.2. (#833)
3a54460 is described below
commit 3a544606baf5e20fbcebdc40391b8cd6c4c78d78
Author: Masatake Iwasaki <iw...@apache.org>
AuthorDate: Fri Dec 24 14:36:59 2021 +0900
BIGTOP-3602. Bump Spark to 3.1.2. (#833)
---
.../src/common/spark/patch0-SPARK-35420.diff | 140 +++++++++++++++++++++
.../src/common/spark/patch2-snappy-java-1.1.8.diff | 52 --------
bigtop.bom | 2 +-
3 files changed, 141 insertions(+), 53 deletions(-)
diff --git a/bigtop-packages/src/common/spark/patch0-SPARK-35420.diff b/bigtop-packages/src/common/spark/patch0-SPARK-35420.diff
new file mode 100644
index 0000000..7ccf8e6
--- /dev/null
+++ b/bigtop-packages/src/common/spark/patch0-SPARK-35420.diff
@@ -0,0 +1,140 @@
+commit b4348b7e562cded183aab6de1521ae364cc7fd14
+Author: Kousuke Saruta <sa...@oss.nttdata.com>
+Date: Mon May 17 21:46:35 2021 +0900
+
+ [SPARK-35420][BUILD] Replace the usage of toStringHelper with ToStringBuilder
+
+ ### What changes were proposed in this pull request?
+
+ This PR replaces `toStringHelper`, an API which breaks in Guava 27.
+
+ ### Why are the changes needed?
+
+ SPARK-30272 (#26911) removed usages which breaks in Guava 27 but `toStringHelper` is instroduced again.
+
+ ### Does this PR introduce _any_ user-facing change?
+
+ No.
+
+ ### How was this patch tested?
+
+ Build successfully finished with the following command.
+ ```
+ build/sbt -Dguava.version=27.0-jre -Phive -Phive-thriftserver -Pyarn -Pmesos -Pkubernetes -Phadoop-cloud -Pdocker-integration-tests -Pkubernetes-integration-tests -Pkinesis-asl -Pspark-ganglia-lgpl package
+ ```
+
+ Closes #32567 from sarutak/remove-old-guava-usage.
+
+ Authored-by: Kousuke Saruta <sa...@oss.nttdata.com>
+ Signed-off-by: Kousuke Saruta <sa...@oss.nttdata.com>
+
+diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
+index 9363efc58d..1ac33cdefe 100644
+--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
++++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java
+@@ -47,6 +47,8 @@ import com.google.common.cache.Weigher;
+ import com.google.common.collect.Maps;
+ import com.google.common.primitives.Ints;
+ import com.google.common.primitives.Longs;
++import org.apache.commons.lang3.builder.ToStringBuilder;
++import org.apache.commons.lang3.builder.ToStringStyle;
+ import org.roaringbitmap.RoaringBitmap;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+@@ -770,9 +772,9 @@ public class RemoteBlockPushResolver implements MergedShuffleFileManager {
+
+ @Override
+ public String toString() {
+- return Objects.toStringHelper(this)
+- .add("appId", appId)
+- .add("shuffleId", shuffleId)
++ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
++ .append("appId", appId)
++ .append("shuffleId", shuffleId)
+ .toString();
+ }
+ }
+diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java
+index 8427837d15..31efbb727b 100644
+--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java
++++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java
+@@ -19,6 +19,8 @@ package org.apache.spark.network.shuffle.protocol;
+
+ import com.google.common.base.Objects;
+ import io.netty.buffer.ByteBuf;
++import org.apache.commons.lang3.builder.ToStringBuilder;
++import org.apache.commons.lang3.builder.ToStringStyle;
+
+ import org.apache.spark.network.protocol.Encoders;
+
+@@ -51,9 +53,9 @@ public class FinalizeShuffleMerge extends BlockTransferMessage {
+
+ @Override
+ public String toString() {
+- return Objects.toStringHelper(this)
+- .add("appId", appId)
+- .add("shuffleId", shuffleId)
++ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
++ .append("appId", appId)
++ .append("shuffleId", shuffleId)
+ .toString();
+ }
+
+diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java
+index d506d9eb2b..142ab73e79 100644
+--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java
++++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java
+@@ -21,6 +21,8 @@ import java.util.Arrays;
+
+ import com.google.common.base.Objects;
+ import io.netty.buffer.ByteBuf;
++import org.apache.commons.lang3.builder.ToStringBuilder;
++import org.apache.commons.lang3.builder.ToStringStyle;
+ import org.roaringbitmap.RoaringBitmap;
+
+ import org.apache.spark.network.protocol.Encoders;
+@@ -76,9 +78,9 @@ public class MergeStatuses extends BlockTransferMessage {
+
+ @Override
+ public String toString() {
+- return Objects.toStringHelper(this)
+- .add("shuffleId", shuffleId)
+- .add("reduceId size", reduceIds.length)
++ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
++ .append("shuffleId", shuffleId)
++ .append("reduceId size", reduceIds.length)
+ .toString();
+ }
+
+diff --git a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java
+index 83fc7b23ac..559f88fc4e 100644
+--- a/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java
++++ b/common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java
+@@ -19,6 +19,8 @@ package org.apache.spark.network.shuffle.protocol;
+
+ import com.google.common.base.Objects;
+ import io.netty.buffer.ByteBuf;
++import org.apache.commons.lang3.builder.ToStringBuilder;
++import org.apache.commons.lang3.builder.ToStringStyle;
+
+ import org.apache.spark.network.protocol.Encoders;
+
+@@ -59,12 +61,12 @@ public class PushBlockStream extends BlockTransferMessage {
+
+ @Override
+ public String toString() {
+- return Objects.toStringHelper(this)
+- .add("appId", appId)
+- .add("shuffleId", shuffleId)
+- .add("mapIndex", mapIndex)
+- .add("reduceId", reduceId)
+- .add("index", index)
++ return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
++ .append("appId", appId)
++ .append("shuffleId", shuffleId)
++ .append("mapIndex", mapIndex)
++ .append("reduceId", reduceId)
++ .append("index", index)
+ .toString();
+ }
+
diff --git a/bigtop-packages/src/common/spark/patch2-snappy-java-1.1.8.diff b/bigtop-packages/src/common/spark/patch2-snappy-java-1.1.8.diff
deleted file mode 100644
index 46befc3..0000000
--- a/bigtop-packages/src/common/spark/patch2-snappy-java-1.1.8.diff
+++ /dev/null
@@ -1,52 +0,0 @@
-diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-1.2 b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
-index e32ea64563..53488f372b 100644
---- a/dev/deps/spark-deps-hadoop-2.7-hive-1.2
-+++ b/dev/deps/spark-deps-hadoop-2.7-hive-1.2
-@@ -188,7 +188,7 @@ shims/0.7.45//shims-0.7.45.jar
- slf4j-api/1.7.30//slf4j-api-1.7.30.jar
- slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
- snakeyaml/1.24//snakeyaml-1.24.jar
--snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
-+snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
- snappy/0.2//snappy-0.2.jar
- spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
- spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
-diff --git a/dev/deps/spark-deps-hadoop-2.7-hive-2.3 b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
-index 168d6196fe..9073cf40fb 100644
---- a/dev/deps/spark-deps-hadoop-2.7-hive-2.3
-+++ b/dev/deps/spark-deps-hadoop-2.7-hive-2.3
-@@ -202,7 +202,7 @@ shims/0.7.45//shims-0.7.45.jar
- slf4j-api/1.7.30//slf4j-api-1.7.30.jar
- slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
- snakeyaml/1.24//snakeyaml-1.24.jar
--snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
-+snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
- spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
- spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
- spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
-diff --git a/dev/deps/spark-deps-hadoop-3.2-hive-2.3 b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
-index d730b4a0e1..f8ea686c7f 100644
---- a/dev/deps/spark-deps-hadoop-3.2-hive-2.3
-+++ b/dev/deps/spark-deps-hadoop-3.2-hive-2.3
-@@ -217,7 +217,7 @@ shims/0.7.45//shims-0.7.45.jar
- slf4j-api/1.7.30//slf4j-api-1.7.30.jar
- slf4j-log4j12/1.7.30//slf4j-log4j12-1.7.30.jar
- snakeyaml/1.24//snakeyaml-1.24.jar
--snappy-java/1.1.7.5//snappy-java-1.1.7.5.jar
-+snappy-java/1.1.8.4//snappy-java-1.1.8.4.jar
- spire-macros_2.12/0.17.0-M1//spire-macros_2.12-0.17.0-M1.jar
- spire-platform_2.12/0.17.0-M1//spire-platform_2.12-0.17.0-M1.jar
- spire-util_2.12/0.17.0-M1//spire-util_2.12-0.17.0-M1.jar
-diff --git a/pom.xml b/pom.xml
-index 7c60f8f99b..0b5b51e7cc 100644
---- a/pom.xml
-+++ b/pom.xml
-@@ -173,7 +173,7 @@
- <scalafmt.skip>true</scalafmt.skip>
- <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
- <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-- <snappy.version>1.1.7.5</snappy.version>
-+ <snappy.version>1.1.8.4</snappy.version>
- <netlib.java.version>1.1.2</netlib.java.version>
- <commons-codec.version>1.10</commons-codec.version>
- <commons-io.version>2.4</commons-io.version>
diff --git a/bigtop.bom b/bigtop.bom
index 28f8192..58626e8 100644
--- a/bigtop.bom
+++ b/bigtop.bom
@@ -222,7 +222,7 @@ bigtop {
* when upgrading spark version.
* See comments in [bigtop-packages/src/common/spark/install_spark.sh] for details.
*/
- version { base = '3.0.1'; pkg = base; release = 1 }
+ version { base = '3.1.2'; pkg = base; release = 1 }
tarball { destination = "$name-${version.base}.tar.gz"
source = "$name-${version.base}.tgz" }
url { download_path = "/$name/$name-${version.base}"