You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ar...@apache.org on 2023/07/20 12:06:04 UTC

[beam] branch master updated: Bump Spark3 version to 3.2.2 (#23805)

This is an automated email from the ASF dual-hosted git repository.

aromanenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 0cd9b90467b Bump Spark3 version to 3.2.2 (#23805)
0cd9b90467b is described below

commit 0cd9b90467b43f37898b077687b931db1634c842
Author: Alexey Romanenko <33...@users.noreply.github.com>
AuthorDate: Thu Jul 20 14:05:58 2023 +0200

    Bump Spark3 version to 3.2.2 (#23805)
    
    * Bump Spark3 version to 3.2.0
    
    * [23804] Add Spark 3.2.0 constructors in EncoderFactory
    
    * Update Hadoop deps
    
    * Bump Spark3 version to 3.2.2
    
    * Add Spark 3.1.1 for compatibility testing
    
    * Update CHANGES.md on Spark version bump
    
    * Fix whitespace check
    
    * Add Spark 3.1.2 for compatibility testing
    
    * Address review comments
---
 CHANGES.md                                                    |  1 +
 .../groovy/org/apache/beam/gradle/BeamModulePlugin.groovy     |  2 +-
 runners/spark/3/build.gradle                                  |  2 +-
 runners/spark/3/job-server/build.gradle                       | 11 +----------
 .../translation/helpers/EncoderFactory.java                   | 10 ++++++++++
 runners/spark/spark_runner.gradle                             |  2 +-
 website/www/site/content/en/documentation/runners/spark.md    |  8 ++++----
 7 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/CHANGES.md b/CHANGES.md
index 7ed75c7ee05..dff482b6b74 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -56,6 +56,7 @@
 
 * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
 * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)).
+* Spark 3.2.2 is used as default version for Spark runner ([#23804](https://github.com/apache/beam/issues/23804)).
 
 ## I/Os
 
diff --git a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
index 9637f17dbf8..852edfe62e4 100644
--- a/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
+++ b/buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy
@@ -559,7 +559,7 @@ class BeamModulePlugin implements Plugin<Project> {
     def singlestore_jdbc_version = "1.1.4"
     def slf4j_version = "1.7.30"
     def spark2_version = "2.4.8"
-    def spark3_version = "3.1.2"
+    def spark3_version = "3.2.2"
     def spotbugs_version = "4.0.6"
     def testcontainers_version = "1.17.3"
     def arrow_version = "5.0.0"
diff --git a/runners/spark/3/build.gradle b/runners/spark/3/build.gradle
index 494d367131b..cb34a1fd972 100644
--- a/runners/spark/3/build.gradle
+++ b/runners/spark/3/build.gradle
@@ -35,7 +35,7 @@ createJavaExamplesArchetypeValidationTask(type: 'Quickstart', runner: 'Spark')
 // Additional supported Spark versions (used in compatibility tests)
 def sparkVersions = [
     "330": "3.3.0",
-    "321": "3.2.1"
+    "312": "3.1.2"
 ]
 
 sparkVersions.each { kv ->
diff --git a/runners/spark/3/job-server/build.gradle b/runners/spark/3/job-server/build.gradle
index d11a1a8edb1..68bb8d9a10e 100644
--- a/runners/spark/3/job-server/build.gradle
+++ b/runners/spark/3/job-server/build.gradle
@@ -28,13 +28,4 @@ project.ext {
 }
 
 // Load the main build script which contains all build logic.
-apply from: "$basePath/spark_job_server.gradle"
-
-
-configurations.runtimeClasspath {
-  resolutionStrategy {
-    // Downgrade the Scala version of the job-server to match the Scala version of a Spark 3.1.2 cluster to prevent
-    // a Scala bug (InvalidClassException when deserializing WrappedArray), see https://github.com/apache/beam/issues/21092
-    force "org.scala-lang:scala-library:2.12.10"
-  }
-}
\ No newline at end of file
+apply from: "$basePath/spark_job_server.gradle"
\ No newline at end of file
diff --git a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
index e70cc7253f8..ceafc1642ba 100644
--- a/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
+++ b/runners/spark/3/src/main/java/org/apache/beam/runners/spark/structuredstreaming/translation/helpers/EncoderFactory.java
@@ -69,6 +69,10 @@ public class EncoderFactory {
           // Spark 3.1.x
           return STATIC_INVOKE_CONSTRUCTOR.newInstance(
               cls, type, fun, seqOf(args), propagateNull, true);
+        case 7:
+          // Spark 3.2.0
+          return STATIC_INVOKE_CONSTRUCTOR.newInstance(
+              cls, type, fun, seqOf(args), emptyList(), propagateNull, true);
         case 8:
           // Spark 3.2.x, 3.3.x
           return STATIC_INVOKE_CONSTRUCTOR.newInstance(
@@ -89,8 +93,14 @@ public class EncoderFactory {
       // created reflectively. This is fine as it's just needed once to create the query plan.
       switch (STATIC_INVOKE_CONSTRUCTOR.getParameterCount()) {
         case 6:
+          // Spark 3.1.x
           return INVOKE_CONSTRUCTOR.newInstance(obj, fun, type, seqOf(args), false, nullable);
+        case 7:
+          // Spark 3.2.0
+          return INVOKE_CONSTRUCTOR.newInstance(
+              obj, fun, type, seqOf(args), emptyList(), false, nullable);
         case 8:
+          // Spark 3.2.x, 3.3.x
           return INVOKE_CONSTRUCTOR.newInstance(
               obj, fun, type, seqOf(args), emptyList(), false, nullable, true);
         default:
diff --git a/runners/spark/spark_runner.gradle b/runners/spark/spark_runner.gradle
index 0e3028d3843..9ee9b48585c 100644
--- a/runners/spark/spark_runner.gradle
+++ b/runners/spark/spark_runner.gradle
@@ -181,7 +181,7 @@ dependencies {
   runtimeOnly library.java.jackson_module_scala_2_12
   // Force paranamer 2.8 to avoid issues when using Scala 2.12
   runtimeOnly "com.thoughtworks.paranamer:paranamer:2.8"
-  provided library.java.hadoop_common
+  provided "org.apache.hadoop:hadoop-client-api:3.3.1"
   provided library.java.commons_io
   provided library.java.hamcrest
   provided "com.esotericsoftware:kryo-shaded:4.0.2"
diff --git a/website/www/site/content/en/documentation/runners/spark.md b/website/www/site/content/en/documentation/runners/spark.md
index 0b3075061e4..dcc166873dc 100644
--- a/website/www/site/content/en/documentation/runners/spark.md
+++ b/website/www/site/content/en/documentation/runners/spark.md
@@ -67,8 +67,8 @@ the portable Runner. For more information on portability, please visit the
 
 ## Spark Runner prerequisites and setup
 
-The Spark runner currently supports Spark's 3.1.x branch.
-> **Note:** Support for Spark 2.4.x was deprecated as of Beam 2.41.0 and finally dropped with the release of Beam 2.46.0.
+The Spark runner currently supports Spark's 3.2.x branch.
+> **Note:** Support for Spark 2.4.x was dropped with Beam 2.46.0.
 
 {{< paragraph class="language-java" >}}
 You can add a dependency on the latest version of the Spark runner by adding to your pom.xml the following:
@@ -215,7 +215,7 @@ options = PipelineOptions([
     "--job_endpoint=localhost:8099",
     "--environment_type=LOOPBACK"
 ])
-with beam.Pipeline(options=options) as p:
+with beam.Pipeline(options) as p:
     ...
 {{< /highlight >}}
 
@@ -243,7 +243,7 @@ See [here](/roadmap/portability/#sdk-harness-config) for details.)
 
 ###  Running on Dataproc cluster (YARN backed)
 
-To run Beam jobs written in Python, Go, and other supported languages, you can use the `SparkRunner` and `PortableRunner` as described on the Beam's [Spark Runner](/documentation/runners/spark/) page (also see [Portability Framework Roadmap](/roadmap/portability/)).
+To run Beam jobs written in Python, Go, and other supported languages, you can use the `SparkRunner` and `PortableRunner` as described on the Beam's [Spark Runner](https://beam.apache.org/documentation/runners/spark/) page (also see [Portability Framework Roadmap](https://beam.apache.org/roadmap/portability/)).
 
 The following example runs a portable Beam job in Python from the Dataproc cluster's master node with Yarn backed.