[beam] 01/04: Deactivate deps resolution forcing that prevent using correct spark transitive dep

[ec...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

echauchot pushed a commit to branch spark-runner_structured-streaming
in repository https://gitbox.apache.org/repos/asf/beam.git

commit a3eb035ea473f3f8ef20dea3f51a1cdf0d42bd94
Author: Etienne Chauchot <ec...@apache.org>
AuthorDate: Thu Jan 10 10:55:54 2019 +0100

    Deactivate deps resolution forcing that prevent using correct spark transitive dep
---
 runners/spark-structured-streaming/build.gradle | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/runners/spark-structured-streaming/build.gradle b/runners/spark-structured-streaming/build.gradle
index 803058f..3154fca 100644
--- a/runners/spark-structured-streaming/build.gradle
+++ b/runners/spark-structured-streaming/build.gradle
@@ -34,6 +34,17 @@ configurations {
   validatesRunner
 }
 
+configurations.all {
+  resolutionStrategy {
+    // Beam parent forces all deps to be the versions defined. In our case we have spark-core
+    // defined to v2.3.2 for "old" spark runner. So spark-core transitive dep of spark-sql v2.4.0
+    // gets forced to 2.3.2. Remove it from the forced deps and add v 2.4.0
+//    forcedModules.removeElement("org.apache.spark:spark-core_2.11:2.3.2")
+//    forcedModules.removeElement("org.apache.spark:spark-network-common_2.11:2.3.2")
+//    forcedModules.removeElement("org.apache.spark:spark-streaming_2.11:2.3.2")
+    forcedModules = []
+  }
+}
 test {
   systemProperty "spark.ui.enabled", "false"
   systemProperty "spark.ui.showConsoleProgress", "false"
@@ -45,10 +56,7 @@ test {
 }
 
 dependencies {
-  compile('org.apache.spark:spark-core_2.11:2.4.0'){
-    exclude group: 'org.json4s', module: 'json4s-jackson_2.11'
-  }
-  shadow "org.json4s:json4s-jackson_2.11:3.6.3"
+  shadow "com.fasterxml.jackson.module:jackson-module-scala_2.11:2.9.8"
   shadow project(path: ":beam-model-pipeline", configuration: "shadow")
   shadow project(path: ":beam-sdks-java-core", configuration: "shadow")
   shadow project(path: ":beam-runners-core-construction-java", configuration: "shadow")