[beam] 01/05: [BEAM-981] Add parameter allowing adding jars to spark context

[lc...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

lcwik pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git

commit dfd16f2b2c782685a02092629970d7b5f87c373b
Author: Łukasz Gajowy <lu...@polidea.com>
AuthorDate: Tue Dec 12 18:03:32 2017 +0100

    [BEAM-981] Add parameter allowing adding jars to spark context
---
 .../apache/beam/runners/spark/SparkPipelineOptions.java   | 15 +++++++++++++++
 .../runners/spark/translation/SparkContextFactory.java    |  5 +++++
 2 files changed, 20 insertions(+)

diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
index 26b549b..2053b44 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/SparkPipelineOptions.java
@@ -18,6 +18,9 @@
 
 package org.apache.beam.runners.spark;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.beam.sdk.options.ApplicationNameOptions;
 import org.apache.beam.sdk.options.Default;
 import org.apache.beam.sdk.options.DefaultValueFactory;
@@ -101,4 +104,16 @@ public interface SparkPipelineOptions
   boolean getUsesProvidedSparkContext();
   void setUsesProvidedSparkContext(boolean value);
 
+  @Description("Jars for spark context")
+  @Default.InstanceFactory(SparkContextOptions.EmptyPathList.class)
+  List<String> getJarsForSparkContext();
+  void setJarsForSparkContext(List<String> jars);
+
+  /** Returns an empty path list, top avoid handling null. */
+  class EmptyPathList implements DefaultValueFactory<List<String>> {
+    @Override
+    public List<String> create(PipelineOptions options) {
+      return new ArrayList<>();
+    }
+  }
 }
diff --git a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
index 0132de3..68c4093 100644
--- a/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
+++ b/runners/spark/src/main/java/org/apache/beam/runners/spark/translation/SparkContextFactory.java
@@ -92,6 +92,11 @@ public final class SparkContextFactory {
         // set master if not set.
         conf.setMaster(contextOptions.getSparkMaster());
       }
+
+      if (contextOptions.getJarsForSparkContext().size() > 0) {
+        conf.setJars(contextOptions.getJarsForSparkContext().toArray(new String[0]));
+      }
+
       conf.setAppName(contextOptions.getAppName());
       // register immutable collections serializers because the SDK uses them.
       conf.set("spark.kryo.registrator", BeamSparkRunnerRegistrator.class.getName());

-- 
To stop receiving notification emails like this one, please contact
"commits@beam.apache.org" <co...@beam.apache.org>.