[beam] branch master updated: [BEAM-9575] Only copy the Spark runner jar, not whatever other jars happen to be on the classloader.

[ib...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new a15dd3b  [BEAM-9575] Only copy the Spark runner jar, not whatever other jars happen to be on the classloader.
     new 141fe93  Merge pull request #12820 from ibzib/BEAM-9575
a15dd3b is described below

commit a15dd3b57a24c3bd234379b0a39579fadf634db6
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Thu Sep 10 17:37:11 2020 -0700

    [BEAM-9575] Only copy the Spark runner jar, not whatever other jars happen to be on the classloader.
---
 .../jobsubmission/PortablePipelineJarCreator.java       | 17 ++---------------
 1 file changed, 2 insertions(+), 15 deletions(-)

diff --git a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java
index ef05075..a0c5983 100644
--- a/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java
+++ b/runners/java-job-service/src/main/java/org/apache/beam/runners/jobsubmission/PortablePipelineJarCreator.java
@@ -28,7 +28,6 @@ import java.nio.channels.WritableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.util.Enumeration;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
@@ -42,17 +41,14 @@ import org.apache.beam.model.jobmanagement.v1.JobApi;
 import org.apache.beam.model.pipeline.v1.RunnerApi;
 import org.apache.beam.model.pipeline.v1.RunnerApi.Pipeline;
 import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
-import org.apache.beam.runners.core.construction.resources.PipelineResources;
 import org.apache.beam.runners.fnexecution.artifact.ArtifactRetrievalService;
 import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
 import org.apache.beam.sdk.io.ClassLoaderFileSystem;
 import org.apache.beam.sdk.metrics.MetricResults;
-import org.apache.beam.sdk.options.PipelineOptions;
 import org.apache.beam.sdk.options.PortablePipelineOptions;
 import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.MessageOrBuilder;
 import org.apache.beam.vendor.grpc.v1p26p0.com.google.protobuf.util.JsonFormat;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
-import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
 import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.io.ByteStreams;
 import org.apache.commons.compress.utils.IOUtils;
@@ -98,7 +94,8 @@ public class PortablePipelineJarCreator implements PortablePipelineRunner {
         new JarOutputStream(new FileOutputStream(outputFile), createManifest(mainClass, jobName));
     outputChannel = Channels.newChannel(outputStream);
     PortablePipelineJarUtils.writeDefaultJobName(outputStream, jobName);
-    writeClassPathResources(mainClass.getClassLoader(), pipelineOptions);
+    copyResourcesFromJar(
+        new JarFile(mainClass.getProtectionDomain().getCodeSource().getLocation().getPath()));
     writeAsJson(
         PipelineOptionsTranslation.toProto(pipelineOptions),
         PortablePipelineJarUtils.getPipelineOptionsUri(jobName));
@@ -135,16 +132,6 @@ public class PortablePipelineJarCreator implements PortablePipelineRunner {
     return manifest;
   }
 
-  /** Copy resources from {@code classLoader} to {@link #outputStream}. */
-  private void writeClassPathResources(ClassLoader classLoader, PipelineOptions options)
-      throws IOException {
-    List<String> classPathResources =
-        PipelineResources.detectClassPathResourcesToStage(classLoader, options);
-    Preconditions.checkArgument(
-        classPathResources.size() == 1, "Expected exactly one jar on " + classLoader.toString());
-    copyResourcesFromJar(new JarFile(classPathResources.get(0)));
-  }
-
   /** Copy resources from {@code inputJar} to {@link #outputStream}. */
   @VisibleForTesting
   protected void copyResourcesFromJar(JarFile inputJar) throws IOException {