You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2019/09/03 19:05:26 UTC

[beam] branch master updated: [BEAM-7966] Write portable Flink application jar

This is an automated email from the ASF dual-hosted git repository.

ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new f0e6c8b  [BEAM-7966] Write portable Flink application jar
     new 4460f03  Merge pull request #9331 from ibzib/flink-create-jar
f0e6c8b is described below

commit f0e6c8be3b46b8a7839be489e30ceb99c92396c1
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Tue Aug 13 10:39:19 2019 -0700

    [BEAM-7966] Write portable Flink application jar
---
 .../apache/beam/runners/flink/FlinkJobInvoker.java |  30 ++-
 .../beam/runners/flink/FlinkSavepointTest.java     |   6 +-
 .../beam/runners/flink/PortableExecutionTest.java  |   4 +-
 .../runners/flink/PortableStateExecutionTest.java  |   4 +-
 .../runners/flink/PortableTimersExecutionTest.java |   4 +-
 .../beam/runners/flink/ReadSourcePortableTest.java |   4 +-
 .../jobsubmission/PortablePipelineJarCreator.java  | 293 +++++++++++++++++++++
 .../jobsubmission/PortablePipelineJarUtils.java    |  54 ++++
 .../PortablePipelineJarCreatorTest.java            | 172 ++++++++++++
 .../beam/sdk/options/PortablePipelineOptions.java  |   5 +
 10 files changed, 552 insertions(+), 24 deletions(-)

diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
index 1ec2a6e..24edb28 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
@@ -20,13 +20,14 @@ package org.apache.beam.runners.flink;
 import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.UUID;
 import javax.annotation.Nullable;
 import org.apache.beam.model.pipeline.v1.RunnerApi;
 import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
 import org.apache.beam.runners.fnexecution.jobsubmission.JobInvocation;
 import org.apache.beam.runners.fnexecution.jobsubmission.JobInvoker;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineJarCreator;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineRunner;
 import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
 import org.apache.beam.sdk.options.PortablePipelineOptions;
 import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.Struct;
@@ -63,7 +64,6 @@ public class FlinkJobInvoker extends JobInvoker {
 
     String invocationId =
         String.format("%s_%s", flinkOptions.getJobName(), UUID.randomUUID().toString());
-    LOG.info("Invoking job {}", invocationId);
 
     if (FlinkPipelineOptions.AUTO.equals(flinkOptions.getFlinkMaster())) {
       flinkOptions.setFlinkMaster(serverConfig.getFlinkMasterUrl());
@@ -74,16 +74,23 @@ public class FlinkJobInvoker extends JobInvoker {
       portableOptions.setSdkWorkerParallelism(serverConfig.getSdkWorkerParallelism());
     }
 
+    PortablePipelineRunner pipelineRunner;
+    if (portableOptions.getOutputExecutablePath() == null
+        || portableOptions.getOutputExecutablePath().isEmpty()) {
+      pipelineRunner =
+          new FlinkPipelineRunner(
+              flinkOptions,
+              serverConfig.getFlinkConfDir(),
+              detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
+    } else {
+      pipelineRunner = new PortablePipelineJarCreator(FlinkPipelineRunner.class);
+    }
+
     flinkOptions.setRunner(null);
 
+    LOG.info("Invoking job {} with pipeline runner {}", invocationId, pipelineRunner);
     return createJobInvocation(
-        invocationId,
-        retrievalToken,
-        executorService,
-        pipeline,
-        flinkOptions,
-        serverConfig.getFlinkConfDir(),
-        detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
+        invocationId, retrievalToken, executorService, pipeline, flinkOptions, pipelineRunner);
   }
 
   static JobInvocation createJobInvocation(
@@ -92,16 +99,13 @@ public class FlinkJobInvoker extends JobInvoker {
       ListeningExecutorService executorService,
       RunnerApi.Pipeline pipeline,
       FlinkPipelineOptions flinkOptions,
-      @Nullable String confDir,
-      List<String> filesToStage) {
+      PortablePipelineRunner pipelineRunner) {
     JobInfo jobInfo =
         JobInfo.create(
             invocationId,
             flinkOptions.getJobName(),
             retrievalToken,
             PipelineOptionsTranslation.toProto(flinkOptions));
-    FlinkPipelineRunner pipelineRunner =
-        new FlinkPipelineRunner(flinkOptions, confDir, filesToStage);
     return new JobInvocation(jobInfo, executorService, pipeline, pipelineRunner);
   }
 }
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
index 0bc14fb..4315e62 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
@@ -206,6 +206,7 @@ public class FlinkSavepointTest implements Serializable {
 
     ListeningExecutorService executorService =
         MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
+    FlinkPipelineOptions pipelineOptions = pipeline.getOptions().as(FlinkPipelineOptions.class);
     try {
       JobInvocation jobInvocation =
           FlinkJobInvoker.createJobInvocation(
@@ -213,9 +214,8 @@ public class FlinkSavepointTest implements Serializable {
               "none",
               executorService,
               pipelineProto,
-              pipeline.getOptions().as(FlinkPipelineOptions.class),
-              null,
-              Collections.emptyList());
+              pipelineOptions,
+              new FlinkPipelineRunner(pipelineOptions, null, Collections.emptyList()));
 
       jobInvocation.start();
 
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
index 15be023..10db32c 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
@@ -148,8 +148,8 @@ public class PortableExecutionTest implements Serializable {
             flinkJobExecutor,
             pipelineProto,
             options.as(FlinkPipelineOptions.class),
-            null,
-            Collections.emptyList());
+            new FlinkPipelineRunner(
+                options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
     jobInvocation.start();
     while (jobInvocation.getState() != Enum.DONE) {
       Thread.sleep(1000);
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
index 7096d7d..91f243b 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
@@ -202,8 +202,8 @@ public class PortableStateExecutionTest implements Serializable {
             flinkJobExecutor,
             pipelineProto,
             options.as(FlinkPipelineOptions.class),
-            null,
-            Collections.emptyList());
+            new FlinkPipelineRunner(
+                options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
 
     jobInvocation.start();
 
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
index cb16b27..9cddcd6 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
@@ -189,8 +189,8 @@ public class PortableTimersExecutionTest implements Serializable {
             flinkJobExecutor,
             pipelineProto,
             options.as(FlinkPipelineOptions.class),
-            null,
-            Collections.emptyList());
+            new FlinkPipelineRunner(
+                options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
 
     jobInvocation.start();
     while (jobInvocation.getState() != Enum.DONE) {
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
index bd72d29..40d621f 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
@@ -110,8 +110,8 @@ public class ReadSourcePortableTest implements Serializable {
             flinkJobExecutor,
             pipelineProto,
             options.as(FlinkPipelineOptions.class),
-            null,
-            Collections.emptyList());
+            new FlinkPipelineRunner(
+                options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
     jobInvocation.start();
     while (jobInvocation.getState() != Enum.DONE) {
       Thread.sleep(100);
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java
new file mode 100644
index 0000000..0314fcd
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.jar.Attributes;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactChunk;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest.Location;
+import org.apache.beam.model.jobmanagement.v1.ArtifactRetrievalServiceGrpc;
+import org.apache.beam.model.jobmanagement.v1.ArtifactRetrievalServiceGrpc.ArtifactRetrievalServiceBlockingStub;
+import org.apache.beam.model.jobmanagement.v1.JobApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Pipeline;
+import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
+import org.apache.beam.runners.core.construction.PipelineResources;
+import org.apache.beam.runners.fnexecution.GrpcFnServer;
+import org.apache.beam.runners.fnexecution.InProcessServerFactory;
+import org.apache.beam.runners.fnexecution.artifact.BeamFileSystemArtifactRetrievalService;
+import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
+import org.apache.beam.sdk.fn.test.InProcessManagedChannelFactory;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.options.PortablePipelineOptions;
+import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.MessageOrBuilder;
+import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.util.JsonFormat;
+import org.apache.beam.vendor.grpc.v1p21p0.io.grpc.ManagedChannel;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.commons.compress.utils.IOUtils;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link PortablePipelineRunner} that bundles the input pipeline along with all dependencies,
+ * artifacts, etc. required to run the pipeline into a jar that can be executed later.
+ *
+ * <p>Each {@link PortablePipelineJarCreator} instance is not threadsafe; a new instance is expected
+ * to be constructed and {@link #run} once per job.
+ */
+public class PortablePipelineJarCreator implements PortablePipelineRunner {
+  private static final Logger LOG = LoggerFactory.getLogger(PortablePipelineJarCreator.class);
+
+  private final Class mainClass;
+
+  @VisibleForTesting JarOutputStream outputStream;
+  /** Wrapper over {@link #outputStream}. */
+  @VisibleForTesting WritableByteChannel outputChannel;
+
+  public PortablePipelineJarCreator(Class mainClass) {
+    this.mainClass = mainClass;
+  }
+
+  /**
+   * <em>Does not actually run the pipeline.</em> Instead bundles the input pipeline along with all
+   * dependencies, artifacts, etc. required to run the pipeline into a jar that can be executed
+   * later.
+   */
+  @Override
+  public PortablePipelineResult run(Pipeline pipeline, JobInfo jobInfo) throws Exception {
+    PortablePipelineOptions pipelineOptions =
+        PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions())
+            .as(PortablePipelineOptions.class);
+
+    File outputFile = new File(pipelineOptions.getOutputExecutablePath());
+    LOG.info("Creating jar {}", outputFile.getAbsolutePath());
+    outputStream = new JarOutputStream(new FileOutputStream(outputFile), createManifest(mainClass));
+    outputChannel = Channels.newChannel(outputStream);
+    writeClassPathResources(mainClass.getClassLoader());
+    writeAsJson(pipeline, PortablePipelineJarUtils.PIPELINE_PATH);
+    writeAsJson(
+        PipelineOptionsTranslation.toProto(pipelineOptions),
+        PortablePipelineJarUtils.PIPELINE_OPTIONS_PATH);
+    writeArtifacts(jobInfo.retrievalToken());
+    // Closing the channel also closes the underlying stream.
+    outputChannel.close();
+
+    LOG.info("Jar {} created successfully.", outputFile.getAbsolutePath());
+    return new JarCreatorPipelineResult();
+  }
+
+  @VisibleForTesting
+  Manifest createManifest(Class mainClass) {
+    Manifest manifest = new Manifest();
+    manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+    boolean classHasMainMethod = false;
+    try {
+      Class returnType = mainClass.getMethod("main", String[].class).getReturnType();
+      if (returnType == Void.TYPE) {
+        classHasMainMethod = true;
+      } else {
+        LOG.warn(
+            "No Main-Class will be set in jar because main method in {} returns {}, expected void",
+            mainClass,
+            returnType);
+      }
+    } catch (NoSuchMethodException e) {
+      LOG.warn("No Main-Class will be set in jar because {} lacks a main method.", mainClass);
+    }
+    if (classHasMainMethod) {
+      manifest.getMainAttributes().put(Name.MAIN_CLASS, mainClass.getName());
+    }
+    return manifest;
+  }
+
+  /** Copy resources from {@code classLoader} to {@link #outputStream}. */
+  private void writeClassPathResources(ClassLoader classLoader) throws IOException {
+    List<String> classPathResources =
+        PipelineResources.detectClassPathResourcesToStage(classLoader);
+    Preconditions.checkArgument(
+        classPathResources.size() == 1, "Expected exactly one jar on " + classLoader.toString());
+    copyResourcesFromJar(new JarFile(classPathResources.get(0)));
+  }
+
+  /** Copy resources from {@code inputJar} to {@link #outputStream}. */
+  @VisibleForTesting
+  protected void copyResourcesFromJar(JarFile inputJar) throws IOException {
+    Enumeration<JarEntry> inputJarEntries = inputJar.entries();
+    // The zip spec allows multiple files with the same name; the Java zip libraries do not.
+    // Keep track of the files we've already written to filter out duplicates.
+    // Also, ignore the old manifest; we want to write our own.
+    Set<String> previousEntryNames =
+        new HashSet<>(
+            ImmutableList.of(
+                JarFile.MANIFEST_NAME,
+                PortablePipelineJarUtils.ARTIFACT_MANIFEST_PATH,
+                PortablePipelineJarUtils.PIPELINE_PATH,
+                PortablePipelineJarUtils.PIPELINE_OPTIONS_PATH));
+    while (inputJarEntries.hasMoreElements()) {
+      JarEntry inputJarEntry = inputJarEntries.nextElement();
+      InputStream inputStream = inputJar.getInputStream(inputJarEntry);
+      String entryName = inputJarEntry.getName();
+      if (previousEntryNames.contains(entryName)) {
+        LOG.debug("Skipping duplicated file {}", entryName);
+      } else {
+        JarEntry outputJarEntry = new JarEntry(inputJarEntry);
+        outputStream.putNextEntry(outputJarEntry);
+        LOG.trace("Copying jar entry {}", inputJarEntry);
+        IOUtils.copy(inputStream, outputStream);
+        previousEntryNames.add(entryName);
+      }
+    }
+  }
+
+  @VisibleForTesting
+  interface ArtifactRetriever {
+    GetManifestResponse getManifest(GetManifestRequest request);
+
+    Iterator<ArtifactChunk> getArtifact(GetArtifactRequest request);
+  }
+
+  /**
+   * Copy all artifacts retrievable via the {@link ArtifactRetrievalServiceBlockingStub} to the
+   * {@code outputStream}.
+   *
+   * @return A {@link ProxyManifest} pointing to the artifacts' location in the output jar.
+   */
+  @VisibleForTesting
+  ProxyManifest copyStagedArtifacts(String retrievalToken, ArtifactRetriever retrievalServiceStub)
+      throws IOException {
+    GetManifestRequest manifestRequest =
+        GetManifestRequest.newBuilder().setRetrievalToken(retrievalToken).build();
+    ArtifactApi.Manifest manifest = retrievalServiceStub.getManifest(manifestRequest).getManifest();
+    // Create a new proxy manifest to locate artifacts at jar runtime.
+    ProxyManifest.Builder proxyManifestBuilder = ProxyManifest.newBuilder().setManifest(manifest);
+    for (ArtifactMetadata artifact : manifest.getArtifactList()) {
+      String outputPath =
+          PortablePipelineJarUtils.ARTIFACT_FOLDER_PATH + "/" + UUID.randomUUID().toString();
+      LOG.trace("Copying artifact {} to {}", artifact.getName(), outputPath);
+      proxyManifestBuilder.addLocation(
+          Location.newBuilder().setName(artifact.getName()).setUri("/" + outputPath).build());
+      outputStream.putNextEntry(new JarEntry(outputPath));
+      GetArtifactRequest artifactRequest =
+          GetArtifactRequest.newBuilder()
+              .setRetrievalToken(retrievalToken)
+              .setName(artifact.getName())
+              .build();
+      Iterator<ArtifactChunk> artifactResponse = retrievalServiceStub.getArtifact(artifactRequest);
+      while (artifactResponse.hasNext()) {
+        artifactResponse.next().getData().writeTo(outputStream);
+      }
+    }
+    return proxyManifestBuilder.build();
+  }
+
+  /**
+   * Uses {@link BeamFileSystemArtifactRetrievalService} to fetch artifacts, then writes the
+   * artifacts to {@code outputStream}. Include a {@link ProxyManifest} to locate artifacts later.
+   */
+  private void writeArtifacts(String retrievalToken) throws Exception {
+    try (GrpcFnServer artifactServer =
+        GrpcFnServer.allocatePortAndCreateFor(
+            BeamFileSystemArtifactRetrievalService.create(), InProcessServerFactory.create())) {
+      ManagedChannel grpcChannel =
+          InProcessManagedChannelFactory.create()
+              .forDescriptor(artifactServer.getApiServiceDescriptor());
+      ArtifactRetrievalServiceBlockingStub retrievalServiceStub =
+          ArtifactRetrievalServiceGrpc.newBlockingStub(grpcChannel);
+      ProxyManifest proxyManifest =
+          copyStagedArtifacts(
+              retrievalToken,
+              new ArtifactRetriever() {
+                @Override
+                public GetManifestResponse getManifest(GetManifestRequest request) {
+                  return retrievalServiceStub.getManifest(request);
+                }
+
+                @Override
+                public Iterator<ArtifactChunk> getArtifact(GetArtifactRequest request) {
+                  return retrievalServiceStub.getArtifact(request);
+                }
+              });
+      writeAsJson(proxyManifest, PortablePipelineJarUtils.ARTIFACT_MANIFEST_PATH);
+      grpcChannel.shutdown();
+    }
+  }
+
+  /** Helper method for writing {@code message} in UTF-8 JSON format. */
+  private void writeAsJson(MessageOrBuilder message, String outputPath) throws IOException {
+    outputStream.putNextEntry(new JarEntry(outputPath));
+    outputChannel.write(StandardCharsets.UTF_8.encode(JsonFormat.printer().print(message)));
+  }
+
+  private static class JarCreatorPipelineResult implements PortablePipelineResult {
+
+    @Override
+    public State getState() {
+      return State.DONE;
+    }
+
+    @Override
+    public State cancel() {
+      return State.DONE;
+    }
+
+    @Override
+    public State waitUntilFinish(Duration duration) {
+      return State.DONE;
+    }
+
+    @Override
+    public State waitUntilFinish() {
+      return State.DONE;
+    }
+
+    @Override
+    public MetricResults metrics() {
+      throw new UnsupportedOperationException("Jar creation does not yield metrics.");
+    }
+
+    @Override
+    public JobApi.MetricResults portableMetrics() throws UnsupportedOperationException {
+      return JobApi.MetricResults.getDefaultInstance();
+    }
+  }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java
new file mode 100644
index 0000000..af712ad
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+/**
+ * Contains common code for writing and reading portable pipeline jars.
+ *
+ * <p>Jar layout:
+ *
+ * <ul>
+ *   <li>META-INF/
+ *       <ul>
+ *         <li>MANIFEST.MF
+ *       </ul>
+ *   <li>BEAM-PIPELINE/
+ *       <ul>
+ *         <li>pipeline.json
+ *         <li>pipeline-options.json
+ *       </ul>
+ *   <li>BEAM-ARTIFACT-STAGING/
+ *       <ul>
+ *         <li>artifact-manifest.json
+ *         <li>artifacts/
+ *             <ul>
+ *               <li>...artifact files...
+ *             </ul>
+ *       </ul>
+ *   <li>...Java classes...
+ * </ul>
+ */
+public abstract class PortablePipelineJarUtils {
+  private static final String ARTIFACT_STAGING_FOLDER_PATH = "BEAM-ARTIFACT-STAGING";
+  static final String ARTIFACT_FOLDER_PATH = ARTIFACT_STAGING_FOLDER_PATH + "/artifacts";
+  private static final String PIPELINE_FOLDER_PATH = "BEAM-PIPELINE";
+  static final String ARTIFACT_MANIFEST_PATH =
+      ARTIFACT_STAGING_FOLDER_PATH + "/artifact-manifest.json";
+  static final String PIPELINE_PATH = PIPELINE_FOLDER_PATH + "/pipeline.json";
+  static final String PIPELINE_OPTIONS_PATH = PIPELINE_FOLDER_PATH + "/pipeline-options.json";
+}
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java
new file mode 100644
index 0000000..458a456
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.channels.WritableByteChannel;
+import java.util.Collections;
+import java.util.List;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest.Location;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineJarCreator.ArtifactRetriever;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+
+/** Unit tests for {@link PortablePipelineJarCreator}. */
+@RunWith(JUnit4.class)
+public class PortablePipelineJarCreatorTest implements Serializable {
+
+  @Mock private JarFile inputJar;
+  @Mock private JarOutputStream outputStream;
+  @Mock private WritableByteChannel outputChannel;
+  @Mock private ArtifactRetriever retrievalServiceStub;
+  private PortablePipelineJarCreator jarCreator;
+
+  @Before
+  public void setup() throws IOException {
+    initMocks(this);
+    JarInputStream emptyInputStream = new JarInputStream(new ByteArrayInputStream(new byte[0]));
+    when(inputJar.getInputStream(any())).thenReturn(emptyInputStream);
+    when(retrievalServiceStub.getArtifact(any())).thenReturn(Collections.emptyIterator());
+    jarCreator = new PortablePipelineJarCreator(null);
+    jarCreator.outputStream = outputStream;
+    jarCreator.outputChannel = outputChannel;
+  }
+
+  @Test
+  public void testCopyResourcesFromJar_copiesResources() throws IOException {
+    List<JarEntry> entries =
+        ImmutableList.of(new JarEntry("foo"), new JarEntry("bar"), new JarEntry("baz"));
+    when(inputJar.entries()).thenReturn(Collections.enumeration(entries));
+    jarCreator.copyResourcesFromJar(inputJar);
+    verify(outputStream, times(3)).putNextEntry(any());
+  }
+
+  @Test
+  public void testCopyResourcesFromJar_ignoresManifest() throws IOException {
+    List<JarEntry> manifestEntry = ImmutableList.of(new JarEntry(JarFile.MANIFEST_NAME));
+    when(inputJar.entries()).thenReturn(Collections.enumeration(manifestEntry));
+    jarCreator.copyResourcesFromJar(inputJar);
+    verify(outputStream, never()).putNextEntry(any());
+  }
+
+  @Test
+  public void testCopyResourcesFromJar_ignoresDuplicates() throws IOException {
+    List<JarEntry> duplicateEntries = ImmutableList.of(new JarEntry("foo"), new JarEntry("foo"));
+    when(inputJar.entries()).thenReturn(Collections.enumeration(duplicateEntries));
+    jarCreator.copyResourcesFromJar(inputJar);
+    verify(outputStream, times(1)).putNextEntry(any());
+  }
+
+  @Test
+  public void testCopyStagedArtifacts_returnsProxyManifest() throws IOException {
+    ArtifactMetadata artifact1 = ArtifactMetadata.newBuilder().setName("foo").build();
+    ArtifactMetadata artifact2 = ArtifactMetadata.newBuilder().setName("bar").build();
+    List<ArtifactMetadata> artifacts = ImmutableList.of(artifact1, artifact2);
+    ArtifactApi.Manifest manifest =
+        ArtifactApi.Manifest.newBuilder().addAllArtifact(artifacts).build();
+    when(retrievalServiceStub.getManifest(any()))
+        .thenReturn(GetManifestResponse.newBuilder().setManifest(manifest).build());
+
+    ProxyManifest proxyManifest =
+        jarCreator.copyStagedArtifacts("retrievalToken", retrievalServiceStub);
+
+    assertEquals(manifest, proxyManifest.getManifest());
+    List<String> outputArtifactNames =
+        proxyManifest.getLocationList().stream()
+            .map(Location::getName)
+            .collect(Collectors.toList());
+    assertThat(outputArtifactNames, containsInAnyOrder("foo", "bar"));
+  }
+
+  @Test
+  public void testCopyStagedArtifacts_copiesArtifacts() throws IOException {
+    ArtifactMetadata artifact1 = ArtifactMetadata.newBuilder().setName("foo").build();
+    ArtifactMetadata artifact2 = ArtifactMetadata.newBuilder().setName("bar").build();
+    List<ArtifactMetadata> artifacts = ImmutableList.of(artifact1, artifact2);
+    ArtifactApi.Manifest manifest =
+        ArtifactApi.Manifest.newBuilder().addAllArtifact(artifacts).build();
+    when(retrievalServiceStub.getManifest(any()))
+        .thenReturn(GetManifestResponse.newBuilder().setManifest(manifest).build());
+
+    jarCreator.copyStagedArtifacts("retrievalToken", retrievalServiceStub);
+
+    verify(outputStream, times(2)).putNextEntry(any());
+  }
+
+  private static class FakePipelineRunnner {
+    public static void main(String[] args) {
+      System.out.println("Hello world");
+    }
+  }
+
+  @Test
+  public void testCreateManifest_withMainMethod() {
+    Manifest manifest = jarCreator.createManifest(FakePipelineRunnner.class);
+    assertEquals(
+        FakePipelineRunnner.class.getName(),
+        manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+  }
+
+  private static class EmptyPipelineRunner {}
+
+  @Test
+  public void testCreateManifest_withoutMainMethod() {
+    Manifest manifest = jarCreator.createManifest(EmptyPipelineRunner.class);
+    assertNull(manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+  }
+
+  private static class EvilPipelineRunner {
+    public static int main(String[] args) {
+      return 0;
+    }
+  }
+
+  @Test
+  public void testCreateManifest_withInvalidMainMethod() {
+    Manifest manifest = jarCreator.createManifest(EvilPipelineRunner.class);
+    assertNull(manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+  }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
index 05f821d..e28cc80 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
@@ -86,4 +86,9 @@ public interface PortablePipelineOptions extends PipelineOptions {
   int getEnvironmentExpirationMillis();
 
   void setEnvironmentExpirationMillis(int environmentExpirationMillis);
+
+  @Description("The output path for the executable file to be created.")
+  String getOutputExecutablePath();
+
+  void setOutputExecutablePath(String outputExecutablePath);
 }