You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2019/09/03 19:05:26 UTC
[beam] branch master updated: [BEAM-7966] Write portable Flink
application jar
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new f0e6c8b [BEAM-7966] Write portable Flink application jar
new 4460f03 Merge pull request #9331 from ibzib/flink-create-jar
f0e6c8b is described below
commit f0e6c8be3b46b8a7839be489e30ceb99c92396c1
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Tue Aug 13 10:39:19 2019 -0700
[BEAM-7966] Write portable Flink application jar
---
.../apache/beam/runners/flink/FlinkJobInvoker.java | 30 ++-
.../beam/runners/flink/FlinkSavepointTest.java | 6 +-
.../beam/runners/flink/PortableExecutionTest.java | 4 +-
.../runners/flink/PortableStateExecutionTest.java | 4 +-
.../runners/flink/PortableTimersExecutionTest.java | 4 +-
.../beam/runners/flink/ReadSourcePortableTest.java | 4 +-
.../jobsubmission/PortablePipelineJarCreator.java | 293 +++++++++++++++++++++
.../jobsubmission/PortablePipelineJarUtils.java | 54 ++++
.../PortablePipelineJarCreatorTest.java | 172 ++++++++++++
.../beam/sdk/options/PortablePipelineOptions.java | 5 +
10 files changed, 552 insertions(+), 24 deletions(-)
diff --git a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
index 1ec2a6e..24edb28 100644
--- a/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
+++ b/runners/flink/src/main/java/org/apache/beam/runners/flink/FlinkJobInvoker.java
@@ -20,13 +20,14 @@ package org.apache.beam.runners.flink;
import static org.apache.beam.runners.core.construction.PipelineResources.detectClassPathResourcesToStage;
import java.io.IOException;
-import java.util.List;
import java.util.UUID;
import javax.annotation.Nullable;
import org.apache.beam.model.pipeline.v1.RunnerApi;
import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
import org.apache.beam.runners.fnexecution.jobsubmission.JobInvocation;
import org.apache.beam.runners.fnexecution.jobsubmission.JobInvoker;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineJarCreator;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineRunner;
import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
import org.apache.beam.sdk.options.PortablePipelineOptions;
import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.Struct;
@@ -63,7 +64,6 @@ public class FlinkJobInvoker extends JobInvoker {
String invocationId =
String.format("%s_%s", flinkOptions.getJobName(), UUID.randomUUID().toString());
- LOG.info("Invoking job {}", invocationId);
if (FlinkPipelineOptions.AUTO.equals(flinkOptions.getFlinkMaster())) {
flinkOptions.setFlinkMaster(serverConfig.getFlinkMasterUrl());
@@ -74,16 +74,23 @@ public class FlinkJobInvoker extends JobInvoker {
portableOptions.setSdkWorkerParallelism(serverConfig.getSdkWorkerParallelism());
}
+ PortablePipelineRunner pipelineRunner;
+ if (portableOptions.getOutputExecutablePath() == null
+ || portableOptions.getOutputExecutablePath().isEmpty()) {
+ pipelineRunner =
+ new FlinkPipelineRunner(
+ flinkOptions,
+ serverConfig.getFlinkConfDir(),
+ detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
+ } else {
+ pipelineRunner = new PortablePipelineJarCreator(FlinkPipelineRunner.class);
+ }
+
flinkOptions.setRunner(null);
+ LOG.info("Invoking job {} with pipeline runner {}", invocationId, pipelineRunner);
return createJobInvocation(
- invocationId,
- retrievalToken,
- executorService,
- pipeline,
- flinkOptions,
- serverConfig.getFlinkConfDir(),
- detectClassPathResourcesToStage(FlinkJobInvoker.class.getClassLoader()));
+ invocationId, retrievalToken, executorService, pipeline, flinkOptions, pipelineRunner);
}
static JobInvocation createJobInvocation(
@@ -92,16 +99,13 @@ public class FlinkJobInvoker extends JobInvoker {
ListeningExecutorService executorService,
RunnerApi.Pipeline pipeline,
FlinkPipelineOptions flinkOptions,
- @Nullable String confDir,
- List<String> filesToStage) {
+ PortablePipelineRunner pipelineRunner) {
JobInfo jobInfo =
JobInfo.create(
invocationId,
flinkOptions.getJobName(),
retrievalToken,
PipelineOptionsTranslation.toProto(flinkOptions));
- FlinkPipelineRunner pipelineRunner =
- new FlinkPipelineRunner(flinkOptions, confDir, filesToStage);
return new JobInvocation(jobInfo, executorService, pipeline, pipelineRunner);
}
}
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
index 0bc14fb..4315e62 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/FlinkSavepointTest.java
@@ -206,6 +206,7 @@ public class FlinkSavepointTest implements Serializable {
ListeningExecutorService executorService =
MoreExecutors.listeningDecorator(Executors.newFixedThreadPool(1));
+ FlinkPipelineOptions pipelineOptions = pipeline.getOptions().as(FlinkPipelineOptions.class);
try {
JobInvocation jobInvocation =
FlinkJobInvoker.createJobInvocation(
@@ -213,9 +214,8 @@ public class FlinkSavepointTest implements Serializable {
"none",
executorService,
pipelineProto,
- pipeline.getOptions().as(FlinkPipelineOptions.class),
- null,
- Collections.emptyList());
+ pipelineOptions,
+ new FlinkPipelineRunner(pipelineOptions, null, Collections.emptyList()));
jobInvocation.start();
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
index 15be023..10db32c 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableExecutionTest.java
@@ -148,8 +148,8 @@ public class PortableExecutionTest implements Serializable {
flinkJobExecutor,
pipelineProto,
options.as(FlinkPipelineOptions.class),
- null,
- Collections.emptyList());
+ new FlinkPipelineRunner(
+ options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
jobInvocation.start();
while (jobInvocation.getState() != Enum.DONE) {
Thread.sleep(1000);
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
index 7096d7d..91f243b 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableStateExecutionTest.java
@@ -202,8 +202,8 @@ public class PortableStateExecutionTest implements Serializable {
flinkJobExecutor,
pipelineProto,
options.as(FlinkPipelineOptions.class),
- null,
- Collections.emptyList());
+ new FlinkPipelineRunner(
+ options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
jobInvocation.start();
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
index cb16b27..9cddcd6 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/PortableTimersExecutionTest.java
@@ -189,8 +189,8 @@ public class PortableTimersExecutionTest implements Serializable {
flinkJobExecutor,
pipelineProto,
options.as(FlinkPipelineOptions.class),
- null,
- Collections.emptyList());
+ new FlinkPipelineRunner(
+ options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
jobInvocation.start();
while (jobInvocation.getState() != Enum.DONE) {
diff --git a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
index bd72d29..40d621f 100644
--- a/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
+++ b/runners/flink/src/test/java/org/apache/beam/runners/flink/ReadSourcePortableTest.java
@@ -110,8 +110,8 @@ public class ReadSourcePortableTest implements Serializable {
flinkJobExecutor,
pipelineProto,
options.as(FlinkPipelineOptions.class),
- null,
- Collections.emptyList());
+ new FlinkPipelineRunner(
+ options.as(FlinkPipelineOptions.class), null, Collections.emptyList()));
jobInvocation.start();
while (jobInvocation.getState() != Enum.DONE) {
Thread.sleep(100);
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java
new file mode 100644
index 0000000..0314fcd
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreator.java
@@ -0,0 +1,293 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.channels.Channels;
+import java.nio.channels.WritableByteChannel;
+import java.nio.charset.StandardCharsets;
+import java.util.Enumeration;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.jar.Attributes;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactChunk;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetArtifactRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestRequest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest.Location;
+import org.apache.beam.model.jobmanagement.v1.ArtifactRetrievalServiceGrpc;
+import org.apache.beam.model.jobmanagement.v1.ArtifactRetrievalServiceGrpc.ArtifactRetrievalServiceBlockingStub;
+import org.apache.beam.model.jobmanagement.v1.JobApi;
+import org.apache.beam.model.pipeline.v1.RunnerApi.Pipeline;
+import org.apache.beam.runners.core.construction.PipelineOptionsTranslation;
+import org.apache.beam.runners.core.construction.PipelineResources;
+import org.apache.beam.runners.fnexecution.GrpcFnServer;
+import org.apache.beam.runners.fnexecution.InProcessServerFactory;
+import org.apache.beam.runners.fnexecution.artifact.BeamFileSystemArtifactRetrievalService;
+import org.apache.beam.runners.fnexecution.provisioning.JobInfo;
+import org.apache.beam.sdk.fn.test.InProcessManagedChannelFactory;
+import org.apache.beam.sdk.metrics.MetricResults;
+import org.apache.beam.sdk.options.PortablePipelineOptions;
+import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.MessageOrBuilder;
+import org.apache.beam.vendor.grpc.v1p21p0.com.google.protobuf.util.JsonFormat;
+import org.apache.beam.vendor.grpc.v1p21p0.io.grpc.ManagedChannel;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.annotations.VisibleForTesting;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.base.Preconditions;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.apache.commons.compress.utils.IOUtils;
+import org.joda.time.Duration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * {@link PortablePipelineRunner} that bundles the input pipeline along with all dependencies,
+ * artifacts, etc. required to run the pipeline into a jar that can be executed later.
+ *
+ * <p>Each {@link PortablePipelineJarCreator} instance is not threadsafe; a new instance is expected
+ * to be constructed and {@link #run} once per job.
+ */
+public class PortablePipelineJarCreator implements PortablePipelineRunner {
+ private static final Logger LOG = LoggerFactory.getLogger(PortablePipelineJarCreator.class);
+
+ private final Class mainClass;
+
+ @VisibleForTesting JarOutputStream outputStream;
+ /** Wrapper over {@link #outputStream}. */
+ @VisibleForTesting WritableByteChannel outputChannel;
+
+ public PortablePipelineJarCreator(Class mainClass) {
+ this.mainClass = mainClass;
+ }
+
+ /**
+ * <em>Does not actually run the pipeline.</em> Instead bundles the input pipeline along with all
+ * dependencies, artifacts, etc. required to run the pipeline into a jar that can be executed
+ * later.
+ */
+ @Override
+ public PortablePipelineResult run(Pipeline pipeline, JobInfo jobInfo) throws Exception {
+ PortablePipelineOptions pipelineOptions =
+ PipelineOptionsTranslation.fromProto(jobInfo.pipelineOptions())
+ .as(PortablePipelineOptions.class);
+
+ File outputFile = new File(pipelineOptions.getOutputExecutablePath());
+ LOG.info("Creating jar {}", outputFile.getAbsolutePath());
+ outputStream = new JarOutputStream(new FileOutputStream(outputFile), createManifest(mainClass));
+ outputChannel = Channels.newChannel(outputStream);
+ writeClassPathResources(mainClass.getClassLoader());
+ writeAsJson(pipeline, PortablePipelineJarUtils.PIPELINE_PATH);
+ writeAsJson(
+ PipelineOptionsTranslation.toProto(pipelineOptions),
+ PortablePipelineJarUtils.PIPELINE_OPTIONS_PATH);
+ writeArtifacts(jobInfo.retrievalToken());
+ // Closing the channel also closes the underlying stream.
+ outputChannel.close();
+
+ LOG.info("Jar {} created successfully.", outputFile.getAbsolutePath());
+ return new JarCreatorPipelineResult();
+ }
+
+ @VisibleForTesting
+ Manifest createManifest(Class mainClass) {
+ Manifest manifest = new Manifest();
+ manifest.getMainAttributes().put(Attributes.Name.MANIFEST_VERSION, "1.0");
+ boolean classHasMainMethod = false;
+ try {
+ Class returnType = mainClass.getMethod("main", String[].class).getReturnType();
+ if (returnType == Void.TYPE) {
+ classHasMainMethod = true;
+ } else {
+ LOG.warn(
+ "No Main-Class will be set in jar because main method in {} returns {}, expected void",
+ mainClass,
+ returnType);
+ }
+ } catch (NoSuchMethodException e) {
+ LOG.warn("No Main-Class will be set in jar because {} lacks a main method.", mainClass);
+ }
+ if (classHasMainMethod) {
+ manifest.getMainAttributes().put(Name.MAIN_CLASS, mainClass.getName());
+ }
+ return manifest;
+ }
+
+ /** Copy resources from {@code classLoader} to {@link #outputStream}. */
+ private void writeClassPathResources(ClassLoader classLoader) throws IOException {
+ List<String> classPathResources =
+ PipelineResources.detectClassPathResourcesToStage(classLoader);
+ Preconditions.checkArgument(
+ classPathResources.size() == 1, "Expected exactly one jar on " + classLoader.toString());
+ copyResourcesFromJar(new JarFile(classPathResources.get(0)));
+ }
+
+ /** Copy resources from {@code inputJar} to {@link #outputStream}. */
+ @VisibleForTesting
+ protected void copyResourcesFromJar(JarFile inputJar) throws IOException {
+ Enumeration<JarEntry> inputJarEntries = inputJar.entries();
+ // The zip spec allows multiple files with the same name; the Java zip libraries do not.
+ // Keep track of the files we've already written to filter out duplicates.
+ // Also, ignore the old manifest; we want to write our own.
+ Set<String> previousEntryNames =
+ new HashSet<>(
+ ImmutableList.of(
+ JarFile.MANIFEST_NAME,
+ PortablePipelineJarUtils.ARTIFACT_MANIFEST_PATH,
+ PortablePipelineJarUtils.PIPELINE_PATH,
+ PortablePipelineJarUtils.PIPELINE_OPTIONS_PATH));
+ while (inputJarEntries.hasMoreElements()) {
+ JarEntry inputJarEntry = inputJarEntries.nextElement();
+ InputStream inputStream = inputJar.getInputStream(inputJarEntry);
+ String entryName = inputJarEntry.getName();
+ if (previousEntryNames.contains(entryName)) {
+ LOG.debug("Skipping duplicated file {}", entryName);
+ } else {
+ JarEntry outputJarEntry = new JarEntry(inputJarEntry);
+ outputStream.putNextEntry(outputJarEntry);
+ LOG.trace("Copying jar entry {}", inputJarEntry);
+ IOUtils.copy(inputStream, outputStream);
+ previousEntryNames.add(entryName);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ interface ArtifactRetriever {
+ GetManifestResponse getManifest(GetManifestRequest request);
+
+ Iterator<ArtifactChunk> getArtifact(GetArtifactRequest request);
+ }
+
+ /**
+ * Copy all artifacts retrievable via the {@link ArtifactRetrievalServiceBlockingStub} to the
+ * {@code outputStream}.
+ *
+ * @return A {@link ProxyManifest} pointing to the artifacts' location in the output jar.
+ */
+ @VisibleForTesting
+ ProxyManifest copyStagedArtifacts(String retrievalToken, ArtifactRetriever retrievalServiceStub)
+ throws IOException {
+ GetManifestRequest manifestRequest =
+ GetManifestRequest.newBuilder().setRetrievalToken(retrievalToken).build();
+ ArtifactApi.Manifest manifest = retrievalServiceStub.getManifest(manifestRequest).getManifest();
+ // Create a new proxy manifest to locate artifacts at jar runtime.
+ ProxyManifest.Builder proxyManifestBuilder = ProxyManifest.newBuilder().setManifest(manifest);
+ for (ArtifactMetadata artifact : manifest.getArtifactList()) {
+ String outputPath =
+ PortablePipelineJarUtils.ARTIFACT_FOLDER_PATH + "/" + UUID.randomUUID().toString();
+ LOG.trace("Copying artifact {} to {}", artifact.getName(), outputPath);
+ proxyManifestBuilder.addLocation(
+ Location.newBuilder().setName(artifact.getName()).setUri("/" + outputPath).build());
+ outputStream.putNextEntry(new JarEntry(outputPath));
+ GetArtifactRequest artifactRequest =
+ GetArtifactRequest.newBuilder()
+ .setRetrievalToken(retrievalToken)
+ .setName(artifact.getName())
+ .build();
+ Iterator<ArtifactChunk> artifactResponse = retrievalServiceStub.getArtifact(artifactRequest);
+ while (artifactResponse.hasNext()) {
+ artifactResponse.next().getData().writeTo(outputStream);
+ }
+ }
+ return proxyManifestBuilder.build();
+ }
+
+ /**
+ * Uses {@link BeamFileSystemArtifactRetrievalService} to fetch artifacts, then writes the
+ * artifacts to {@code outputStream}. Include a {@link ProxyManifest} to locate artifacts later.
+ */
+ private void writeArtifacts(String retrievalToken) throws Exception {
+ try (GrpcFnServer artifactServer =
+ GrpcFnServer.allocatePortAndCreateFor(
+ BeamFileSystemArtifactRetrievalService.create(), InProcessServerFactory.create())) {
+ ManagedChannel grpcChannel =
+ InProcessManagedChannelFactory.create()
+ .forDescriptor(artifactServer.getApiServiceDescriptor());
+ ArtifactRetrievalServiceBlockingStub retrievalServiceStub =
+ ArtifactRetrievalServiceGrpc.newBlockingStub(grpcChannel);
+ ProxyManifest proxyManifest =
+ copyStagedArtifacts(
+ retrievalToken,
+ new ArtifactRetriever() {
+ @Override
+ public GetManifestResponse getManifest(GetManifestRequest request) {
+ return retrievalServiceStub.getManifest(request);
+ }
+
+ @Override
+ public Iterator<ArtifactChunk> getArtifact(GetArtifactRequest request) {
+ return retrievalServiceStub.getArtifact(request);
+ }
+ });
+ writeAsJson(proxyManifest, PortablePipelineJarUtils.ARTIFACT_MANIFEST_PATH);
+ grpcChannel.shutdown();
+ }
+ }
+
+ /** Helper method for writing {@code message} in UTF-8 JSON format. */
+ private void writeAsJson(MessageOrBuilder message, String outputPath) throws IOException {
+ outputStream.putNextEntry(new JarEntry(outputPath));
+ outputChannel.write(StandardCharsets.UTF_8.encode(JsonFormat.printer().print(message)));
+ }
+
+ private static class JarCreatorPipelineResult implements PortablePipelineResult {
+
+ @Override
+ public State getState() {
+ return State.DONE;
+ }
+
+ @Override
+ public State cancel() {
+ return State.DONE;
+ }
+
+ @Override
+ public State waitUntilFinish(Duration duration) {
+ return State.DONE;
+ }
+
+ @Override
+ public State waitUntilFinish() {
+ return State.DONE;
+ }
+
+ @Override
+ public MetricResults metrics() {
+ throw new UnsupportedOperationException("Jar creation does not yield metrics.");
+ }
+
+ @Override
+ public JobApi.MetricResults portableMetrics() throws UnsupportedOperationException {
+ return JobApi.MetricResults.getDefaultInstance();
+ }
+ }
+}
diff --git a/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java
new file mode 100644
index 0000000..af712ad
--- /dev/null
+++ b/runners/java-fn-execution/src/main/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarUtils.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+/**
+ * Contains common code for writing and reading portable pipeline jars.
+ *
+ * <p>Jar layout:
+ *
+ * <ul>
+ * <li>META-INF/
+ * <ul>
+ * <li>MANIFEST.MF
+ * </ul>
+ * <li>BEAM-PIPELINE/
+ * <ul>
+ * <li>pipeline.json
+ * <li>pipeline-options.json
+ * </ul>
+ * <li>BEAM-ARTIFACT-STAGING/
+ * <ul>
+ * <li>artifact-manifest.json
+ * <li>artifacts/
+ * <ul>
+ * <li>...artifact files...
+ * </ul>
+ * </ul>
+ * <li>...Java classes...
+ * </ul>
+ */
+public abstract class PortablePipelineJarUtils {
+ private static final String ARTIFACT_STAGING_FOLDER_PATH = "BEAM-ARTIFACT-STAGING";
+ static final String ARTIFACT_FOLDER_PATH = ARTIFACT_STAGING_FOLDER_PATH + "/artifacts";
+ private static final String PIPELINE_FOLDER_PATH = "BEAM-PIPELINE";
+ static final String ARTIFACT_MANIFEST_PATH =
+ ARTIFACT_STAGING_FOLDER_PATH + "/artifact-manifest.json";
+ static final String PIPELINE_PATH = PIPELINE_FOLDER_PATH + "/pipeline.json";
+ static final String PIPELINE_OPTIONS_PATH = PIPELINE_FOLDER_PATH + "/pipeline-options.json";
+}
diff --git a/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java
new file mode 100644
index 0000000..458a456
--- /dev/null
+++ b/runners/java-fn-execution/src/test/java/org/apache/beam/runners/fnexecution/jobsubmission/PortablePipelineJarCreatorTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.beam.runners.fnexecution.jobsubmission;
+
+import static org.hamcrest.Matchers.containsInAnyOrder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.MockitoAnnotations.initMocks;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.Serializable;
+import java.nio.channels.WritableByteChannel;
+import java.util.Collections;
+import java.util.List;
+import java.util.jar.Attributes.Name;
+import java.util.jar.JarEntry;
+import java.util.jar.JarFile;
+import java.util.jar.JarInputStream;
+import java.util.jar.JarOutputStream;
+import java.util.jar.Manifest;
+import java.util.stream.Collectors;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ArtifactMetadata;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.GetManifestResponse;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest;
+import org.apache.beam.model.jobmanagement.v1.ArtifactApi.ProxyManifest.Location;
+import org.apache.beam.runners.fnexecution.jobsubmission.PortablePipelineJarCreator.ArtifactRetriever;
+import org.apache.beam.vendor.guava.v26_0_jre.com.google.common.collect.ImmutableList;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+import org.mockito.Mock;
+
+/** Unit tests for {@link PortablePipelineJarCreator}. */
+@RunWith(JUnit4.class)
+public class PortablePipelineJarCreatorTest implements Serializable {
+
+ @Mock private JarFile inputJar;
+ @Mock private JarOutputStream outputStream;
+ @Mock private WritableByteChannel outputChannel;
+ @Mock private ArtifactRetriever retrievalServiceStub;
+ private PortablePipelineJarCreator jarCreator;
+
+ @Before
+ public void setup() throws IOException {
+ initMocks(this);
+ JarInputStream emptyInputStream = new JarInputStream(new ByteArrayInputStream(new byte[0]));
+ when(inputJar.getInputStream(any())).thenReturn(emptyInputStream);
+ when(retrievalServiceStub.getArtifact(any())).thenReturn(Collections.emptyIterator());
+ jarCreator = new PortablePipelineJarCreator(null);
+ jarCreator.outputStream = outputStream;
+ jarCreator.outputChannel = outputChannel;
+ }
+
+ @Test
+ public void testCopyResourcesFromJar_copiesResources() throws IOException {
+ List<JarEntry> entries =
+ ImmutableList.of(new JarEntry("foo"), new JarEntry("bar"), new JarEntry("baz"));
+ when(inputJar.entries()).thenReturn(Collections.enumeration(entries));
+ jarCreator.copyResourcesFromJar(inputJar);
+ verify(outputStream, times(3)).putNextEntry(any());
+ }
+
+ @Test
+ public void testCopyResourcesFromJar_ignoresManifest() throws IOException {
+ List<JarEntry> manifestEntry = ImmutableList.of(new JarEntry(JarFile.MANIFEST_NAME));
+ when(inputJar.entries()).thenReturn(Collections.enumeration(manifestEntry));
+ jarCreator.copyResourcesFromJar(inputJar);
+ verify(outputStream, never()).putNextEntry(any());
+ }
+
+ @Test
+ public void testCopyResourcesFromJar_ignoresDuplicates() throws IOException {
+ List<JarEntry> duplicateEntries = ImmutableList.of(new JarEntry("foo"), new JarEntry("foo"));
+ when(inputJar.entries()).thenReturn(Collections.enumeration(duplicateEntries));
+ jarCreator.copyResourcesFromJar(inputJar);
+ verify(outputStream, times(1)).putNextEntry(any());
+ }
+
+ @Test
+ public void testCopyStagedArtifacts_returnsProxyManifest() throws IOException {
+ ArtifactMetadata artifact1 = ArtifactMetadata.newBuilder().setName("foo").build();
+ ArtifactMetadata artifact2 = ArtifactMetadata.newBuilder().setName("bar").build();
+ List<ArtifactMetadata> artifacts = ImmutableList.of(artifact1, artifact2);
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder().addAllArtifact(artifacts).build();
+ when(retrievalServiceStub.getManifest(any()))
+ .thenReturn(GetManifestResponse.newBuilder().setManifest(manifest).build());
+
+ ProxyManifest proxyManifest =
+ jarCreator.copyStagedArtifacts("retrievalToken", retrievalServiceStub);
+
+ assertEquals(manifest, proxyManifest.getManifest());
+ List<String> outputArtifactNames =
+ proxyManifest.getLocationList().stream()
+ .map(Location::getName)
+ .collect(Collectors.toList());
+ assertThat(outputArtifactNames, containsInAnyOrder("foo", "bar"));
+ }
+
+ @Test
+ public void testCopyStagedArtifacts_copiesArtifacts() throws IOException {
+ ArtifactMetadata artifact1 = ArtifactMetadata.newBuilder().setName("foo").build();
+ ArtifactMetadata artifact2 = ArtifactMetadata.newBuilder().setName("bar").build();
+ List<ArtifactMetadata> artifacts = ImmutableList.of(artifact1, artifact2);
+ ArtifactApi.Manifest manifest =
+ ArtifactApi.Manifest.newBuilder().addAllArtifact(artifacts).build();
+ when(retrievalServiceStub.getManifest(any()))
+ .thenReturn(GetManifestResponse.newBuilder().setManifest(manifest).build());
+
+ jarCreator.copyStagedArtifacts("retrievalToken", retrievalServiceStub);
+
+ verify(outputStream, times(2)).putNextEntry(any());
+ }
+
+ private static class FakePipelineRunnner {
+ public static void main(String[] args) {
+ System.out.println("Hello world");
+ }
+ }
+
+ @Test
+ public void testCreateManifest_withMainMethod() {
+ Manifest manifest = jarCreator.createManifest(FakePipelineRunnner.class);
+ assertEquals(
+ FakePipelineRunnner.class.getName(),
+ manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+ }
+
+ private static class EmptyPipelineRunner {}
+
+ @Test
+ public void testCreateManifest_withoutMainMethod() {
+ Manifest manifest = jarCreator.createManifest(EmptyPipelineRunner.class);
+ assertNull(manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+ }
+
+ private static class EvilPipelineRunner {
+ public static int main(String[] args) {
+ return 0;
+ }
+ }
+
+ @Test
+ public void testCreateManifest_withInvalidMainMethod() {
+ Manifest manifest = jarCreator.createManifest(EvilPipelineRunner.class);
+ assertNull(manifest.getMainAttributes().getValue(Name.MAIN_CLASS));
+ }
+}
diff --git a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
index 05f821d..e28cc80 100644
--- a/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
+++ b/sdks/java/core/src/main/java/org/apache/beam/sdk/options/PortablePipelineOptions.java
@@ -86,4 +86,9 @@ public interface PortablePipelineOptions extends PipelineOptions {
int getEnvironmentExpirationMillis();
void setEnvironmentExpirationMillis(int environmentExpirationMillis);
+
+ @Description("The output path for the executable file to be created.")
+ String getOutputExecutablePath();
+
+ void setOutputExecutablePath(String outputExecutablePath);
}