You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by mx...@apache.org on 2018/08/17 14:37:43 UTC
[beam] branch master updated: [BEAM-4130] Build Docker image for
Flink's JobServer
This is an automated email from the ASF dual-hosted git repository.
mxm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 102a972 [BEAM-4130] Build Docker image for Flink's JobServer
new d44a54d Merge pull request #6238 from mxm/job-server-startup
102a972 is described below
commit 102a97293af08db5d639406879c87c4f3a6ddc40
Author: Maximilian Michels <mx...@apache.org>
AuthorDate: Thu Aug 16 14:55:56 2018 +0200
[BEAM-4130] Build Docker image for Flink's JobServer
This adds a new Gradle module flink-job-server-container which builds a docker
image during the `docker` task. The image contains the FlinkJobServerDriver
which is the entry point for submitting Beam pipelines to the cluster.
The image can then be used to spawn a JobServer container when executing a Beam
pipeline, i.e. `p.run()`. The SDKs (Java/Python/Go) need to be updated to either
spawn up a container or use the address of a remote JobServer.
---
runners/flink/job-server-container/Dockerfile | 26 +++++++++++
runners/flink/job-server-container/build.gradle | 54 ++++++++++++++++++++++
.../flink/job-server-container/flink-job-server.sh | 29 ++++++++++++
settings.gradle | 2 +
4 files changed, 111 insertions(+)
diff --git a/runners/flink/job-server-container/Dockerfile b/runners/flink/job-server-container/Dockerfile
new file mode 100644
index 0000000..a9aff21
--- /dev/null
+++ b/runners/flink/job-server-container/Dockerfile
@@ -0,0 +1,26 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM openjdk:8
+MAINTAINER "Apache Beam <de...@beam.apache.org>"
+
+ADD target/beam-runners-flink_2.11-job-server.jar /opt/apache/beam/jars/
+ADD target/flink-job-server.sh /opt/apache/beam/
+
+WORKDIR /opt/apache/beam
+ENTRYPOINT ["./flink-job-server.sh"]
diff --git a/runners/flink/job-server-container/build.gradle b/runners/flink/job-server-container/build.gradle
new file mode 100644
index 0000000..4d5f533
--- /dev/null
+++ b/runners/flink/job-server-container/build.gradle
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Build a Docker image to bootstrap FlinkJobServerDriver which requires a Java environment.
+ * Alternatively, it can also be bootstrapped through :beam-runners-flink_2.11-job-server:runShadow
+ * or by directly running the generated JAR file.
+ */
+
+apply plugin: org.apache.beam.gradle.BeamModulePlugin
+applyDockerNature()
+
+description = "Apache Beam :: Runners :: Flink :: Job Server :: Container"
+
+configurations {
+ dockerDependency
+}
+
+dependencies {
+ dockerDependency project(path: ":beam-runners-flink_2.11-job-server", configuration: "shadow")
+}
+
+task copyDockerfileDependencies(type: Copy) {
+ // Required Jars
+ from configurations.dockerDependency
+ rename 'beam-runners-flink_2.11-job-server.*.jar', 'beam-runners-flink_2.11-job-server.jar'
+ into "build/target"
+ // Entry script
+ from file("./flink-job-server.sh")
+ into "build/target"
+}
+
+docker {
+ name containerImageName(name: "flink-job-server")
+ files "./build/"
+}
+
+// Ensure that we build the required resources and copy and file dependencies from related projects
+dockerPrepare.dependsOn copyDockerfileDependencies
diff --git a/runners/flink/job-server-container/flink-job-server.sh b/runners/flink/job-server-container/flink-job-server.sh
new file mode 100755
index 0000000..399e5e4
--- /dev/null
+++ b/runners/flink/job-server-container/flink-job-server.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+### Just a simple script to bootstrap the FlinkJobServerDriver
+### For the environment, see the Dockerfile
+
+# The following (forking to the background, then waiting) enables to use CTRL+C to kill the container.
+# We're PID 1 which doesn't handle signals. By forking the Java process to the background,
+# a PID > 1 is created which handles signals. After the command shuts down, the script and
+# thus the container will also exit.
+
+java -cp "jars/*" org.apache.beam.runners.flink.FlinkJobServerDriver "$@" &
+wait
diff --git a/settings.gradle b/settings.gradle
index 01ea889..a606b17 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -42,6 +42,8 @@ include "beam-runners-flink_2.11"
project(":beam-runners-flink_2.11").dir = file("runners/flink")
include "beam-runners-flink_2.11-job-server"
project(":beam-runners-flink_2.11-job-server").dir = file("runners/flink/job-server")
+include "beam-runners-flink_2.11-job-server-container"
+project(":beam-runners-flink_2.11-job-server-container").dir = file("runners/flink/job-server-container")
include "beam-runners-gcp-gcemd"
project(":beam-runners-gcp-gcemd").dir = file("runners/gcp/gcemd")
include "beam-runners-gcp-gcsproxy"