You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ib...@apache.org on 2019/12/23 16:36:54 UTC
[beam] branch master updated: [BEAM-8296] containerize spark job
server
This is an automated email from the ASF dual-hosted git repository.
ibzib pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new c29214f [BEAM-8296] containerize spark job server
new 267d828 Merge pull request #10407 from ibzib/spark-container
c29214f is described below
commit c29214f5e3368026c51e9abd93280a2c7f910d66
Author: Kyle Weaver <kc...@google.com>
AuthorDate: Wed Oct 2 10:18:01 2019 -0700
[BEAM-8296] containerize spark job server
---
runners/spark/job-server/build.gradle | 2 +-
runners/spark/job-server/container/Dockerfile | 28 ++++++++++
runners/spark/job-server/container/build.gradle | 64 ++++++++++++++++++++++
.../spark/job-server/container/spark-job-server.sh | 29 ++++++++++
settings.gradle | 1 +
5 files changed, 123 insertions(+), 1 deletion(-)
diff --git a/runners/spark/job-server/build.gradle b/runners/spark/job-server/build.gradle
index 63ab04f..30dc24d 100644
--- a/runners/spark/job-server/build.gradle
+++ b/runners/spark/job-server/build.gradle
@@ -38,7 +38,7 @@ applyJavaNature(
def sparkRunnerProject = project.parent.path
-description = project(sparkRunnerProject).description + " :: Job Server"
+description = "Apache Beam :: Runners :: Spark :: Job Server"
configurations {
validatesPortableRunner
diff --git a/runners/spark/job-server/container/Dockerfile b/runners/spark/job-server/container/Dockerfile
new file mode 100644
index 0000000..11a5f0e
--- /dev/null
+++ b/runners/spark/job-server/container/Dockerfile
@@ -0,0 +1,28 @@
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+FROM openjdk:8
+MAINTAINER "Apache Beam <de...@beam.apache.org>"
+
+RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y libltdl7
+
+ADD beam-runners-spark-job-server.jar /opt/apache/beam/jars/
+ADD spark-job-server.sh /opt/apache/beam/
+
+WORKDIR /opt/apache/beam
+ENTRYPOINT ["./spark-job-server.sh"]
diff --git a/runners/spark/job-server/container/build.gradle b/runners/spark/job-server/container/build.gradle
new file mode 100644
index 0000000..c7244a3
--- /dev/null
+++ b/runners/spark/job-server/container/build.gradle
@@ -0,0 +1,64 @@
+import org.apache.beam.gradle.BeamModulePlugin
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Build a Docker image to bootstrap SparkJobServerDriver which requires a Java environment.
+ * Alternatively, it can also be bootstrapped through the runShadow goal
+ * or by directly running the generated JAR file.
+ */
+
+apply plugin: 'org.apache.beam.module'
+apply plugin: 'base'
+applyDockerNature()
+
+def sparkJobServerProject = project.parent.path
+
+description = "Apache Beam :: Runners :: Spark :: Job Server :: Container"
+
+configurations {
+ dockerDependency
+}
+
+dependencies {
+ dockerDependency project(path: sparkJobServerProject, configuration: "shadow")
+}
+
+task copyDockerfileDependencies(type: Copy) {
+ // Required Jars
+ from configurations.dockerDependency
+ rename 'beam-runners-spark-job-server.*.jar', 'beam-runners-spark-job-server.jar'
+ into "build"
+ // Entry script
+ from "spark-job-server.sh"
+ into "build"
+ // Dockerfile
+ from "Dockerfile"
+ into "build"
+}
+
+docker {
+ name containerImageName(name: 'spark-job-server',
+ root: project.rootProject.hasProperty(["docker-repository-root"]) ?
+ project.rootProject["docker-repository-root"] : "apachebeam")
+ files "./build/"
+}
+
+// Ensure that we build the required resources and copy and file dependencies from related projects
+dockerPrepare.dependsOn copyDockerfileDependencies
diff --git a/runners/spark/job-server/container/spark-job-server.sh b/runners/spark/job-server/container/spark-job-server.sh
new file mode 100755
index 0000000..e39c5ae
--- /dev/null
+++ b/runners/spark/job-server/container/spark-job-server.sh
@@ -0,0 +1,29 @@
+#!/bin/sh
+###############################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+###############################################################################
+
+### Just a simple script to bootstrap the SparkJobServerDriver
+### For the environment, see the Dockerfile
+
+# The following (forking to the background, then waiting) enables to use CTRL+C to kill the container.
+# We're PID 1 which doesn't handle signals. By forking the Java process to the background,
+# a PID > 1 is created which handles signals. After the command shuts down, the script and
+# thus the container will also exit.
+
+java -cp "jars/*" org.apache.beam.runners.spark.SparkJobServerDriver "$@" &
+wait
diff --git a/settings.gradle b/settings.gradle
index 55784ca..091f9b0 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -54,6 +54,7 @@ include ":runners:local-java"
include ":runners:portability:java"
include ":runners:spark"
include ":runners:spark:job-server"
+include ":runners:spark:job-server:container"
include ":runners:samza"
include ":runners:samza:job-server"
include ":sdks:go"