You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mx...@apache.org on 2016/10/21 12:20:51 UTC
flink git commit: [FLINK-4881][docker] remove shared volume for
configuration
Repository: flink
Updated Branches:
refs/heads/master a27451d8e -> 58a16c100
[FLINK-4881][docker] remove shared volume for configuration
README.md - Description of the bluemix specific steps
build.sh - modify permissions to make script executable by default
docker-compose-bluemix.yml - allow to specify image names with path to private BM registry
docker-compose.sh - find out path to docker registry, extend build time to 120 secs, then call docker-compose
docker-compose.yml - make use of container linking instead of shared volumes
docker-entrypoint.sh - use 'jobmanager' entry in /etc/hosts arising from container linking so that taskmanager can register itself with jobmanager
docker-compose.sh -> bluemix-docker-compose.sh - renamed on request
docker-entrypoint.sh - made sure the taskmanager's slots correlate to the number of CPUs
This closes #2667
Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/58a16c10
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/58a16c10
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/58a16c10
Branch: refs/heads/master
Commit: 58a16c1007c97809bdd46c93632abd90c0ce5c4b
Parents: a27451d
Author: Markus M�ller <ma...@de.ibm.com>
Authored: Thu Oct 20 10:08:34 2016 +0200
Committer: Maximilian Michels <mx...@apache.org>
Committed: Fri Oct 21 14:19:38 2016 +0200
----------------------------------------------------------------------
flink-contrib/docker-flink/README.md | 13 ++++++
.../docker-flink/bluemix-docker-compose.sh | 22 ++++++++++
.../docker-flink/docker-compose-bluemix.yml | 42 ++++++++++++++++++++
flink-contrib/docker-flink/docker-compose.yml | 12 ++++--
flink-contrib/docker-flink/docker-entrypoint.sh | 13 +++++-
5 files changed, 96 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/README.md
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/README.md b/flink-contrib/docker-flink/README.md
index 6ff3aea..c7d94bb 100644
--- a/flink-contrib/docker-flink/README.md
+++ b/flink-contrib/docker-flink/README.md
@@ -8,6 +8,19 @@ https://docs.docker.com/installation/
Install the most recent stable version of docker-compose
https://docs.docker.com/compose/install/
+## Bluemix PaaS
+
+If you want to build the image on Bluemix redirect the docker cli to the remote endpoints. There is plenty of documentation how
+to obtain a Bluemix account, so I'm not specifying details. Once you have it and log in using the cloud foundry CLI 'cf' you also
+need the container specific plugin 'ic'. With that you can get the URL to the remote docker host as well as the path to
+the certificates. If you search for "Logging in to the IBM Containers CLI plug-in" you get the details. <br>Here an example:<br>
+export DOCKER_HOST=tcp://containers-api.eu-gb.bluemix.net:8443<br>
+export DOCKER_CERT_PATH=/home/markus/.ice/certs/containers-api.eu-gb.bluemix.net/3c63cb44-86d8-4e89-9a40-f8f3f894a09f<br>
+export DOCKER_TLS_VERIFY=1<br>
+
+Now when proceeding to the next paragraph 'build' the docker commands build and run the image (provided you use ./docker-compose.sh instead of the native docker-compose command.
+Do not forget to allocate and bind a public IP address with the 'cf ic ip' set of commands afterwards.
+
# Build
Images are based on the official Java Alpine (OpenJDK 8) image. If you want to
http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/bluemix-docker-compose.sh
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/bluemix-docker-compose.sh b/flink-contrib/docker-flink/bluemix-docker-compose.sh
new file mode 100755
index 0000000..9d090a7
--- /dev/null
+++ b/flink-contrib/docker-flink/bluemix-docker-compose.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+export COMPOSE_HTTP_TIMEOUT=120
+export IMAGENAME="registry.eu-gb.bluemix.net/"`cf ic namespace get`"/flink"
+docker-compose -f docker-compose-bluemix.yml up -d
http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-compose-bluemix.yml
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-compose-bluemix.yml b/flink-contrib/docker-flink/docker-compose-bluemix.yml
new file mode 100644
index 0000000..0155f01
--- /dev/null
+++ b/flink-contrib/docker-flink/docker-compose-bluemix.yml
@@ -0,0 +1,42 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+# image specification requires the full path to your private bluemix image registry
+
+version: "2"
+services:
+ jobmanager:
+ #image example: registry.eu-gb.bluemix.net/markussaidhiho/flink
+ image: ${IMAGENAME}
+ container_name: "jobmanager"
+ expose:
+ - "6123"
+ ports:
+ - "48081:8081"
+ command: jobmanager
+
+ taskmanager:
+ image: ${IMAGENAME}
+ expose:
+ - "6121"
+ - "6122"
+ depends_on:
+ - jobmanager
+ command: taskmanager
+ links:
+ - "jobmanager:jobmanager"
http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-compose.yml
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-compose.yml b/flink-contrib/docker-flink/docker-compose.yml
index 08956c6..eaa30a6 100644
--- a/flink-contrib/docker-flink/docker-compose.yml
+++ b/flink-contrib/docker-flink/docker-compose.yml
@@ -20,16 +20,20 @@ version: "2"
services:
jobmanager:
image: flink
+ container_name: "jobmanager"
+ expose:
+ - "6123"
ports:
- "48081:8081"
command: jobmanager
- volumes:
- - /opt/flink/conf
taskmanager:
image: flink
+ expose:
+ - "6121"
+ - "6122"
depends_on:
- jobmanager
command: taskmanager
- volumes_from:
- - jobmanager:ro
+ links:
+ - "jobmanager:jobmanager"
http://git-wip-us.apache.org/repos/asf/flink/blob/58a16c10/flink-contrib/docker-flink/docker-entrypoint.sh
----------------------------------------------------------------------
diff --git a/flink-contrib/docker-flink/docker-entrypoint.sh b/flink-contrib/docker-flink/docker-entrypoint.sh
index 0da8809..a2ca0e9 100755
--- a/flink-contrib/docker-flink/docker-entrypoint.sh
+++ b/flink-contrib/docker-flink/docker-entrypoint.sh
@@ -20,11 +20,20 @@
if [ "$1" = "jobmanager" ]; then
echo "Starting Job Manager"
- sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -f`/g" $FLINK_HOME/conf/flink-conf.yaml
- sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c ^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+ #sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: `hostname -f`/g" $FLINK_HOME/conf/flink-conf.yaml
+
+ # make use of container linking and exploit the jobmanager entry in /etc/hosts
+ sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g" $FLINK_HOME/conf/flink-conf.yaml
+
echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
$FLINK_HOME/bin/jobmanager.sh start cluster
elif [ "$1" = "taskmanager" ]; then
+
+ # make use of container linking and exploit the jobmanager entry in /etc/hosts
+ sed -i -e "s/jobmanager.rpc.address: localhost/jobmanager.rpc.address: jobmanager/g" $FLINK_HOME/conf/flink-conf.yaml
+
+ sed -i -e "s/taskmanager.numberOfTaskSlots: 1/taskmanager.numberOfTaskSlots: `grep -c ^processor /proc/cpuinfo`/g" $FLINK_HOME/conf/flink-conf.yaml
+
echo "Starting Task Manager"
echo "config file: " && grep '^[^\n#]' $FLINK_HOME/conf/flink-conf.yaml
$FLINK_HOME/bin/taskmanager.sh start