You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by mb...@apache.org on 2022/10/27 19:01:08 UTC
[flink] 09/34: [apple] Base and Hadoop image publish in Rio
This is an automated email from the ASF dual-hosted git repository.
mbalassi pushed a commit to branch release-1.15.2.4-acs
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 9b247b78773318b51e17089479acd54c7e913077
Author: Marton Balassi <ma...@apple.com>
AuthorDate: Tue Feb 1 17:11:20 2022 +0100
[apple] Base and Hadoop image publish in Rio
---
.gitignore | 1 +
apple/docker/{Dockerfile => Dockerfile.base} | 2 +-
apple/docker/{Dockerfile => Dockerfile.hadoop} | 29 +++++++++++++++-
apple/docker/README.md | 46 ++++++++++++++++++++++++--
rio.yml | 35 ++++++++------------
5 files changed, 87 insertions(+), 26 deletions(-)
diff --git a/.gitignore b/.gitignore
index c51b69a6412..420f8e94213 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,3 +63,4 @@ docs/go.mod
docs/go.sum
docs/.hugo_build.lock
apple/docker/flink.tgz
+apple/docker/hadoop-*.tar.gz
diff --git a/apple/docker/Dockerfile b/apple/docker/Dockerfile.base
similarity index 98%
copy from apple/docker/Dockerfile
copy to apple/docker/Dockerfile.base
index 0ae51096898..e5322edca36 100644
--- a/apple/docker/Dockerfile
+++ b/apple/docker/Dockerfile.base
@@ -3,7 +3,7 @@ FROM docker.apple.com/base-images/ubi8-minimal/java11-runtime
USER root
# Install dependencies
RUN set -ex; \
- microdnf install -y wget gnupg2 curl tar hostname gettext shadow-utils findutils
+ microdnf install -y wget gnupg2 curl tar hostname gettext shadow-utils findutils gzip bzip2
# Suboptimal fix for missing libjemalloc.so from LD_PRELOAD
RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
diff --git a/apple/docker/Dockerfile b/apple/docker/Dockerfile.hadoop
similarity index 59%
rename from apple/docker/Dockerfile
rename to apple/docker/Dockerfile.hadoop
index 0ae51096898..ccd35ae82f7 100644
--- a/apple/docker/Dockerfile
+++ b/apple/docker/Dockerfile.hadoop
@@ -3,7 +3,7 @@ FROM docker.apple.com/base-images/ubi8-minimal/java11-runtime
USER root
# Install dependencies
RUN set -ex; \
- microdnf install -y wget gnupg2 curl tar hostname gettext shadow-utils findutils
+ microdnf install -y wget gnupg2 curl tar hostname gettext shadow-utils findutils gzip bzip2
# Suboptimal fix for missing libjemalloc.so from LD_PRELOAD
RUN rpm -ivh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm
@@ -48,6 +48,33 @@ RUN set -ex; \
\
chown -R flink:flink .;
+ENV HADOOP_USER hdfs
+ENV HADOOP_PREFIX /usr/local/hadoop-3.3.1
+ENV HADOOP_COMMON_HOME /usr/local/hadoop-3.3.1
+ENV HADOOP_HDFS_HOME /usr/local/hadoop-3.3.1
+ENV HADOOP_CONF_DIR /usr/local/hadoop-3.3.1/etc/hadoop
+
+# Download hadoop
+COPY hadoop-3.3.1.tar.gz /usr/local/
+RUN set -ex; \
+ cd /usr/local/; \
+ tar -xvf hadoop-3.3.1.tar.gz; \
+ rm hadoop-3.3.1.tar.gz
+
+RUN ln -s /usr/local/hadoop-3.1.1 /usr/local/hadoop \
+&& groupadd -r hadoop \
+&& groupadd -r $HADOOP_USER && useradd -r -g $HADOOP_USER -G hadoop $HADOOP_USER
+
+# RUN mkdir -p $HADOOP_CONF_DIR
+
+# Setup permissions and ownership
+RUN chown -R $HADOOP_USER:hadoop /usr/local/hadoop-3.3.1 # && chmod -R 775 $HADOOP_CONF_DIR
+
+# Set up hadoop user and bin path
+ENV HADOOP_USER_NAME $HADOOP_USER
+ENV PATH="${HADOOP_PREFIX}/bin:${PATH}"
+ENV HADOOP_CLASSPATH="/usr/local/hadoop-3.3.1/etc/hadoop:/usr/local/hadoop-3.3.1/share/hadoop/common/lib/*:/usr/local/hadoop-3.3.1/share/hadoop/common/*:/usr/local/hadoop-3.3.1/share/hadoop/hdfs:/usr/local/hadoop-3.3.1/share/hadoop/hdfs/lib/*:/usr/local/hadoop-3.3.1/share/hadoop/hdfs/*:/usr/local/hadoop-3.3.1/share/hadoop/mapreduce/*:/usr/local/hadoop-3.3.1/share/hadoop/yarn:/usr/local/hadoop-3.3.1/share/hadoop/yarn/lib/*:/usr/local/hadoop-3.3.1/share/hadoop/yarn/*"
+
# Configure container
COPY docker-entrypoint.sh /
ENTRYPOINT ["/docker-entrypoint.sh"]
diff --git a/apple/docker/README.md b/apple/docker/README.md
index 4f9bace25ef..248187c1cd6 100644
--- a/apple/docker/README.md
+++ b/apple/docker/README.md
@@ -1,4 +1,44 @@
-# apple / flink
+# Apple Flink docker images
-For this image build you need the flink distribution being available on a local web server as explained in the main readme in the repo root.
-The difference is that this image is based on the apple internal jdk base image.
+These image builds follow the upstream [flink-docker](https://github.com/apache/flink-docker) repository,
+but use Apple internal base images.
+
+## Prerequisites
+
+Build Flink, then compress the distro and copy it here:
+
+```bash
+pushd ../../flink-dist/target/flink-*-bin
+tar czf flink.tgz flink-*-SNAPSHOT
+cp flink.tgz ../../../apple/docker
+popd
+```
+
+For the Hadoop image download the Hadoop distro:
+
+```bash
+wget https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz
+```
+
+## Build
+
+Create the base image as follows, optionally replace it with your own version tag.
+
+```bash
+docker build . -f Dockerfile.base -t docker.apple.com/acs-flink/flink:1.14.3.1-apple
+```
+
+Create the Hadoop image as follows:
+
+```bash
+docker build . -f Dockerfile.hadoop -t docker.apple.com/acs-flink/flink:1.14.3.1-apple-hadoop
+```
+
+These images are regularly published via our Rio CI pipeline.
+
+## Verification
+
+You may verify your image being able to start a jobmanager:
+```bash
+docker run -it docker.apple.com/acs-flink/flink:1.14.3.1-apple jobmanager
+```
diff --git a/rio.yml b/rio.yml
index d007b3738df..de822252866 100644
--- a/rio.yml
+++ b/rio.yml
@@ -1,12 +1,6 @@
schemaVersion: 2.0
timeout: 60
-# Set defaults for all pipelines.
-package:
- dockerfile:
- perApplication: false
- dockerfilePath: Dockerfile
-
pipelines:
# Pipeline to build Flink from Apple branch.
- name: publish-branch-1.14.3-apple
@@ -14,8 +8,6 @@ pipelines:
timeout: 120
machine:
baseImage: docker.apple.com/thomas-weise/flink-ci:latest
- trigger:
- gitPush: false
build:
template: freestyle:v4:publish
steps:
@@ -23,14 +15,15 @@ pipelines:
#- ls -la ~/.m2
#- mvn help:effective-settings -X -rf :flink-avro-confluent-registry
- rm -rf .staged-artifacts
- - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,docs-and-source -DskipTests -Dfast -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts
+ - mvn clean deploy -Dscala-2.12 -Phive-3.1.3.0-apple,s3-plugin,docs-and-source -DskipTests -Dfast -DaltDeploymentRepository=local-staging::default::file://`pwd`/.staged-artifacts
# see https://docs.aci.apple.com/rio/guide-to-rio/package-and-publish/freeform.html
# must exclude maven-metadata.xml
- ci stage-lib --exclude maven-metadata.xml --exclude *.md5 --exclude *.sha1 --many-many-artifacts ./.staged-artifacts/**
#- ci stage-lib "./target/artifacts/org/apache/flink/*/*/*.jar,org/apache/flink/"
#- ci stage-lib "./target/artifacts/org/apache/flink/*/*/*.pom,org/apache/flink/"
- # Prepare for docker image build
- - cd flink-dist/target/flink-*-bin/; tar cvzf tar cvf flink.tgz flink-*; cp flink.tgz /workspace/apple/docker/
+ - cd flink-dist/target/flink-*-bin/; tar cvzf flink.tgz flink-*; cp flink.tgz /workspace/apple/docker/; ls -la /workspace/apple/docker/
+ - cd /workspace/apple/docker; wget https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz; cd /workspace
package:
# enable for release (non -SNAPSHOT) builds
# release: true
@@ -38,27 +31,27 @@ pipelines:
- publish:
- repo: m2:oss-patched
dockerfile:
- - context: apple/docker/
+ - dockerfilePath: apple/docker/Dockerfile.base
+ context: apple/docker/
+ perApplication: false
version: "1.14.3.1-apple"
publish:
- repo: docker.apple.com/acs-flink/flink
+ - dockerfilePath: apple/docker/Dockerfile.hadoop
+ context: apple/docker/
+ perApplication: false
+ version: "1.14.3.1-apple-hadoop"
+ publish:
+ - repo: docker.apple.com/acs-flink/flink
- name: pull-request-branch-1.14.3-apple
branchName: branch-1.14.3-apple
- timeout: 120
+ timeout: 600
machine:
baseImage: docker.apple.com/thomas-weise/flink-ci:latest
- trigger:
- gitPush: false
build:
template: freestyle:v4:prb
steps:
- cd /workspace
- rm -rf .staged-artifacts
- - mvn clean deploy -Pdocs-and-source -DskipTests -Drat.skip -Dscala-2.12
- package:
- # enable for release (non -SNAPSHOT) builds
- # release: true
- freeform:
- - publish:
- - repo: m2:oss-patched
+ - mvn clean package -Phive-3.1.3.0-apple,s3-plugin -Dscala-2.12 -Dmaven.test.failure.ignore=true