You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ab...@apache.org on 2021/02/12 15:28:55 UTC
[tez] branch master updated: TEZ-4290: Dockerfile improvements
(#111) (Laszlo Bodor reviewed by Jonathan Turner Eagles)
This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new f88aa6f TEZ-4290: Dockerfile improvements (#111) (Laszlo Bodor reviewed by Jonathan Turner Eagles)
f88aa6f is described below
commit f88aa6f21980ba85160b6338515efc54e974c267
Author: Bodor Laszlo <bo...@gmail.com>
AuthorDate: Fri Feb 12 16:28:48 2021 +0100
TEZ-4290: Dockerfile improvements (#111) (Laszlo Bodor reviewed by Jonathan Turner Eagles)
---
build-tools/docker/Dockerfile | 368 +++++++++++++++++++----------------
build-tools/docker/tez_env_checks.sh | 117 -----------
2 files changed, 205 insertions(+), 280 deletions(-)
diff --git a/build-tools/docker/Dockerfile b/build-tools/docker/Dockerfile
index ae26e41..f51cc3b 100644
--- a/build-tools/docker/Dockerfile
+++ b/build-tools/docker/Dockerfile
@@ -1,4 +1,4 @@
-
+#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -15,219 +15,261 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Dockerfile for installing the necessary dependencies for building Hadoop.
-# See BUILDING.txt.
+###############
+#
+# Apache Yetus Dockerfile for Apache Tez
+# NOTE: This file is compatible with Docker BuildKit. It will work
+# with standard docker build, but it is a lot faster
+# if BuildKit is enabled.
+#
+###############
-FROM ubuntu:bionic
+FROM ubuntu:focal AS tezbase
WORKDIR /root
-
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-#####
-# Disable suggests/recommends
-#####
-RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
-RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
-
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
######
-# Install common dependencies from packages. Versions here are either
-# sufficient or irrelevant.
-#
-# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
-# Ubuntu Java. See Java section below!
+# Install some basic Apache Yetus requirements
+# some git repos need ssh-client so do it too
+# Adding libffi-dev for all the programming languages
+# that take advantage of it.
######
# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends \
- apt-utils \
- build-essential \
- bzip2 \
- clang \
- curl \
- doxygen \
- fuse \
- g++ \
- gcc \
- git \
- gnupg-agent \
- libbz2-dev \
- libfuse-dev \
- libprotobuf-dev \
- libprotoc-dev \
- libsasl2-dev \
- libsnappy-dev \
- libssl-dev \
- libtool \
- locales \
- make \
- pinentry-curses \
- pkg-config \
- python \
- python2.7 \
- python-pip \
- python-pkg-resources \
- python-setuptools \
- python-wheel \
- rsync \
- software-properties-common \
- sudo \
- valgrind \
- zlib1g-dev \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ apt-transport-https \
+ apt-utils \
+ ca-certificates \
+ curl \
+ dirmngr \
+ git \
+ gpg \
+ gpg-agent \
+ libffi-dev \
+ locales \
+ make \
+ pkg-config \
+ rsync \
+ software-properties-common \
+ ssh-client \
+ xz-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
+###
+# Set the locale
+###
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
-#######
-# OpenJDK 8
-#######
+####
+# Install GNU C/C++ (everything generally needs this)
+####
# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends openjdk-8-jdk libbcprov-java \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ g++ \
+ gcc \
+ libc-dev \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
+###
+# Install golang as part of base so we can do each
+# helper utility in parallel. go bins are typically
+# statically linked, so this is perfectly safe.
+###
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:longsleep/golang-backports \
+ && apt-get -q update \
+ && apt-get -q install --no-install-recommends -y golang-go \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
-######
-# Install cmake 3.1.0 (3.5.1 ships with Xenial)
-######
-RUN mkdir -p /opt/cmake \
- && curl -L -s -S \
- https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
- -o /opt/cmake.tar.gz \
- && tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
-ENV CMAKE_HOME /opt/cmake
-ENV PATH "${PATH}:/opt/cmake/bin"
+############
+# Fetch all of the non-conflicting bits in parallel
+#############
######
-# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
+# Install Google Protobuf 2.5.0
######
-# hadolint ignore=DL3003
+FROM tezbase AS protobuf250
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN mkdir -p /opt/protobuf-src \
&& curl -L -s -S \
https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
-o /opt/protobuf.tar.gz \
- && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
- && cd /opt/protobuf-src \
- && ./configure --prefix=/opt/protobuf \
- && make install \
- && cd /root \
- && rm -rf /opt/protobuf-src
-ENV PROTOBUF_HOME /opt/protobuf
-ENV PATH "${PATH}:/opt/protobuf/bin"
+ && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src
+WORKDIR /opt/protobuf-src
+RUN ./configure --prefix=/opt/protobuf \
+ && make install
+WORKDIR /root
+RUN rm -rf /opt/protobuf-src
-######
-# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
-######
+####
+# Install shellcheck (shell script lint)
+####
+FROM tezbase AS shellcheck
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+ https://github.com/koalaman/shellcheck/releases/download/v0.7.1/shellcheck-v0.7.1.linux.x86_64.tar.xz \
+ | tar --strip-components 1 --wildcards -xJf - '*/shellcheck' \
+ && chmod a+rx shellcheck \
+ && mv shellcheck /bin/shellcheck \
+ && shasum -a 512 /bin/shellcheck \
+ | awk '$1!="aae813283d49f18f95a205dca1c5184267d07534a08abc952ebea1958fee06f8a0207373b6770a083079ba875458ea9da443f2b9910a50dcd93b935048bc14f5" {exit(1)}'
+
+####
+# Install hadolint (dockerfile lint)
+####
+FROM tezbase AS hadolint
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+ https://github.com/hadolint/hadolint/releases/download/v1.18.0/hadolint-Linux-x86_64 \
+ -o /bin/hadolint \
+ && chmod a+rx /bin/hadolint \
+ && shasum -a 512 /bin/hadolint \
+ | awk '$1!="df27253d374c143a606483b07a26234ac7b4bca40b4eba53e79609c81aa70146e7d5c145f90dcec71d6d1aad1048b7d9d2de68d92284f48a735d04d19c5c5559" {exit(1)}'
+
+####
+# Install buf (protobuf lint)
+####
+FROM tezbase AS buf
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL \
+ https://github.com/bufbuild/buf/releases/download/v0.21.0/buf-Linux-x86_64.tar.gz \
+ -o buf.tar.gz \
+ && shasum -a 256 buf.tar.gz \
+ | awk '$1!="95aba62ac0ecc5a9120cc58c65cdcc85038633a816bddfe8398c5ae3b32803f1" {exit(1)}' \
+ && tar -xzf buf.tar.gz -C /usr/local --strip-components 1 \
+ && rm buf.tar.gz
+
+########
+#
+#
+# Content that needs to be installed in order due to packages...
+#
+#
+########
+
+FROM tezbase
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+####
+# Install java (first, since we want to dicate what form of Java)
+####
+
+####
+# OpenJDK 8
+####
# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends maven \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y openjdk-8-jdk-headless \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-ENV MAVEN_HOME /usr
+
+####
+# OpenJDK 11 (but keeps default to JDK8)
+# NOTE: This default only works when Apache Yetus is launched
+# _in_ the container and not outside of it!
+####
+# hadolint ignore=DL3008
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y default-jre-headless openjdk-11-jdk-headless \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/* \
+ && update-java-alternatives -s java-1.8.0-openjdk-amd64 \
+ && rm -f /usr/lib/jvm/default-java \
+ && ln -s java-8-openjdk-amd64 /usr/lib/jvm/default-java
+ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
######
-# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
-# Ant is needed for findbugs
+# Install findbugs
######
# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends findbugs ant \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y findbugs \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
ENV FINDBUGS_HOME /usr
-####
-# Install shellcheck (0.4.6, the latest as of 2017-09-26)
-####
+######
+# Install maven
+######
# hadolint ignore=DL3008
-RUN add-apt-repository -y ppa:hvr/ghc \
- && apt-get -q update \
- && apt-get -q install -y --no-install-recommends shellcheck ghc-8.0.2 \
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y maven \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/*
-####
-# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
-####
-# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends bats \
+######
+# Install python3 and pylint3
+# astroid and pylint go hand-in-hand. Upgrade both at the same time.
+######
+# hadolint ignore=DL3008,DL3013
+RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
+ python3 \
+ python3-bcrypt \
+ python3-cffi \
+ python3-cryptography \
+ python3-dateutil \
+ python3-dev \
+ python3-dev \
+ python3-isort \
+ python3-dockerpty \
+ python3-nacl \
+ python3-pyrsistent \
+ python3-setuptools \
+ python3-setuptools \
+ python3-singledispatch \
+ python3-six \
+ python3-wheel \
+ python3-wrapt \
+ python3-yaml \
&& apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-####
-# Install pylint at fixed version (2.0.0 removed python2 support)
-# https://github.com/PyCQA/pylint/issues/2294
-####
-RUN pip2 install pylint==1.9.2
-
-####
-# Install dateutil.parser
-####
-RUN pip2 install python-dateutil==2.7.3
+ && rm -rf /var/lib/apt/lists/* \
+ && curl -sSL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py \
+ && python3 /tmp/get-pip.py \
+ && rm /usr/local/bin/pip /tmp/get-pip.py \
+ && pip3 install -v \
+ astroid==2.4.2 \
+ codespell==2.0 \
+ pylint==2.5.3 \
+ yamllint==1.24.2 \
+ && rm -rf /root/.cache \
+ && mv /usr/local/bin/pylint /usr/local/bin/pylint3
+RUN ln -s /usr/local/bin/pylint3 /usr/local/bin/pylint
+RUN ln -s /usr/local/bin/pip3 /usr/local/bin/pip
###
-# Install node.js for web UI framework (8.10.0 ships with Bionic, let's override with 10.x)
+# Install npm and JSHint
###
-RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
-# hadolint ignore=DL3008, DL3015
-RUN apt-get install -y nodejs
-# hadolint ignore=DL3016
-RUN apt-get clean \
+# hadolint ignore=DL3008
+RUN curl -sSL https://deb.nodesource.com/setup_14.x | bash - \
+ && apt-get -q install --no-install-recommends -y nodejs \
+ && apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
- && npm install npm@latest -g \
- && npm install -g jshint
+ && npm install -g \
+ jshint@2.12.0 \
+ markdownlint-cli@0.23.2 \
+ && rm -rf /root/.npm
-###
-# Install hadolint
-####
-RUN curl -L -s -S \
- https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
- -o /bin/hadolint \
- && chmod a+rx /bin/hadolint \
- && shasum -a 512 /bin/hadolint | \
- awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
-
-###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx1536m
+#####
+# Now all the stuff that was built in parallel
+#####
+COPY --from=shellcheck /bin/shellcheck /bin/shellcheck
+COPY --from=hadolint /bin/hadolint /bin/hadolint
+COPY --from=buf /usr/local/bin/buf /usr/local/bin/buf
+COPY --from=protobuf250 /opt/protobuf /opt/protobuf
-###
-# Everything past this point is either not needed for testing or breaks Yetus.
-# So tell Yetus not to read the rest of the file:
-# YETUS CUT HERE
-###
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PROTOC_PATH /opt/protobuf/bin/protoc
+ENV PATH "${PATH}:/opt/protobuf/bin"
####
-# Install svn & Forrest (for Apache Hadoop website)
+# YETUS CUT HERE
+# Magic text above! Everything from here on is ignored
+# by Yetus, so could include anything not needed
+# by your testing environment
###
-# hadolint ignore=DL3008
-RUN apt-get -q update \
- && apt-get -q install -y --no-install-recommends subversion \
- && apt-get clean \
- && rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p /opt/apache-forrest \
- && curl -L -s -S \
- https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
- -o /opt/forrest.tar.gz \
- && tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
-RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
-ENV FORREST_HOME=/opt/apache-forrest
-
-# Hugo static website generator (for new tez site and Ozone docs)
-RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
- && dpkg --install hugo.deb \
- && rm hugo.deb
-
-# Add a welcome message and environment checks.
-COPY tez_env_checks.sh /root/tez_env_checks.sh
-RUN chmod 755 /root/tez_env_checks.sh
-# hadolint ignore=SC2016
-RUN echo '${HOME}/tez_env_checks.sh' >> /root/.bashrc
diff --git a/build-tools/docker/tez_env_checks.sh b/build-tools/docker/tez_env_checks.sh
deleted file mode 100644
index c9420e7..0000000
--- a/build-tools/docker/tez_env_checks.sh
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env bash
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# SHELLDOC-IGNORE
-
-# -------------------------------------------------------
-function showWelcome {
-cat <<Welcome-message
-
- _ _ _ ______
-| | | | | | | _ \\
-| |_| | __ _ __| | ___ ___ _ __ | | | |_____ __
-| _ |/ _\` |/ _\` |/ _ \\ / _ \\| '_ \\ | | | / _ \\ \\ / /
-| | | | (_| | (_| | (_) | (_) | |_) | | |/ / __/\\ V /
-\\_| |_/\\__,_|\\__,_|\\___/ \\___/| .__/ |___/ \\___| \\_(_)
- | |
- |_|
-
-This is the standard Hadoop Developer build environment.
-This has all the right tools installed required to build
-Hadoop from source.
-
-Welcome-message
-}
-
-# -------------------------------------------------------
-
-function showAbort {
- cat <<Abort-message
-
- ___ _ _ _
- / _ \\| | | | (_)
-/ /_\\ \\ |__ ___ _ __| |_ _ _ __ __ _
-| _ | '_ \\ / _ \\| '__| __| | '_ \\ / _\` |
-| | | | |_) | (_) | | | |_| | | | | (_| |
-\\_| |_/_.__/ \\___/|_| \\__|_|_| |_|\\__, |
- __/ |
- |___/
-
-Abort-message
-}
-
-# -------------------------------------------------------
-
-function failIfUserIsRoot {
- if [ "$(id -u)" -eq "0" ]; # If you are root then something went wrong.
- then
- cat <<End-of-message
-
-Apparently you are inside this docker container as the user root.
-Putting it simply:
-
- This should not occur.
-
-Known possible causes of this are:
-1) Running this script as the root user ( Just don't )
-2) Running an old docker version ( upgrade to 1.4.1 or higher )
-
-End-of-message
-
- showAbort
-
- logout
-
- fi
-}
-
-# -------------------------------------------------------
-
-function warnIfLowMemory {
- MINIMAL_MEMORY=2046755
- INSTALLED_MEMORY=$(grep -F MemTotal /proc/meminfo | awk '{print $2}')
- if [[ $((INSTALLED_MEMORY)) -lt $((MINIMAL_MEMORY)) ]]; then
- cat <<End-of-message
-
- _ ___ ___
-| | | \\/ |
-| | _____ __ | . . | ___ _ __ ___ ___ _ __ _ _
-| | / _ \\ \\ /\\ / / | |\\/| |/ _ \\ '_ \` _ \\ / _ \\| '__| | | |
-| |___| (_) \\ V V / | | | | __/ | | | | | (_) | | | |_| |
-\\_____/\\___/ \\_/\\_/ \\_| |_/\\___|_| |_| |_|\\___/|_| \\__, |
- __/ |
- |___/
-
-Your system is running on very little memory.
-This means it may work but it wil most likely be slower than needed.
-
-If you are running this via boot2docker you can simply increase
-the available memory to at least ${MINIMAL_MEMORY}KiB
-(you have ${INSTALLED_MEMORY}KiB )
-
-End-of-message
- fi
-}
-
-# -------------------------------------------------------
-
-showWelcome
-warnIfLowMemory
-failIfUserIsRoot
-
-# -------------------------------------------------------