You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/02/02 01:20:22 UTC
[hadoop] 01/02: HADOOP-16053
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch branch-2-jhung-test
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 9b7932a2ec2a3aa477d252f721e731e7913cefb3
Author: Jonathan Hung <jh...@linkedin.com>
AuthorDate: Fri Feb 1 17:12:47 2019 -0800
HADOOP-16053
---
dev-support/docker/Dockerfile | 264 +++++++++++++++++++++++++++---------------
1 file changed, 169 insertions(+), 95 deletions(-)
diff --git a/dev-support/docker/Dockerfile b/dev-support/docker/Dockerfile
index f6dc6e4..6594c8b 100644
--- a/dev-support/docker/Dockerfile
+++ b/dev-support/docker/Dockerfile
@@ -18,150 +18,224 @@
# Dockerfile for installing the necessary dependencies for building Hadoop.
# See BUILDING.txt.
-
-FROM ubuntu:trusty
+FROM ubuntu:xenial
WORKDIR /root
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+#####
+# Disable suggests/recommends
+#####
+RUN echo APT::Install-Recommends "0"\; > /etc/apt/apt.conf.d/10disableextras
+RUN echo APT::Install-Suggests "0"\; >> /etc/apt/apt.conf.d/10disableextras
+
ENV DEBIAN_FRONTEND noninteractive
ENV DEBCONF_TERSE true
######
-# Install common dependencies from packages
-######
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
- ant \
- build-essential \
- bzip2 \
- cmake \
- curl \
- doxygen \
- fuse \
- g++ \
- gcc \
- git \
- gnupg-agent \
- make \
- maven \
- libbz2-dev \
- libcurl4-openssl-dev \
- libfuse-dev \
- libjansson-dev \
- libprotobuf-dev \
- libprotoc-dev \
- libsnappy-dev \
- libssl-dev \
- libtool \
- openjdk-7-jdk \
- pinentry-curses \
- pkg-config \
- protobuf-compiler \
- protobuf-c-compiler \
- python \
- python2.7 \
- python2.7-dev \
- python-pip \
- rsync \
- snappy \
- zlib1g-dev
-
-######
-# Install Apache Maven
+# Install common dependencies from packages. Versions here are either
+# sufficient or irrelevant.
+#
+# WARNING: DO NOT PUT JAVA APPS HERE! Otherwise they will install default
+# Ubuntu Java. See Java section below!
######
-RUN mkdir -p /opt/maven && \
- curl -L -s -S \
- http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz \
- -o /opt/maven.tar.gz && \
- tar xzf /opt/maven.tar.gz --strip-components 1 -C /opt/maven
-ENV MAVEN_HOME /opt/maven
-
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends \
+ apt-utils \
+ build-essential \
+ bzip2 \
+ curl \
+ doxygen \
+ fuse \
+ g++ \
+ gcc \
+ git \
+ gnupg-agent \
+ libbz2-dev \
+ libcurl4-openssl-dev \
+ libfuse-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libsasl2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libtool \
+ libzstd1-dev \
+ locales \
+ make \
+ pinentry-curses \
+ pkg-config \
+ python \
+ python2.7 \
+ python-pip \
+ python-pkg-resources \
+ python-setuptools \
+ python-wheel \
+ rsync \
+ software-properties-common \
+ snappy \
+ sudo \
+ zlib1g-dev \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
#######
-# Java OpenJDK
+# OpenJDK 7
#######
+# hadolint ignore=DL3008
+RUN add-apt-repository ppa:openjdk-r/ppa
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends openjdk-7-jdk \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
-RUN echo "dot_style = mega" > "/root/.wgetrc"
-RUN echo "quiet = on" >> "/root/.wgetrc"
+######
+# Install cmake 3.1.0 (3.5.1 ships with Xenial)
+######
+RUN mkdir -p /opt/cmake \
+ && curl -L -s -S \
+ https://cmake.org/files/v3.1/cmake-3.1.0-Linux-x86_64.tar.gz \
+ -o /opt/cmake.tar.gz \
+ && tar xzf /opt/cmake.tar.gz --strip-components 1 -C /opt/cmake
+ENV CMAKE_HOME /opt/cmake
+ENV PATH "${PATH}:/opt/cmake/bin"
-RUN apt-get -q install --no-install-recommends -y software-properties-common
-RUN add-apt-repository -y ppa:webupd8team/java
-RUN apt-get -q update
+######
+# Install Google Protobuf 2.5.0 (2.6.0 ships with Xenial)
+######
+# hadolint ignore=DL3003
+RUN mkdir -p /opt/protobuf-src \
+ && curl -L -s -S \
+ https://github.com/google/protobuf/releases/download/v2.5.0/protobuf-2.5.0.tar.gz \
+ -o /opt/protobuf.tar.gz \
+ && tar xzf /opt/protobuf.tar.gz --strip-components 1 -C /opt/protobuf-src \
+ && cd /opt/protobuf-src \
+ && ./configure --prefix=/opt/protobuf \
+ && make install \
+ && cd /root \
+ && rm -rf /opt/protobuf-src
+ENV PROTOBUF_HOME /opt/protobuf
+ENV PATH "${PATH}:/opt/protobuf/bin"
-# Install OpenJDK 7
-RUN apt-get -q install --no-install-recommends -y openjdk-7-jdk
+######
+# Install Apache Maven 3.3.9 (3.3.9 ships with Xenial)
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends maven \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+ENV MAVEN_HOME /usr
######
-# Install spotbugs (successor of findbugs)
+# Install Apache Ant 1.9.6 (1.9.6 ships with Xenial)
######
-RUN mkdir -p /opt/spotbugs && \
- curl -L -s -S \
- http://repo.maven.apache.org/maven2/com/github/spotbugs/spotbugs/3.1.2/spotbugs-3.1.2.tgz \
- -o /opt/spotbugs.tar.gz && \
- tar xzf /opt/spotbugs.tar.gz --strip-components 1 -C /opt/spotbugs
-# Hadoop uses FINDBUGS_HOME to run spotbugs
-ENV FINDBUGS_HOME /opt/spotbugs
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends ant \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+######
+# Install findbugs 3.0.1 (3.0.1 ships with Xenial)
+######
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends findbugs \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+ENV FINDBUGS_HOME /usr
####
-# Install shellcheck
+# Install shellcheck (0.4.6, the latest as of 2017-09-26)
####
-RUN apt-get -q install -y cabal-install
-RUN mkdir /root/.cabal
-RUN echo "remote-repo: hackage.fpcomplete.com:http://hackage.fpcomplete.com/" >> /root/.cabal/config
-#RUN echo "remote-repo: hackage.haskell.org:http://hackage.haskell.org/" > /root/.cabal/config
-RUN echo "remote-repo-cache: /root/.cabal/packages" >> /root/.cabal/config
-RUN cabal update
-RUN cabal install shellcheck --global
+# hadolint ignore=DL3008
+RUN add-apt-repository -y ppa:jonathonf/ghc-8.0.2 \
+ && apt-get -q update \
+ && apt-get -q install -y --no-install-recommends shellcheck \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
####
-# Install bats
+# Install bats (0.4.0, the latest as of 2017-09-26, ships with Xenial)
####
-RUN add-apt-repository -y ppa:duggan/bats
-RUN apt-get -q update
-RUN apt-get -q install --no-install-recommends -y bats
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends bats \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
####
-# Install pylint
+# Install pylint at fixed version (2.0.0 removed python2 support)
+# https://github.com/PyCQA/pylint/issues/2294
####
-RUN pip install pylint==1.9.2
+RUN pip2 install pylint==1.9.2
####
# Install dateutil.parser
####
-RUN pip install python-dateutil
-
-###
-# Avoid out of memory errors in builds
-###
-ENV MAVEN_OPTS -Xms256m -Xmx1536m
+RUN pip2 install python-dateutil==2.7.3
###
-# Install node js tools for web UI frameowkr
+# Install node.js for web UI framework (4.2.6 ships with Xenial)
###
+# hadolint ignore=DL3008, DL3016
RUN apt-get -q update \
&& apt-get install -y --no-install-recommends nodejs npm \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* \
&& ln -s /usr/bin/nodejs /usr/bin/node \
- && npm config set strict-ssl false \
&& npm install npm@latest -g \
&& npm install -g jshint
###
+# Install hadolint
+####
+RUN curl -L -s -S \
+ https://github.com/hadolint/hadolint/releases/download/v1.11.1/hadolint-Linux-x86_64 \
+ -o /bin/hadolint \
+ && chmod a+rx /bin/hadolint \
+ && shasum -a 512 /bin/hadolint | \
+ awk '$1!="734e37c1f6619cbbd86b9b249e69c9af8ee1ea87a2b1ff71dccda412e9dac35e63425225a95d71572091a3f0a11e9a04c2fc25d9e91b840530c26af32b9891ca" {exit(1)}'
+
+###
+# Avoid out of memory errors in builds
+###
+ENV MAVEN_OPTS -Xms256m -Xmx1536m
+
+
+###
# Everything past this point is either not needed for testing or breaks Yetus.
# So tell Yetus not to read the rest of the file:
# YETUS CUT HERE
###
####
-# Install Forrest (for Apache Hadoop website)
+# Install svn & Forrest (for Apache Hadoop website)
###
-RUN mkdir -p /usr/local/apache-forrest ; \
- curl -s -S -O http://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz ; \
- tar xzf *forrest* --strip-components 1 -C /usr/local/apache-forrest ; \
- echo 'forrest.home=/usr/local/apache-forrest' > build.properties
+# hadolint ignore=DL3008
+RUN apt-get -q update \
+ && apt-get -q install -y --no-install-recommends subversion \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /opt/apache-forrest \
+ && curl -L -s -S \
+ https://archive.apache.org/dist/forrest/0.8/apache-forrest-0.8.tar.gz \
+ -o /opt/forrest.tar.gz \
+ && tar xzf /opt/forrest.tar.gz --strip-components 1 -C /opt/apache-forrest
+RUN echo 'forrest.home=/opt/apache-forrest' > build.properties
+ENV FORREST_HOME=/opt/apache-forrest
+
+# Hugo static website generator (for new hadoop site and Ozone docs)
+RUN curl -L -o hugo.deb https://github.com/gohugoio/hugo/releases/download/v0.30.2/hugo_0.30.2_Linux-64bit.deb \
+ && dpkg --install hugo.deb \
+ && rm hugo.deb
# Add a welcome message and environment checks.
-ADD hadoop_env_checks.sh /root/hadoop_env_checks.sh
+COPY hadoop_env_checks.sh /root/hadoop_env_checks.sh
RUN chmod 755 /root/hadoop_env_checks.sh
-RUN echo '~/hadoop_env_checks.sh' >> /root/.bashrc
-
+# hadolint ignore=SC2016
+RUN echo '${HOME}/hadoop_env_checks.sh' >> /root/.bashrc
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org