You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by do...@apache.org on 2021/03/22 02:29:31 UTC
[orc] branch master updated: ORC-766: Generalize the docker scripts
to handle build arguments. (#658)
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new 01b98f2 ORC-766: Generalize the docker scripts to handle build arguments. (#658)
01b98f2 is described below
commit 01b98f2f4eeb7dc22a2dbf9040e58672e7d44579
Author: Owen O'Malley <oo...@linkedin.com>
AuthorDate: Mon Mar 22 02:29:22 2021 +0000
ORC-766: Generalize the docker scripts to handle build arguments. (#658)
### What changes were proposed in this pull request?
This replaces the build matrix loops in the docker reinit.sh and run-all.sh scripts with expanding the
versions that we want to build in os-list.txt. The pattern for the build args is based on the name:
ubuntu20 builds ubuntu 20 with default parameters jdk=8 and cc=gcc
ubuntu20_jdk=11_cc=clang builds ubuntu 20 with jdk=11 and cc=clang
Those names are carried through to the docker tags with s/=/-/.
### How was this patch tested?
I ran reinit.sh and run-all.sh and both ran fine. The docker images default to the proper c++ and java compilers.
---
docker/os-list.txt | 4 ++-
docker/reinit.sh | 20 ++++++++-------
docker/run-all.sh | 26 +++++++++++---------
docker/run-one.sh | 28 +++++++++------------
docker/ubuntu20-clang/Dockerfile | 53 ----------------------------------------
docker/ubuntu20/Dockerfile | 19 +++++++++++---
6 files changed, 56 insertions(+), 94 deletions(-)
diff --git a/docker/os-list.txt b/docker/os-list.txt
index 98755e8..3f9570a 100644
--- a/docker/os-list.txt
+++ b/docker/os-list.txt
@@ -5,4 +5,6 @@ debian10
ubuntu16
ubuntu18
ubuntu20
-ubuntu20-clang
+centos8_jdk=11
+ubuntu20_jdk=11
+ubuntu20_jdk=11_cc=clang
diff --git a/docker/reinit.sh b/docker/reinit.sh
index e3a36ba..ac8b795 100755
--- a/docker/reinit.sh
+++ b/docker/reinit.sh
@@ -16,15 +16,17 @@
# limitations under the License.
start=`date`
-for jdk in 8 11; do
- for os in `cat os-list.txt`; do
- if [[ "$os" = "debian10" && "$jdk" = "8" ]] || [[ "$os" = "debian9" && "$jdk" = "11" ]] || [[ "$os" = "ubuntu16" && "$jdk" = "11" ]]; then
- echo "Skip an initialize $os with $jdk"
- continue
- fi
- echo "Re-initialize $os with $jdk"
- ( cd $os && docker build --no-cache -t "orc-${os}-jdk${jdk}" --build-arg jdk=${jdk} . )
- done
+for build in `cat os-list.txt`; do
+ echo "Re-initialize $build"
+ OS=$(echo "$build" | cut -d '_' -f1)
+ REST=$(echo "$build" | cut -d '_' -f2- -s)
+ if [ -z "$REST" ]; then
+ ARGS=""
+ else
+ ARGS=$(echo "$REST" | sed -e 's/^/--build-arg /' -e 's/_/ --build-arg /g')
+ fi
+ TAG=$(echo "orc-$build" | sed -e 's/=/-/g')
+ ( cd $OS && docker build --no-cache -t "$TAG" $ARGS . )
done
echo "Start: $start"
echo "End:" `date`
diff --git a/docker/run-all.sh b/docker/run-all.sh
index 6979e1d..fffef58 100755
--- a/docker/run-all.sh
+++ b/docker/run-all.sh
@@ -32,22 +32,24 @@ function failure {
rm -f logs/pids.txt logs/*.log
start=`date`
-for jdk in 8 11; do
- for os in `cat os-list.txt`; do
- if [[ "$os" = "debian10" && "$jdk" = "8" ]] || [[ "$os" = "debian9" && "$jdk" = "11" ]] || [[ "$os" = "ubuntu16" && "$jdk" = "11" ]]; then
- echo "Skip building $os with $jdk"
- continue
- fi
- echo "Building $os for $jdk"
- ( cd $os && docker build -t "orc-$os-jdk${jdk}" --build-arg jdk=$jdk . ) > logs/${os}-jdk${jdk}-build.log 2>&1 || exit 1
- done
+for build in `cat os-list.txt`; do
+ echo "Building $build"
+ OS=$(echo "$build" | cut -d '_' -f1)
+ REST=$(echo "$build" | cut -d '_' -f2- -s)
+ if [ -z "$REST" ]; then
+ ARGS=""
+ else
+ ARGS=$(echo "$REST" | sed -e 's/^/--build-arg /' -e 's/_/ --build-arg /g')
+ fi
+ TAG=$(echo "orc-$build" | sed -e 's/=/-/g')
+ ( cd $OS && docker build -t "$TAG" $ARGS . ) > logs/$build-build.log 2>&1 || exit 1
done
testStart=`date`
-for os in `cat os-list.txt`; do
- ./run-one.sh $1 $2 $os > logs/$os-test.log 2>&1 &
+for build in `cat os-list.txt`; do
+ ./run-one.sh $1 $2 $build > logs/$build-test.log 2>&1 &
echo "$!" >> logs/pids.txt
- echo "Launching $os as $!"
+ echo "Launching $build as $!"
done
for job in `cat logs/pids.txt`; do
diff --git a/docker/run-one.sh b/docker/run-one.sh
index 9c049e4..361438f 100755
--- a/docker/run-one.sh
+++ b/docker/run-one.sh
@@ -18,26 +18,27 @@
GITHUB_USER=$1
URL=https://github.com/$GITHUB_USER/orc.git
BRANCH=$2
-OS=$3
+BUILD=$3
function failure {
- echo "FAILED $OS"
+ echo "FAILED $BUILD"
exit 1
}
VOLUME="--volume m2cache:/root/.m2/repository"
+TAG=$(echo "orc-$BUILD" | sed -e 's/=/-/g')
if [ $GITHUB_USER == "local" ]; then
BRANCH=`git status| head -1 | sed -e 's/On branch //'`
- echo "Started local run for $BRANCH on $OS at $(date)"
- docker run $VOLUME -v`pwd`/..:/root/orc "orc-$OS" || failure
+ echo "Started local run for $BRANCH on $BUILD at $(date)"
+ docker run $VOLUME -v`pwd`/..:/root/orc "$TAG" || failure
else
CLONE="git clone $URL -b $BRANCH"
MAKEDIR="mkdir orc/build && cd orc/build"
- echo "Started $GITHUB_USER/$BRANCH on $OS at $(date)"
+ echo "Started $GITHUB_USER/$BRANCH on $BUILD at $(date)"
- case $OS in
- debian8)
+ case $BUILD in
+ debian8*)
OPTS="-DSNAPPY_HOME=/usr/local"
;;
*)
@@ -45,13 +46,8 @@ else
;;
esac
- for jdk in 8 11; do
- if [[ "$OS" = "debian10" && "$jdk" = "8" ]] || [[ "$OS" = "debian9" && "$jdk" = "11" ]] || [[ "$OS" = "ubuntu16" && "$jdk" = "11" ]]; then
- continue
- fi
- docker run $VOLUME "orc-$OS-jdk${jdk}" /bin/bash -c \
- "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
- || failure
- done
+ docker run $VOLUME "$TAG" /bin/bash -c \
+ "$CLONE && $MAKEDIR && cmake $OPTS .. && make package test-out" \
+ || failure
fi
-echo "Finished $OS at $(date)"
+echo "Finished $BUILD at $(date)"
diff --git a/docker/ubuntu20-clang/Dockerfile b/docker/ubuntu20-clang/Dockerfile
deleted file mode 100644
index fff0537..0000000
--- a/docker/ubuntu20-clang/Dockerfile
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# ORC compile for Ubuntu 20 with clang
-#
-
-FROM ubuntu:20.04
-LABEL maintainer="Apache ORC project <de...@orc.apache.org>"
-ARG jdk=8
-
-RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
-RUN apt-get update
-RUN apt-get install -y \
- cmake \
- clang \
- git \
- libsasl2-dev \
- libssl-dev \
- make \
- curl \
- maven \
- openjdk-${jdk}-jdk \
- tzdata
-RUN update-java-alternatives --set java-1.${jdk}.0-openjdk-amd64
-
-ENV CC=clang
-ENV CXX=clang++
-
-WORKDIR /root
-VOLUME /root/.m2/repository
-
-CMD if [ ! -d orc ]; then \
- echo "No volume provided, building from apache master."; \
- echo "Pass '-v`pwd`:/root/orc' to docker run to build local source."; \
- git clone https://github.com/apache/orc.git -b master; \
- fi && \
- mkdir build && \
- cd build && \
- cmake ../orc && \
- make package test-out
diff --git a/docker/ubuntu20/Dockerfile b/docker/ubuntu20/Dockerfile
index 9862b04..0519003 100644
--- a/docker/ubuntu20/Dockerfile
+++ b/docker/ubuntu20/Dockerfile
@@ -20,13 +20,12 @@
FROM ubuntu:20.04
LABEL maintainer="Apache ORC project <de...@orc.apache.org>"
ARG jdk=8
+ARG cc=gcc
RUN ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime
RUN apt-get update
RUN apt-get install -y \
cmake \
- gcc \
- g++ \
git \
libsasl2-dev \
libssl-dev \
@@ -34,9 +33,23 @@ RUN apt-get install -y \
curl \
maven \
openjdk-${jdk}-jdk \
- tzdata
+ tzdata; \
+ if [ "${cc}" = "gcc" ] ; then \
+ apt-get install -y \
+ gcc \
+ g++ \
+ ; else \
+ apt-get install -y \
+ clang \
+ && \
+ update-alternatives --set cc /usr/bin/clang && \
+ update-alternatives --set c++ /usr/bin/clang++ \
+ ; fi
RUN update-java-alternatives --set java-1.${jdk}.0-openjdk-amd64
+ENV CC=cc
+ENV CXX=c++
+
WORKDIR /root
VOLUME /root/.m2/repository