You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by pa...@apache.org on 2023/12/16 01:34:20 UTC

(arrow-nanoarrow) branch main updated: chore(ci): Update docker images (#339)

This is an automated email from the ASF dual-hosted git repository.

paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git


The following commit(s) were added to refs/heads/main by this push:
     new dfe700f  chore(ci): Update docker images (#339)
dfe700f is described below

commit dfe700f0e366af6fdb335952df2cb47af75624cc
Author: Dewey Dunnington <de...@dunnington.ca>
AuthorDate: Fri Dec 15 21:34:15 2023 -0400

    chore(ci): Update docker images (#339)
    
    This fixes a few problems:
    
    - Install `which` on archlinux, since its absence was causing R to break
    - Use `pip` to install into a venv instead of running `pip` as root to
    install into system directories (which was broken on alpine)
---
 ci/docker/alpine.dockerfile    | 21 +++++++++++----------
 ci/docker/archlinux.dockerfile | 13 ++++++++++---
 ci/docker/centos7.dockerfile   | 20 +++++++++++++-------
 ci/docker/fedora.dockerfile    |  9 ++++++---
 ci/docker/ubuntu.dockerfile    | 21 +++++++++++----------
 ci/scripts/build-docs.sh       |  8 ++++++++
 6 files changed, 59 insertions(+), 33 deletions(-)

diff --git a/ci/docker/alpine.dockerfile b/ci/docker/alpine.dockerfile
index 6bf5a6f..7839bba 100644
--- a/ci/docker/alpine.dockerfile
+++ b/ci/docker/alpine.dockerfile
@@ -19,28 +19,29 @@ ARG NANOARROW_ARCH
 
 FROM --platform=linux/${NANOARROW_ARCH} alpine:latest
 
-RUN apk add bash linux-headers git cmake R R-dev g++ gnupg curl py3-pip python3-dev
-
-RUN pip3 install build Cython pytest
-
-# There's a missing define that numpy's build needs on s390x and there is no wheel
-RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true
-RUN pip3 install numpy
+RUN apk add bash linux-headers git cmake R R-dev g++ gfortran gnupg curl py3-virtualenv python3-dev
 
 # For Arrow C++
-RUN curl -L https://github.com/apache/arrow/archive/refs/tags/apache-arrow-11.0.0.tar.gz | tar -zxf - && \
+RUN curl -L https://github.com/apache/arrow/archive/refs/tags/apache-arrow-14.0.1.tar.gz | tar -zxf - && \
     mkdir /arrow-build && \
     cd /arrow-build && \
-    cmake ../arrow-apache-arrow-11.0.0/cpp \
+    cmake ../arrow-apache-arrow-14.0.1/cpp \
         -DARROW_JEMALLOC=OFF \
         -DARROW_SIMD_LEVEL=NONE \
+        -DARROW_WITH_ZLIB=ON \
         -DCMAKE_INSTALL_PREFIX=../arrow && \
     cmake --build . && \
     cmake --install . --prefix=../arrow
 
+# There's a missing define that numpy's build needs on s390x and there is no wheel
+RUN (grep -e "S390" /usr/include/bits/hwcap.h && echo "#define HWCAP_S390_VX HWCAP_S390_VXRS" >> /usr/include/bits/hwcap.h) || true
+RUN virtualenv -v --download /venv
+RUN source /venv/bin/activate && pip install build Cython pytest pytest-cython numpy
+ENV NANOARROW_PYTHON_VENV "/venv"
+
 # For R. Note that arrow is not installed (takes too long).
 RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "bit64"), repos = "https://cloud.r-project.org")'
 RUN rm -f ~/.R/Makevars
 
 ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/archlinux.dockerfile b/ci/docker/archlinux.dockerfile
index cc91abb..2a63d61 100644
--- a/ci/docker/archlinux.dockerfile
+++ b/ci/docker/archlinux.dockerfile
@@ -17,10 +17,17 @@
 
 FROM archlinux:latest
 
-RUN pacman -Syu --noconfirm git gcc make cmake r-base gnupg curl arrow \
-    python-pip python-build cython python-numpy python-pytest python-pyarrow
+RUN pacman -Syu --noconfirm git gcc gcc-fortran make cmake gnupg curl which \
+    python-pip python-build cython python-numpy python-pytest python-virtualenv \
+    r \
+    arrow python-pyarrow
+
+# For Python
+RUN virtualenv -v --download --system-site-packages /venv
+RUN source /venv/bin/activate && pip install pytest-cython
+ENV NANOARROW_PYTHON_VENV "/venv"
 
 # For R
 RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN R -e 'utils::install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "bit64"), repos = "https://cloud.r-project.org")'
 RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/centos7.dockerfile b/ci/docker/centos7.dockerfile
index 4ce8487..521100c 100644
--- a/ci/docker/centos7.dockerfile
+++ b/ci/docker/centos7.dockerfile
@@ -20,12 +20,7 @@ ARG NANOARROW_ARCH
 FROM --platform=linux/${NANOARROW_ARCH} centos:7
 
 RUN yum install -y epel-release
-RUN yum install -y git gnupg curl R gcc-c++ cmake3 python3-devel
-# On centos7/arm64, cythonize/build is required and fails with cython >= 3
-RUN pip3 install build "Cython < 3.0.0" numpy pytest
-
-RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
-ENV LC_ALL en_US.UTF-8
+RUN yum install -y git gnupg curl R gcc-c++ gcc-gfortran cmake3
 
 # For Arrow C++. Use 9.0.0 because this version works fine with the default gcc
 RUN curl -L https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0.tar.gz | tar -zxf - && \
@@ -34,13 +29,24 @@ RUN curl -L https://github.com/apache/arrow/archive/refs/tags/apache-arrow-9.0.0
     cmake3 ../arrow-apache-arrow-9.0.0/cpp \
         -DARROW_JEMALLOC=OFF \
         -DARROW_SIMD_LEVEL=NONE \
+        -DARROW_WITH_ZLIB=ON \
         -DCMAKE_INSTALL_PREFIX=../arrow && \
     cmake3 --build . && \
     make install
 
+RUN python3 -m venv /venv
+RUN source /venv/bin/activate && \
+    pip install --upgrade pip && \
+    pip install build Cython pytest pytest-cython numpy
+ENV NANOARROW_PYTHON_VENV "/venv"
+
+# Locale required for R CMD check
+RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+
 # For R. Note that arrow is not installed (takes too long).
 RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "bit64"), repos = "https://cloud.r-project.org")'
 RUN rm -f ~/.R/Makevars
 
 ENV NANOARROW_CMAKE_OPTIONS -DArrow_DIR=/arrow/lib/cmake/Arrow
diff --git a/ci/docker/fedora.dockerfile b/ci/docker/fedora.dockerfile
index a42455a..dd654f3 100644
--- a/ci/docker/fedora.dockerfile
+++ b/ci/docker/fedora.dockerfile
@@ -20,11 +20,14 @@ ARG NANOARROW_ARCH
 FROM --platform=linux/${NANOARROW_ARCH} fedora:latest
 
 RUN dnf install -y git cmake R gnupg curl libarrow-devel glibc-langpack-en \
-    python3-pip python3-devel
+   python3-devel python3-virtualenv
 
-RUN pip3 install build Cython numpy pytest pyarrow
+# For Python
+RUN python3 -m venv --upgrade-deps /venv
+RUN source /venv/bin/activate && pip install build Cython pytest pytest-cython numpy pyarrow
+ENV NANOARROW_PYTHON_VENV "/venv"
 
 # For R. Note that arrow is not installed (takes too long).
 RUN mkdir ~/.R && echo "MAKEFLAGS = -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr"), repos = "https://cloud.r-project.org")'
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "bit64"), repos = "https://cloud.r-project.org")'
 RUN rm -f ~/.R/Makevars
diff --git a/ci/docker/ubuntu.dockerfile b/ci/docker/ubuntu.dockerfile
index b6e5ef0..4a95074 100644
--- a/ci/docker/ubuntu.dockerfile
+++ b/ci/docker/ubuntu.dockerfile
@@ -20,12 +20,10 @@ ARG NANOARROW_ARCH
 FROM --platform=linux/${NANOARROW_ARCH} ubuntu:latest
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \
-    locales git cmake r-base gnupg curl valgrind python3-pip python3-venv doxygen pandoc lcov \
+    locales git cmake r-base gnupg curl valgrind gfortran python3-venv doxygen pandoc lcov \
     libxml2-dev libfontconfig1-dev libfreetype6-dev libfribidi-dev libharfbuzz-dev \
     libjpeg-dev libpng-dev libtiff-dev
 
-RUN locale-gen en_US.UTF-8 && update-locale en_US.UTF-8
-
 # For Arrow C++
 RUN apt-get install -y -V ca-certificates lsb-release wget && \
     wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb && \
@@ -34,16 +32,19 @@ RUN apt-get install -y -V ca-certificates lsb-release wget && \
     apt-get install -y -V libarrow-dev
 
 # For documentation build + Python build
-# Note: sphinx can be unpinned when the interaction between sphinx and breathe
-# has been sorted: https://github.com/sphinx-doc/sphinx/issues/11605
-RUN pip3 install pydata-sphinx-theme "sphinx<7.2.0" breathe build Cython numpy pytest pyarrow
+RUN python3 -m venv --upgrade-deps /venv
+RUN echo "source /venv/bin/activate && pip install pydata-sphinx-theme sphinx breathe build Cython numpy pytest pytest-cython pytest-cov pyarrow" | bash
+ENV NANOARROW_PYTHON_VENV "/venv"
+
+# Locale required for R CMD check
+RUN locale-gen en_US.UTF-8 && update-locale en_US.UTF-8
 
-# For R. Note that we install arrow here so that the integration tests for R run
-# in at least one test image.
+# For R
 RUN mkdir ~/.R && echo "MAKEFLAGS += -j$(nproc)" > ~/.R/Makevars
-RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "pkgdown", "covr", "pkgbuild"), repos = "https://cloud.r-project.org")'
+RUN R -e 'install.packages(c("blob", "hms", "tibble", "rlang", "testthat", "tibble", "vctrs", "withr", "bit64", "pkgdown", "covr", "pkgbuild"), repos = "https://cloud.r-project.org")'
 
-# Required for this to work on MacOS/arm64
+# Install arrow here so that the integration tests for R run in at least one test image.
+# -fPIC required for this to work on MacOS/arm64
 RUN echo "CXX17FLAGS += -fPIC" >> ~/.R/Makevars
 RUN ARROW_USE_PKG_CONFIG=false ARROW_R_DEV=true R -e 'install.packages("arrow", repos = "https://cloud.r-project.org"); library(arrow)'
 RUN rm -f ~/.R/Makevars
diff --git a/ci/scripts/build-docs.sh b/ci/scripts/build-docs.sh
index fd39f5c..d9c1325 100755
--- a/ci/scripts/build-docs.sh
+++ b/ci/scripts/build-docs.sh
@@ -52,7 +52,15 @@ case $# in
      ;;
 esac
 
+maybe_activate_venv() {
+  if [ ! -z "${NANOARROW_PYTHON_VENV}" ]; then
+    source "${NANOARROW_PYTHON_VENV}/bin/activate"
+  fi
+}
+
 main() {
+   maybe_activate_venv
+
    pushd "${TARGET_NANOARROW_DIR}"
 
    # Clean the previous build