You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2021/07/05 19:59:58 UTC
[arrow] branch master updated: ARROW-13199: [R] add ubuntu 21.04 to
nightly builds
This is an automated email from the ASF dual-hosted git repository.
jonkeane pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 41c4143 ARROW-13199: [R] add ubuntu 21.04 to nightly builds
41c4143 is described below
commit 41c4143992905cc85eb61a417cf9460c6db6b4df
Author: Jonathan Keane <jk...@gmail.com>
AuthorDate: Mon Jul 5 14:58:49 2021 -0500
ARROW-13199: [R] add ubuntu 21.04 to nightly builds
Closes #10611 from jonkeane/ARROW-13199-r-ubuntu-21.04
Authored-by: Jonathan Keane <jk...@gmail.com>
Signed-off-by: Jonathan Keane <jk...@gmail.com>
---
.env | 1 +
ci/docker/linux-apt-r.dockerfile | 10 ++
ci/docker/ubuntu-21.04-cpp.dockerfile | 160 ++++++++++++++++++++++++++++++++
dev/tasks/docker-tests/github.linux.yml | 8 +-
dev/tasks/tasks.yml | 22 +++++
docker-compose.yml | 10 ++
r/configure | 7 +-
r/tools/nixlibs.R | 5 +
8 files changed, 219 insertions(+), 4 deletions(-)
diff --git a/.env b/.env
index e06c6e5..579895e 100644
--- a/.env
+++ b/.env
@@ -61,6 +61,7 @@ SPARK=master
DOTNET=3.1
R=4.1
ARROW_R_DEV=TRUE
+GCC_VERSION=""
# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
R_ORG=rhub
R_IMAGE=ubuntu-gcc-release
diff --git a/ci/docker/linux-apt-r.dockerfile b/ci/docker/linux-apt-r.dockerfile
index 7b78da9..36f4fb2 100644
--- a/ci/docker/linux-apt-r.dockerfile
+++ b/ci/docker/linux-apt-r.dockerfile
@@ -61,6 +61,16 @@ RUN apt-get update -y && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
+ARG gcc_version=""
+RUN if [ "${gcc_version}" != "" ]; then \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+ update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 30 && \
+ update-alternatives --set cc /usr/bin/gcc && \
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 30 && \
+ update-alternatives --set c++ /usr/bin/g++; \
+ fi
+
# Ensure parallel R package installation, set CRAN repo mirror,
# and use pre-built binaries where possible
COPY ci/etc/rprofile /arrow/ci/etc/
diff --git a/ci/docker/ubuntu-21.04-cpp.dockerfile b/ci/docker/ubuntu-21.04-cpp.dockerfile
new file mode 100644
index 0000000..18c3778
--- /dev/null
+++ b/ci/docker/ubuntu-21.04-cpp.dockerfile
@@ -0,0 +1,160 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG base=amd64/ubuntu:20.04
+FROM ${base}
+ARG arch
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+
+RUN echo "debconf debconf/frontend select Noninteractive" | \
+ debconf-set-selections
+
+# Installs LLVM toolchain, for Gandiva and testing other compilers
+#
+# Note that this is installed before the base packages to improve iteration
+# while debugging package list with docker build.
+ARG clang_tools
+ARG llvm
+RUN if [ "${llvm}" -gt "10" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ apt-transport-https \
+ ca-certificates \
+ gnupg \
+ wget && \
+ wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
+ echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${llvm} main" > \
+ /etc/apt/sources.list.d/llvm.list && \
+ if [ "${clang_tools}" != "${llvm}" -a "${clang_tools}" -gt 10 ]; then \
+ echo "deb https://apt.llvm.org/hirsute/ llvm-toolchain-hirsute-${clang_tools} main" > \
+ /etc/apt/sources.list.d/clang-tools.list; \
+ fi \
+ fi && \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ clang-${clang_tools} \
+ clang-${llvm} \
+ clang-format-${clang_tools} \
+ clang-tidy-${clang_tools} \
+ llvm-${llvm}-dev && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists*
+
+# Installs C++ toolchain and dependencies
+RUN apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ autoconf \
+ ca-certificates \
+ ccache \
+ cmake \
+ gdb \
+ git \
+ libbenchmark-dev \
+ libboost-filesystem-dev \
+ libboost-system-dev \
+ libbrotli-dev \
+ libbz2-dev \
+ libc-ares-dev \
+ libcurl4-openssl-dev \
+ libgflags-dev \
+ libgoogle-glog-dev \
+ libgrpc++-dev \
+ liblz4-dev \
+ libprotobuf-dev \
+ libprotoc-dev \
+ libre2-dev \
+ libsnappy-dev \
+ libssl-dev \
+ libthrift-dev \
+ libutf8proc-dev \
+ libzstd-dev \
+ make \
+ ninja-build \
+ pkg-config \
+ protobuf-compiler \
+ protobuf-compiler-grpc \
+ rapidjson-dev \
+ tzdata \
+ wget && \
+ apt-get clean && \
+ rm -rf /var/lib/apt/lists*
+
+COPY ci/scripts/install_minio.sh \
+ /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local
+
+# Prioritize system packages and local installation
+# The following dependencies will be downloaded due to missing/invalid packages
+# provided by the distribution:
+# - libc-ares-dev does not install CMake config files
+# - flatbuffer is not packaged
+# - libgtest-dev only provide sources
+# - libprotobuf-dev only provide sources
+ENV ARROW_BUILD_TESTS=ON \
+ ARROW_DEPENDENCY_SOURCE=SYSTEM \
+ ARROW_DATASET=ON \
+ ARROW_FLIGHT=OFF \
+ ARROW_GANDIVA=ON \
+ ARROW_HDFS=ON \
+ ARROW_HOME=/usr/local \
+ ARROW_INSTALL_NAME_RPATH=OFF \
+ ARROW_NO_DEPRECATED_API=ON \
+ ARROW_ORC=ON \
+ ARROW_PARQUET=ON \
+ ARROW_PLASMA=ON \
+ ARROW_S3=ON \
+ ARROW_USE_ASAN=OFF \
+ ARROW_USE_CCACHE=ON \
+ ARROW_USE_UBSAN=OFF \
+ ARROW_WITH_BROTLI=ON \
+ ARROW_WITH_BZ2=ON \
+ ARROW_WITH_LZ4=ON \
+ ARROW_WITH_SNAPPY=ON \
+ ARROW_WITH_ZLIB=ON \
+ ARROW_WITH_ZSTD=ON \
+ AWSSDK_SOURCE=BUNDLED \
+ GTest_SOURCE=BUNDLED \
+ ORC_SOURCE=BUNDLED \
+ PARQUET_BUILD_EXAMPLES=ON \
+ PARQUET_BUILD_EXECUTABLES=ON \
+ PATH=/usr/lib/ccache/:$PATH \
+ PYTHON=python3
+
+ARG gcc_version=""
+RUN if [ "${gcc_version}" = "" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ g++ \
+ gcc; \
+ else \
+ if [ "${gcc_version}" -gt "10" ]; then \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends software-properties-common && \
+ add-apt-repository ppa:ubuntu-toolchain-r/volatile; \
+ fi; \
+ apt-get update -y -q && \
+ apt-get install -y -q --no-install-recommends \
+ g++-${gcc_version} \
+ gcc-${gcc_version} && \
+ update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${gcc_version} 100 && \
+ update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${gcc_version} 100 && \
+ update-alternatives --install /usr/bin/cc cc /usr/bin/gcc 100 && \
+ update-alternatives --set cc /usr/bin/gcc && \
+ update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++ 100 && \
+ update-alternatives --set c++ /usr/bin/g++; \
+ fi
\ No newline at end of file
diff --git a/dev/tasks/docker-tests/github.linux.yml b/dev/tasks/docker-tests/github.linux.yml
index 06837a8..dd347b8 100644
--- a/dev/tasks/docker-tests/github.linux.yml
+++ b/dev/tasks/docker-tests/github.linux.yml
@@ -39,7 +39,13 @@ jobs:
- name: Execute Docker Build
shell: bash
- run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }}
+ run: archery docker run -e SETUPTOOLS_SCM_PRETEND_VERSION="{{ arrow.no_rc_version }}" {{ flags|default("") }} {{ image }} {{ command|default("") }}
+
+ {% if '-r-' in image %}
+ - name: Dump R install logs
+ run: cat arrow/r/check/arrow.Rcheck/00install.out
+ continue-on-error: true
+ {% endif %}
{% if arrow.branch == 'master' %}
{{ macros.github_login_dockerhub()|indent }}
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index e908a63..7aea18f 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -934,6 +934,28 @@ tasks:
r_tag: {{ r_tag }}
{% endfor %}
+ test-r-ubuntu-21.04:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ env:
+ UBUNTU: 21.04
+ CLANG_TOOLS: 9 # can remove this when >=9 is the default
+ flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE'
+ image: ubuntu-r
+
+ test-r-gcc-11:
+ ci: github
+ template: docker-tests/github.linux.yml
+ params:
+ env:
+ UBUNTU: 21.04
+ CLANG_TOOLS: 9 # can remove this when >=9 is the default
+ GCC_VERSION: 11
+ # S3 support is not buildable with gcc11 right now
+ flags: '-e ARROW_SOURCE_HOME="/arrow" -e FORCE_BUNDLED_BUILD=TRUE -e LIBARROW_BUILD=TRUE -e ARROW_S3=OFF'
+ image: ubuntu-r-only-r
+
test-r-rstudio-r-base-3.6-centos7-devtoolset-8:
ci: azure
template: r/azure.linux.yml
diff --git a/docker-compose.yml b/docker-compose.yml
index f65609a..6b435e8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -110,6 +110,7 @@ x-hierarchy:
- ubuntu-docs
- ubuntu-python-sdist-test
- ubuntu-r
+ - ubuntu-r-only-r
- ubuntu-cuda-cpp:
- ubuntu-cuda-python
- ubuntu-csharp
@@ -300,6 +301,7 @@ services:
base: "${ARCH}/ubuntu:${UBUNTU}"
clang_tools: ${CLANG_TOOLS}
llvm: ${LLVM}
+ gcc_version: ${GCC_VERSION}
shm_size: *shm-size
ulimits: *ulimits
environment:
@@ -1008,12 +1010,14 @@ services:
arch: ${ARCH}
r: ${R}
base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
+ gcc_version: ${GCC_VERSION}
shm_size: *shm-size
environment:
<<: *ccache
ARROW_R_CXXFLAGS: '-Werror'
LIBARROW_BUILD: 'false'
NOT_CRAN: 'true'
+ ARROW_R_DEV: ${ARROW_R_DEV}
volumes: *ubuntu-volumes
command: >
/bin/bash -c "
@@ -1021,6 +1025,12 @@ services:
/arrow/ci/scripts/python_build.sh /arrow /build &&
/arrow/ci/scripts/r_test.sh /arrow"
+ ubuntu-r-only-r:
+ extends: ubuntu-r
+ command: >
+ /bin/bash -c "
+ /arrow/ci/scripts/r_test.sh /arrow"
+
r:
# This lets you test building/installing the arrow R package
# (including building the C++ library) on any Docker image that contains R
diff --git a/r/configure b/r/configure
index aa7e7a8..22a34bc 100755
--- a/r/configure
+++ b/r/configure
@@ -69,7 +69,7 @@ if [ "$FORCE_AUTOBREW" = "true" ] || [ "$FORCE_BUNDLED_BUILD" = "true" ]; then
fi
# Note that cflags may be empty in case of success
-if [ "$ARROW_HOME" ]; then
+if [ "$ARROW_HOME" ] && [ "$FORCE_BUNDLED_BUILD" != "true" ]; then
echo "*** Using ARROW_HOME as the source of libarrow"
PKG_CFLAGS="-I$ARROW_HOME/include $PKG_CFLAGS"
PKG_DIRS="-L$ARROW_HOME/lib"
@@ -88,7 +88,8 @@ else
fi
if [ "$PKGCONFIG_CFLAGS" ] && [ "$PKGCONFIG_LIBS" ]; then
- echo "*** Arrow C++ libraries found via pkg-config"
+ FOUND_LIB_DIR=`echo $PKG_DIRS | sed -e 's/^-L//'`
+ echo "*** Arrow C++ libraries found via pkg-config at $FOUND_LIB_DIR"
PKG_CFLAGS="$PKGCONFIG_CFLAGS"
PKG_LIBS=${PKGCONFIG_LIBS}
PKG_DIRS=${PKGCONFIG_DIRS}
@@ -244,7 +245,7 @@ if [ $? -eq 0 ] || [ "$UNAME" = "Darwin" ]; then
fi
fi
# prepend PKG_DIRS and append BUNDLED_LIBS to PKG_LIBS
- PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS"
+ PKG_LIBS="$PKG_DIRS $PKG_LIBS $BUNDLED_LIBS -fno-lto"
echo "PKG_CFLAGS=$PKG_CFLAGS"
echo "PKG_LIBS=$PKG_LIBS"
else
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index 97cc69f..f638b7b 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -326,6 +326,7 @@ build_libarrow <- function(src_dir, dst_dir) {
CC = R_CMD_config("CC"),
CXX = paste(R_CMD_config("CXX11"), R_CMD_config("CXX11STD")),
# CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols
+ ARROW_R_CXXFLAGS = paste(Sys.getenv("ARROW_R_CXXFLAGS", ""), "-fno-lto"),
LDFLAGS = R_CMD_config("LDFLAGS")
)
env_vars <- paste0(names(env_var_list), '="', env_var_list, '"', collapse = " ")
@@ -415,6 +416,10 @@ cmake_version <- function(cmd = "cmake") {
with_s3_support <- function(env_vars) {
arrow_s3 <- toupper(Sys.getenv("ARROW_S3")) == "ON" || tolower(Sys.getenv("LIBARROW_MINIMAL")) == "false"
+ # but if ARROW_S3=OFF explicitly, we are definitely off, so override
+ if (toupper(Sys.getenv("ARROW_S3")) == "OFF" ) {
+ arrow_s3 <- FALSE
+ }
if (arrow_s3) {
# User wants S3 support. If they're using gcc, let's make sure the version is >= 4.9
# and make sure that we have curl and openssl system libs