You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2022/04/21 21:50:24 UTC
[arrow] branch master updated: ARROW-14638: [C++][R] Unknown C compiler / ccache on Arch Linux
This is an automated email from the ASF dual-hosted git repository.
jonkeane pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new f7bccc51cc ARROW-14638: [C++][R] Unknown C compiler / ccache on Arch Linux
f7bccc51cc is described below
commit f7bccc51cc8ab384134ee50a8dd0af03d937e8cd
Author: Jonathan Keane <jk...@gmail.com>
AuthorDate: Thu Apr 21 16:50:14 2022 -0500
ARROW-14638: [C++][R] Unknown C compiler / ccache on Arch Linux
Closes #11666 from jonkeane/ARROW-14638-ccache
Lead-authored-by: Jonathan Keane <jk...@gmail.com>
Co-authored-by: Neal Richardson <ne...@gmail.com>
Signed-off-by: Jonathan Keane <jk...@gmail.com>
---
.env | 6 ++++--
ci/docker/linux-r.dockerfile | 3 +++
ci/scripts/r_docker_configure.sh | 24 ++++++++++++++++++++++++
dev/tasks/r/azure.linux.yml | 1 +
dev/tasks/tasks.yml | 9 +++++++++
docker-compose.yml | 1 +
r/tools/nixlibs.R | 10 ++++++++--
7 files changed, 50 insertions(+), 4 deletions(-)
diff --git a/.env b/.env
index a972654497..629dd04980 100644
--- a/.env
+++ b/.env
@@ -73,11 +73,13 @@ SPARK=master
TURBODBC=latest
# These correspond to images on Docker Hub that contain R, e.g. rhub/ubuntu-gcc-release:latest
-ARROW_R_DEV=TRUE
R_IMAGE=ubuntu-gcc-release
R_ORG=rhub
-R_PRUNE_DEPS=FALSE
R_TAG=latest
+
+# Env vars for R builds
+ARROW_R_DEV=TRUE
+R_PRUNE_DEPS=FALSE
TZ=UTC
# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
index 1cbde3207e..804fb09f09 100644
--- a/ci/docker/linux-r.dockerfile
+++ b/ci/docker/linux-r.dockerfile
@@ -33,6 +33,9 @@ ENV DEVTOOLSET_VERSION=${devtoolset_version}
ARG r_prune_deps=FALSE
ENV R_PRUNE_DEPS=${r_prune_deps}
+ARG r_custom_ccache=false
+ENV R_CUSTOM_CCACHE=${r_custom_ccache}
+
ARG tz="UTC"
ENV TZ=${tz}
diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
index 518df1040d..9f93ba2b61 100755
--- a/ci/scripts/r_docker_configure.sh
+++ b/ci/scripts/r_docker_configure.sh
@@ -42,6 +42,30 @@ else
apt-get update
fi
+# Enable ccache if requested based on http://dirk.eddelbuettel.com/blog/2017/11/27/
+: ${R_CUSTOM_CCACHE:=FALSE}
+R_CUSTOM_CCACHE=`echo $R_CUSTOM_CCACHE | tr '[:upper:]' '[:lower:]'`
+if [ ${R_CUSTOM_CCACHE} = "true" ]; then
+ # install ccache
+ $PACKAGE_MANAGER install -y epel-release || true
+ $PACKAGE_MANAGER install -y ccache
+
+ mkdir -p ~/.R
+ echo "VER=
+CCACHE=ccache
+CC=\$(CCACHE) gcc\$(VER)
+CXX=\$(CCACHE) g++\$(VER)
+CXX11=\$(CCACHE) g++\$(VER)" >> ~/.R/Makevars
+
+ mkdir -p ~/.ccache/
+ echo "max_size = 5.0G
+# important for R CMD INSTALL *.tar.gz as tarballs are expanded freshly -> fresh ctime
+sloppiness = include_file_ctime
+# also important as the (temp.) directory name will differ
+hash_dir = false" >> ~/.ccache/ccache.conf
+fi
+
+
# Special hacking to try to reproduce quirks on fedora-clang-devel on CRAN
# which uses a bespoke clang compiled to use libc++
# https://www.stats.ox.ac.uk/pub/bdr/Rconfig/r-devel-linux-x86_64-fedora-clang
diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
index 50b27aa7be..fd48141961 100644
--- a/dev/tasks/r/azure.linux.yml
+++ b/dev/tasks/r/azure.linux.yml
@@ -43,6 +43,7 @@ jobs:
export R_IMAGE={{ r_image }}
export R_TAG={{ r_tag }}
export DEVTOOLSET_VERSION={{ devtoolset_version|default("-1") }}
+ export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index b45dec61ff..a6a41ca274 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1279,6 +1279,15 @@ tasks:
template: r/github.linux.offline.build.yml
+ test-r-rhub-debian-gcc-release-custom-ccache:
+ ci: azure
+ template: r/azure.linux.yml
+ params:
+ r_org: rhub
+ r_image: debian-gcc-release
+ r_tag: latest
+ r_custom_ccache: true
+
{% for r_org, r_image, r_tag in [("rhub", "ubuntu-gcc-release", "latest"),
("rocker", "r-base", "latest"),
("rstudio", "r-base", "4.1-focal"),
diff --git a/docker-compose.yml b/docker-compose.yml
index f3c67fc4af..cff1a1665c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1209,6 +1209,7 @@ services:
devtoolset_version: ${DEVTOOLSET_VERSION}
tz: ${TZ}
r_prune_deps: ${R_PRUNE_DEPS}
+ r_custom_ccache: ${R_CUSTOM_CCACHE}
shm_size: *shm-size
environment:
LIBARROW_DOWNLOAD: "false"
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index b7ef688b5e..3ab7c30f38 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -301,8 +301,14 @@ build_libarrow <- function(src_dir, dst_dir) {
# EXTRA_CMAKE_FLAGS will often be "", but it's convenient later to have it defined
EXTRA_CMAKE_FLAGS = Sys.getenv("EXTRA_CMAKE_FLAGS"),
# Make sure we build with the same compiler settings that R is using
- CC = R_CMD_config("CC"),
- CXX = paste(R_CMD_config("CXX11"), R_CMD_config("CXX11STD")),
+ # Exception: if you've added ccache to CC and CXX following
+ # http://dirk.eddelbuettel.com/blog/2017/11/27/, some libarrow
+ # third party dependencies will error on compilation. But don't
+ # worry, `ARROW_USE_CCACHE=ON` by default, so if ccache
+ # is found, it will be used by the libarrow build, and this does
+ # not affect how R compiles the arrow bindings.
+ CC = sub("^.*ccache", "", R_CMD_config("CC")),
+ CXX = paste(sub("^.*ccache", "", R_CMD_config("CXX11")), R_CMD_config("CXX11STD")),
# CXXFLAGS = R_CMD_config("CXX11FLAGS"), # We don't want the same debug symbols
LDFLAGS = R_CMD_config("LDFLAGS")
)