You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ko...@apache.org on 2022/11/04 21:41:55 UTC
[arrow] branch master updated: ARROW-18016: [CI] Add sccache to r jobs (#14570)
This is an automated email from the ASF dual-hosted git repository.
kou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a3ef64b958 ARROW-18016: [CI] Add sccache to r jobs (#14570)
a3ef64b958 is described below
commit a3ef64b9583c3a92fac73e74e9e03e5d22a6d8f6
Author: Jacob Wujciak-Jens <ja...@wujciak.de>
AuthorDate: Fri Nov 4 22:41:46 2022 +0100
ARROW-18016: [CI] Add sccache to r jobs (#14570)
This is #14390 with reduced scope to make it possible to review. I have added sccache to all jobs in the `r` group except the conda feedstock jobs: `conda-*-r*` as those pull external images where sccache can't be easily added. I also did not add sccache to the rchk and sanitizer jobs as to not interfere with the results.
Lead-authored-by: Jacob Wujciak-Jens <ja...@wujciak.de>
Co-authored-by: Sutou Kouhei <ko...@cozmixng.org>
Signed-off-by: Sutou Kouhei <ko...@clear-code.com>
---
ci/docker/linux-r.dockerfile | 4 +
ci/scripts/install_sccache.sh | 14 ++-
ci/scripts/r_docker_configure.sh | 4 +-
dev/tasks/docker-tests/azure.linux.yml | 2 +
dev/tasks/macros.jinja | 15 ++-
dev/tasks/r/azure.linux.yml | 5 +
.../r/github.linux.arrow.version.back.compat.yml | 8 +-
dev/tasks/r/github.linux.cran.yml | 2 +
dev/tasks/r/github.linux.offline.build.yml | 6 +-
dev/tasks/r/github.linux.versions.yml | 2 +
dev/tasks/r/github.macos-linux.local.yml | 25 ++--
dev/tasks/r/github.macos.autobrew.yml | 4 +-
dev/tasks/r/github.macos.brew.yml | 6 +-
dev/tasks/tasks.yml | 127 ++++++++++++---------
docker-compose.yml | 5 +-
15 files changed, 148 insertions(+), 81 deletions(-)
diff --git a/ci/docker/linux-r.dockerfile b/ci/docker/linux-r.dockerfile
index 96420fcfc6..db5e5f656d 100644
--- a/ci/docker/linux-r.dockerfile
+++ b/ci/docker/linux-r.dockerfile
@@ -49,6 +49,10 @@ COPY ci/scripts/install_minio.sh /arrow/ci/scripts/
COPY ci/scripts/install_gcs_testbench.sh /arrow/ci/scripts/
RUN /arrow/ci/scripts/r_docker_configure.sh
+# this has to come after r_docker_configure to ensure curl is installed
+COPY ci/scripts/install_sccache.sh /arrow/ci/scripts/
+RUN /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
+
# Set up Python 3 and its dependencies
RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip
diff --git a/ci/scripts/install_sccache.sh b/ci/scripts/install_sccache.sh
index 34485577a4..2ee3486699 100755
--- a/ci/scripts/install_sccache.sh
+++ b/ci/scripts/install_sccache.sh
@@ -41,14 +41,24 @@ SCCACHE_ARCHIVE=sccache.tar.gz
# Download archive and checksum
curl -L $SCCACHE_URL --output $SCCACHE_ARCHIVE
curl -L $SCCACHE_URL.sha256 --output $SCCACHE_ARCHIVE.sha256
+echo " $SCCACHE_ARCHIVE" >> $SCCACHE_ARCHIVE.sha256
-echo "$(cat $SCCACHE_ARCHIVE.sha256) $SCCACHE_ARCHIVE" | sha256sum --check --status
+SHA_ARGS="--check --status"
+
+# Busybox sha256sum uses different flags
+if sha256sum --version 2>&1 | grep -q BusyBox; then
+ SHA_ARGS="-sc"
+fi
+
+sha256sum $SHA_ARGS $SCCACHE_ARCHIVE.sha256
if [ ! -d $PREFIX ]; then
mkdir -p $PREFIX
fi
-tar -xzvf $SCCACHE_ARCHIVE --strip-component=1 --directory $PREFIX --wildcards sccache*/sccache*
+# Extract only the sccache binary into $PREFIX and ignore README and LCIENSE.
+# --wildcards doesn't work on busybox.
+tar -xzvf $SCCACHE_ARCHIVE --strip-component=1 --directory $PREFIX --exclude="sccache*/*E*E*"
chmod u+x $PREFIX/sccache
if [ "${GITHUB_ACTIONS}" = "true" ]; then
diff --git a/ci/scripts/r_docker_configure.sh b/ci/scripts/r_docker_configure.sh
index c801f90d41..c671271e22 100755
--- a/ci/scripts/r_docker_configure.sh
+++ b/ci/scripts/r_docker_configure.sh
@@ -105,8 +105,8 @@ if [ "$ARROW_S3" == "ON" ] || [ "$ARROW_GCS" == "ON" ] || [ "$ARROW_R_DEV" == "T
fi
fi
-# Install rsync for bundling cpp source
-$PACKAGE_MANAGER install -y rsync
+# Install rsync for bundling cpp source and curl to make sure it is installed on all images
+$PACKAGE_MANAGER install -y rsync curl
# Workaround for html help install failure; see https://github.com/r-lib/devtools/issues/2084#issuecomment-530912786
Rscript -e 'x <- file.path(R.home("doc"), "html"); if (!file.exists(x)) {dir.create(x, recursive=TRUE); file.copy(system.file("html/R.css", package="stats"), x)}'
diff --git a/dev/tasks/docker-tests/azure.linux.yml b/dev/tasks/docker-tests/azure.linux.yml
index db310fd05e..8f12bcf63b 100644
--- a/dev/tasks/docker-tests/azure.linux.yml
+++ b/dev/tasks/docker-tests/azure.linux.yml
@@ -59,6 +59,8 @@ jobs:
{{ image }} \
{{ command|default("") }}
displayName: Execute Docker Build
+ env:
+ {{ macros.azure_set_sccache_envvars()|indent(4) }}
{% if post_script is defined %}
- script: |
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index bd3358e073..03b95a4ae7 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -373,8 +373,8 @@ on:
{% macro github_set_sccache_envvars(sccache_key_prefix = "sccache") %}
{% set sccache_vars = {
- "AWS_SECRET_ACCESS_KEY": '${{ secrets.AWS_SECRET_ACCESS_KEY }}',
"AWS_ACCESS_KEY_ID": '${{ secrets.AWS_ACCESS_KEY_ID }}',
+ "AWS_SECRET_ACCESS_KEY": '${{ secrets.AWS_SECRET_ACCESS_KEY }}',
"SCCACHE_BUCKET": '${{ secrets.SCCACHE_BUCKET }}',
"SCCACHE_S3_KEY_PREFIX": sccache_key_prefix
}
@@ -383,3 +383,16 @@ on:
{{ key }}: "{{ value }}"
{% endfor %}
{% endmacro %}
+
+{% macro azure_set_sccache_envvars(sccache_key_prefix = "sccache") %}
+ {% set sccache_vars = {
+ "AWS_ACCESS_KEY_ID": '$(AWS_ACCESS_KEY_ID)',
+ "AWS_SECRET_ACCESS_KEY": '$(AWS_SECRET_ACCESS_KEY)',
+ "SCCACHE_BUCKET": '$(SCCACHE_BUCKET)',
+ "SCCACHE_S3_KEY_PREFIX": sccache_key_prefix
+ }
+ %}
+ {% for key, value in sccache_vars.items() %}
+ {{ key }}: "{{ value }}"
+ {% endfor %}
+{% endmacro %}
diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
index 4528390298..0cbc70ed00 100644
--- a/dev/tasks/r/azure.linux.yml
+++ b/dev/tasks/r/azure.linux.yml
@@ -13,6 +13,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+{% import 'macros.jinja' as macros with context %}
jobs:
- job: linux
@@ -47,6 +48,8 @@ jobs:
docker-compose pull --ignore-pull-failures r
docker-compose build r
displayName: Docker build
+ env:
+ {{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
@@ -59,6 +62,8 @@ jobs:
# if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
docker-compose run {{ flags|default("") }} r
displayName: Docker run
+ env:
+ {{ macros.azure_set_sccache_envvars()|indent(8) }}
- script: |
set -ex
diff --git a/dev/tasks/r/github.linux.arrow.version.back.compat.yml b/dev/tasks/r/github.linux.arrow.version.back.compat.yml
index 89c244c6e0..795a2076fd 100644
--- a/dev/tasks/r/github.linux.arrow.version.back.compat.yml
+++ b/dev/tasks/r/github.linux.arrow.version.back.compat.yml
@@ -31,7 +31,7 @@ jobs:
steps:
{{ macros.github_checkout_arrow()|indent }}
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
- name: Prepare PKG_CONFIG_PATH for Homebrew
run: |
# zstd is installed by Homebrew on GitHub Actions.
@@ -42,7 +42,11 @@ jobs:
install.packages(c("remotes", "glue", "sys"))
remotes::install_deps("arrow/r", dependencies = TRUE)
shell: Rscript {0}
+ - name: Install sccache
+ run: arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
- name: Install Arrow
+ env:
+ {{ macros.github_set_sccache_envvars()|indent(8) }}
run: |
cd arrow/r
R CMD INSTALL .
@@ -83,7 +87,7 @@ jobs:
steps:
{{ macros.github_checkout_arrow()|indent }}
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
with:
r-version: {{ '${{ matrix.config.r }}' }}
- name: Install old Arrow
diff --git a/dev/tasks/r/github.linux.cran.yml b/dev/tasks/r/github.linux.cran.yml
index 5da0b377dd..0aeb7cfa2b 100644
--- a/dev/tasks/r/github.linux.cran.yml
+++ b/dev/tasks/r/github.linux.cran.yml
@@ -44,6 +44,8 @@ jobs:
- name: Docker Run
shell: bash
+ env:
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
# setting ARROW_SOURCE_HOME='' here ensures that we use the cpp source copied into tools/
run: archery docker run -e ARROW_SOURCE_HOME='' r
- name: Dump install logs
diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml
index bf43174ff7..bc97e1bfd0 100644
--- a/dev/tasks/r/github.linux.offline.build.yml
+++ b/dev/tasks/r/github.linux.offline.build.yml
@@ -31,7 +31,7 @@ jobs:
steps:
{{ macros.github_checkout_arrow()|indent }}
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
- name: Pull Arrow dependencies
run: |
cd arrow/r
@@ -58,7 +58,7 @@ jobs:
steps:
{{ macros.github_checkout_arrow()|indent }}
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
- name: Download artifacts
uses: actions/download-artifact@v2
with:
@@ -68,6 +68,7 @@ jobs:
run: |
sudo apt-get update
sudo apt install libcurl4-openssl-dev libssl-dev
+ arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
- name: Prepare PKG_CONFIG_PATH for Homebrew
run: |
# zstd is installed by Homebrew on GitHub Actions.
@@ -82,6 +83,7 @@ jobs:
env:
TEST_OFFLINE_BUILD: true
LIBARROW_MINIMAL: false
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
run: |
cd arrow/r
R CMD INSTALL --install-tests --no-test-load --no-docs --no-help --no-byte-compile arrow_with_deps.tar.gz
diff --git a/dev/tasks/r/github.linux.versions.yml b/dev/tasks/r/github.linux.versions.yml
index b8ebc9c232..07e55dda6b 100644
--- a/dev/tasks/r/github.linux.versions.yml
+++ b/dev/tasks/r/github.linux.versions.yml
@@ -46,6 +46,8 @@ jobs:
- name: Docker Run
shell: bash
+ env:
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
run: archery docker run r
- name: Dump install logs
run: cat arrow/r/check/arrow.Rcheck/00install.out
diff --git a/dev/tasks/r/github.macos-linux.local.yml b/dev/tasks/r/github.macos-linux.local.yml
index aaefafdce1..fd2345e918 100644
--- a/dev/tasks/r/github.macos-linux.local.yml
+++ b/dev/tasks/r/github.macos-linux.local.yml
@@ -39,31 +39,27 @@ jobs:
- name: Configure non-autobrew dependencies (macos)
run: |
- cd arrow/r
brew install openssl
+ brew install sccache
if: contains(matrix.os, 'macOS')
- name: Configure non-autobrew dependencies (linux)
run: |
- cd arrow/r
sudo apt-get update
sudo apt install libcurl4-openssl-dev libssl-dev
+ arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin
# zstd is installed by Homebrew on GitHub Actions.
echo "PKG_CONFIG_PATH=$(brew --prefix)/lib/pkgconfig${PKG_CONFIG_PATH:+:${PKG_CONFIG_PATH}}" \
>> "$GITHUB_ENV"
if: contains(matrix.os, 'ubuntu')
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
- name: Install dependencies
- run: |
- install.packages("remotes")
- remotes::install_deps("arrow/r", dependencies = TRUE)
- remotes::install_cran(c("rcmdcheck", "sys", "sessioninfo"))
- shell: Rscript {0}
- - name: Session info
- run: |
- options(width = 100)
- pkgs <- installed.packages()[, "Package"]
- sessioninfo::session_info(pkgs, include_base = TRUE)
- shell: Rscript {0}
+ uses: r-lib/actions/setup-r-dependencies@v2
+ with:
+ cache: false # cache does not work on across branches
+ working-directory: arrow/r
+ extra-packages: |
+ any::rcmdcheck
+ any::sys
- name: Install
env:
_R_CHECK_CRAN_INCOMING_: false
@@ -71,6 +67,7 @@ jobs:
FORCE_BUNDLED_BUILD: true
LIBARROW_MINIMAL: false
ARROW_R_DEV: TRUE
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
run: |
cd arrow/r
R CMD INSTALL . --install-tests
diff --git a/dev/tasks/r/github.macos.autobrew.yml b/dev/tasks/r/github.macos.autobrew.yml
index 2664455479..67157e854f 100644
--- a/dev/tasks/r/github.macos.autobrew.yml
+++ b/dev/tasks/r/github.macos.autobrew.yml
@@ -37,10 +37,11 @@ jobs:
{{ macros.github_checkout_arrow()|indent }}
- name: Configure autobrew script
run: |
- # minio is pre-installed on the self-hosted 10.13 runer
+ # minio and sccache are pre-installed on the self-hosted 10.13 runner
if [ {{ '${{ matrix.platform }}' }} != macos-10.13 ]; then
# install minio for tests
brew install minio
+ brew install sccache
fi
cd arrow/r
{{ macros.pin_brew_formulae(is_fork)|indent }}
@@ -70,6 +71,7 @@ jobs:
NOT_CRAN: true
ARROW_USE_PKG_CONFIG: false
ARROW_R_DEV: true
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
run: arrow/ci/scripts/r_test.sh arrow
- name: Dump install logs
run: cat arrow/r/check/arrow.Rcheck/00install.out
diff --git a/dev/tasks/r/github.macos.brew.yml b/dev/tasks/r/github.macos.brew.yml
index c2cbcc5e3b..5f426ab42c 100644
--- a/dev/tasks/r/github.macos.brew.yml
+++ b/dev/tasks/r/github.macos.brew.yml
@@ -27,9 +27,11 @@ jobs:
{{ macros.github_checkout_arrow()|indent }}
{{ macros.configure_homebrew_arrow(formula)|indent }}
-
- name: Install apache-arrow
+ env:
+ {{ macros.github_set_sccache_envvars()|indent(8)}}
run: |
+ brew install sccache
# TODO: Update the TODO for ARROW-16907 below to refer to main instead of master
# after migrating the default branch to main.
# TODO(ARROW-16907): apache/arrow@master seems to be installed already
@@ -37,7 +39,7 @@ jobs:
brew install -v --HEAD apache-arrow
# for testing
brew install minio
- - uses: r-lib/actions/setup-r@v1
+ - uses: r-lib/actions/setup-r@v2
- name: Install dependencies
run: |
install.packages("remotes")
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 79795bd736..be51c9a02b 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -17,7 +17,7 @@
groups:
# these groups are just for convenience
- # makes it easier to submit related tasks
+ # makes it easier to submit related taskshttps://github.com/github/release-radar
{############################# Packaging tasks ###############################}
@@ -250,7 +250,7 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython
+ config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython
r_config: linux_64_r_base4.0
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -260,7 +260,7 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.7.____cpython
+ config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.7.____cpython
r_config: linux_64_r_base4.1
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -270,7 +270,7 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.8.____cpython
+ config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.18python3.8.____cpython
artifacts:
- arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -279,7 +279,7 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.20python3.9.____cpython
+ config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.19python3.9.____cpython
artifacts:
- arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -288,14 +288,14 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_64_c_compiler_version10cuda_compiler_versionNonecxx_compiler_version10numpy1.21python3.10.____cpython
+ config: linux_64_c_compiler_version9cuda_compiler_versionNonecxx_compiler_version9numpy1.21python3.10.____cpython
artifacts:
- arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
-{% for python_version, numpy_version in [("3.7", "1.20"),
- ("3.8", "1.20"),
- ("3.9", "1.20"),
+{% for python_version, numpy_version in [("3.7", "1.18"),
+ ("3.8", "1.18"),
+ ("3.9", "1.19"),
("3.10", "1.21")] %}
{% set pyver = python_version | replace(".", "") %}
@@ -312,7 +312,7 @@ tasks:
ci: azure
template: conda-recipes/azure.linux.yml
params:
- config: linux_aarch64_cuda_compiler_versionNonenumpy{{ numpy_version }}python{{ python_version }}.____cpython
+ config: linux_aarch64_numpy{{ numpy_version }}python{{ python_version }}.____cpython
artifacts:
- arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -334,7 +334,7 @@ tasks:
ci: azure
template: conda-recipes/azure.osx.yml
params:
- config: osx_64_numpy1.20python3.7.____cpython
+ config: osx_64_numpy1.18python3.7.____cpython
r_config: osx_64_r_base4.0
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -344,37 +344,65 @@ tasks:
ci: azure
template: conda-recipes/azure.osx.yml
params:
- config: osx_64_numpy1.20python3.7.____cpython
+ config: osx_64_numpy1.18python3.7.____cpython
r_config: osx_64_r_base4.1
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
-{% for python_version, numpy_version in [("3.7", "1.20"),
- ("3.8", "1.20"),
- ("3.9", "1.20"),
- ("3.10", "1.21")] %}
- {% set pyver = python_version | replace(".", "") %}
+ conda-osx-clang-py38:
+ ci: azure
+ template: conda-recipes/azure.osx.yml
+ params:
+ config: osx_64_numpy1.18python3.8.____cpython
+ artifacts:
+ - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
- conda-osx-clang-py{{ pyver }}:
+ conda-osx-clang-py39:
ci: azure
template: conda-recipes/azure.osx.yml
params:
- config: osx_64_numpy{{ numpy_version }}python{{ python_version }}.____cpython
+ config: osx_64_numpy1.19python3.9.____cpython
artifacts:
- - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
- - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
+ - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
- conda-osx-arm64-clang-py{{ pyver }}:
+ conda-osx-clang-py310:
ci: azure
template: conda-recipes/azure.osx.yml
params:
- config: osx_arm64_numpy{{ numpy_version }}python{{ python_version }}.____cpython
+ config: osx_64_numpy1.21python3.10.____cpython
artifacts:
- - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
- - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
+ - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
-{% endfor %}
+ conda-osx-arm64-clang-py38:
+ ci: azure
+ template: conda-recipes/azure.osx.yml
+ params:
+ config: osx_arm64_numpy1.19python3.8.____cpython
+ artifacts:
+ - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+
+ conda-osx-arm64-clang-py39:
+ ci: azure
+ template: conda-recipes/azure.osx.yml
+ params:
+ config: osx_arm64_numpy1.19python3.9.____cpython
+ artifacts:
+ - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+
+ conda-osx-arm64-clang-py310:
+ ci: azure
+ template: conda-recipes/azure.osx.yml
+ params:
+ config: osx_arm64_numpy1.21python3.10.____cpython
+ artifacts:
+ - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
############################## Conda Windows ################################
@@ -382,7 +410,7 @@ tasks:
ci: azure
template: conda-recipes/azure.win.yml
params:
- config: win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython
+ config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython
r_config: win_64_r_base4.0
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
@@ -392,37 +420,40 @@ tasks:
ci: azure
template: conda-recipes/azure.win.yml
params:
- config: win_64_cuda_compiler_versionNonenumpy1.20python3.7.____cpython
+ config: win_64_cuda_compiler_versionNonenumpy1.18python3.7.____cpython
r_config: win_64_r_base4.0
artifacts:
- arrow-cpp-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
- pyarrow-{no_rc_version}-py37(h[a-z0-9]+)_0_cpu.tar.bz2
-{% for python_version, numpy_version in [("3.7", "1.20"),
- ("3.8", "1.20"),
- ("3.9", "1.20"),
- ("3.10", "1.21")] %}
- {% set pyver = python_version | replace(".", "") %}
+ conda-win-vs2019-py38:
+ ci: azure
+ template: conda-recipes/azure.win.yml
+ params:
+ config: win_64_cuda_compiler_versionNonenumpy1.18python3.8.____cpython
+ artifacts:
+ - arrow-cpp-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py38(h[a-z0-9]+)_0_cpu.tar.bz2
- conda-win-vs2019-py{{ pyver }}-cpu:
+ conda-win-vs2019-py39:
ci: azure
template: conda-recipes/azure.win.yml
params:
- config: win_64_cuda_compiler_versionNonenumpy{{ numpy_version }}python{{ python_version }}.____cpython
+ config: win_64_cuda_compiler_versionNonenumpy1.19python3.9.____cpython
artifacts:
- - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
- - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
+ - arrow-cpp-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py39(h[a-z0-9]+)_0_cpu.tar.bz2
- conda-win-vs2019-py{{ pyver }}-cuda:
+ conda-win-vs2019-py310:
ci: azure
template: conda-recipes/azure.win.yml
params:
- config: win_64_cuda_compiler_version10.2numpy{{ numpy_version }}python{{ python_version }}.____cpython
+ config: win_64_cuda_compiler_versionNonenumpy1.21python3.10.____cpython
artifacts:
- - arrow-cpp-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
- - pyarrow-{no_rc_version}-py{{ pyver }}(h[a-z0-9]+)_0_cpu.tar.bz2
+ - arrow-cpp-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
+ - pyarrow-{no_rc_version}-py310(h[a-z0-9]+)_0_cpu.tar.bz2
-{% endfor %}
+# TODO: Windows CUDA
{% for python_version, python_tag, abi_tag in [("3.7", "cp37", "cp37m"),
("3.8", "cp38", "cp38"),
@@ -1434,27 +1465,17 @@ tasks:
- docs.tar.gz
############################## CUDA tests #################################
- # The CUDA image has to match the runner image 1:1 to work
- # so these values are hardcoded to override any changes to .env
- {% set cuda_ver='11.4.2' %}
- {% set cuda_ubuntu='20.04' %}
-
+
test-cuda-cpp:
ci: github
template: docker-tests/github.cuda.yml
params:
- env:
- CUDA: {{ cuda_ver }}
- UBUNTU: {{ cuda_ubuntu }}
image: ubuntu-cuda-cpp
test-cuda-python:
ci: github
template: docker-tests/github.cuda.yml
params:
- env:
- CUDA: {{ cuda_ver }}
- UBUNTU: {{ cuda_ubuntu }}
image: ubuntu-cuda-python
############################## Fuzz tests #################################
diff --git a/docker-compose.yml b/docker-compose.yml
index e88beb08eb..e0424f6fe4 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1268,7 +1268,7 @@ services:
r_duckdb_dev: ${R_DUCKDB_DEV:-}
shm_size: *shm-size
environment:
- <<: *ccache
+ <<: [*ccache, *sccache]
ARROW_R_CXXFLAGS: '-Werror'
ARROW_FLIGHT: 'ON'
LIBARROW_BUILD: 'false'
@@ -1314,6 +1314,7 @@ services:
r_custom_ccache: ${R_CUSTOM_CCACHE}
shm_size: *shm-size
environment:
+ <<: *sccache
LIBARROW_DOWNLOAD: "false"
ARROW_SOURCE_HOME: "/arrow"
ARROW_R_DEV: ${ARROW_R_DEV}
@@ -1395,7 +1396,7 @@ services:
r_bin: RDvalgrind
tz: ${TZ}
environment:
- <<: *ccache
+ <<: [*ccache, *sccache]
ARROW_R_DEV: ${ARROW_R_DEV}
# AVX512 not supported by Valgrind (similar to ARROW-9851) some runners support AVX512 and some do not
# so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner.