You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2022/05/18 21:40:04 UTC
[GitHub] [arrow] jonkeane commented on a diff in pull request #13149: ARROW-16403:[R][CI] Create Crossbow task for R nightly builds
jonkeane commented on code in PR #13149:
URL: https://github.com/apache/arrow/pull/13149#discussion_r876403772
##########
dev/tasks/r/github.nightly.yml:
##########
@@ -0,0 +1,381 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+{% import 'macros.jinja' as macros with context %}
+{% set r_root = "nightly/r" %}
+{% set webdav_domain = "https://nightly.wujciak.de:8080" %}
+{% set repo_domain = "https://nightly.wujciak.de/r" %}
+
+{{ macros.github_header() }}
+
+jobs:
+ source:
+ name: Source Package
+ runs-on: ubuntu-latest
+ outputs:
+ version: {{ '${{ steps.save-version.outputs.version }}' }}
+ date: {{ '${{ steps.save-version.outputs.date }}' }}
+ steps:
+ {{ macros.github_checkout_arrow()|indent }}
+ {{ macros.change_r_pkg_version()|indent }}
+ - name: Save Version
+ id: save-version
+ shell: bash
+ run: |
+ echo "::set-output name=version::$(grep ^Version arrow/r/DESCRIPTION | sed s/Version:\ //)"
+ echo "::set-output name=date::$(date +%Y%m%d)"
+
+ - uses: r-lib/actions/setup-r@v2
+ with:
+ install-r: false
+
+ - name: Build R source package
+ shell: bash
+ run: |
+ cd arrow/r
+ # Copy in the Arrow C++ source
+ make sync-cpp
+ R CMD build --no-build-vignettes .
+
+ - name: Install davfs2 & Mount Repo
+ run: |
+ sudo apt update && sudo apt install davfs2
+ mkdir nightly
+ sudo bash -c 'echo "{{ webdav_domain }} {{ '${{ secrets.CROSSBOW_NIGHTLIES_USER }} ${{ secrets.CROSSBOW_NIGHTLIES_TOKEN }}' }}" >> /etc/davfs2/secrets'
+ sudo mount -t davfs {{ webdav_domain }} nightly -o rw
+
+ - name: Upload Source Package
+ run: |
+ # ensure repo structure is set up, this job is the fastest and should fix any issue before the other jobs try
+ # to push into non existent folder with curl, which would fail silently.
+ sudo mkdir -p {{ r_root }}/src/contrib
+ {% for os in ["ubuntu-18.04",
+ "centos-7",
+ "windows"] %}
+ sudo mkdir -p {{ r_root }}/libarrow/bin/{{ os }}
+ {% endfor %}
+ {% for os in ["windows", "macosx", "macosx/big-sur-arm64"] %}
+ {% for r_version in ["4.1", "4.2"] %}
+ sudo mkdir -p {{ r_root }}/bin/{{ os }}/contrib/{{ r_version }}
+ {% endfor %}
+ {% endfor %}
+
+ sudo cp arrow/r/arrow_*.tar.gz {{ r_root }}/src/contrib
+ - name: Update Repo
+ shell: sudo Rscript {0}
+ run: |
+ if(file.exists("{{ r_root }}/src/contrib/PACKAGES")) {
+ tools::update_PACKAGES("{{ r_root }}/src/contrib" , type = "source", latestOnly = FALSE)
+ } else {
+ tools::write_PACKAGES("{{ r_root }}/src/contrib" , type = "source", latestOnly = FALSE)
+ }
+
+ # ensures all changes are written
+ - run: sudo umount nightly
+ - name: Upload binary artifact (temp)
+ uses: actions/upload-artifact@v3
+ with:
+ name: r-src-pkg
+ path: arrow/r/arrow_*.tar.gz
+
+ linux-cpp:
+ name: C++ Binary {{ '${{ matrix.config.os }}-${{ matrix.config.version }}' }}
+ runs-on: ubuntu-latest
+ needs: source
+ strategy:
+ fail-fast: false
+ matrix:
+ config:
+ - { os: ubuntu, version: "18.04" }
+ - { os: centos, version: "7" }
+ env:
+ UBUNTU: {{ '${{ matrix.config.version }}' }}
+ R: 3.6
Review Comment:
Could we? Should we? bump this version?
##########
dev/tasks/macros.jinja:
##########
@@ -221,3 +222,35 @@ on:
cp ${formula} $(brew --repository homebrew/core)/Formula/
done
{% endmacro %}
+
+{%- macro change_r_pkg_version(date = '$(date +%Y%m%d)') -%}
+ - name: Modify version
+ shell: bash
+ run: |
+ cd arrow/r
+ sed -i.bak -E -e \
+ 's/(^Version: [0-9]+\.[0-9]+\.[0-9]+).*$/\1.'"{{ date }}"'/' \
+ DESCRIPTION
+ head DESCRIPTION
+ rm -f DESCRIPTION.bak
+ cp ../dev/tasks/homebrew-formulae/autobrew/apache-arrow.rb tools/apache-arrow.rb
+
+ # Pin the git commit in the formula to match
+ cd tools
+ sed -i.bak -E -e 's/arrow.git"$/arrow.git", :revision => "'"{{ arrow.head }}"'"/' apache-arrow.rb
+ rm -f apache-arrow.rb.bak
+{% endmacro %}
+
+{%- macro test_r_src_pkg() -%}
+ source("https://raw.githubusercontent.com/apache/arrow/master/ci/etc/rprofile")
+ options(arrow.dev_repo = "https://nightly.wujciak.de/r")
+
+ install.packages(
+ "arrow",
+ repos = c("https://nightly.wujciak.de/r", getOption("repos")),
+ verbose = TRUE
+ )
+ library(arrow)
+ read_parquet(system.file("v0.7.1.parquet", package = "arrow"))
+ stopifnot(packageVersion("arrow") == {{ '"${{needs.source.outputs.version}}"' }})
+{% endmacro %}
Review Comment:
Is the indention here what we want?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: github-unsubscribe@arrow.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org