You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by th...@apache.org on 2024/03/08 03:05:27 UTC

(arrow) 05/12: GH-39697: [R] Source build should check if offline (#39699)

This is an automated email from the ASF dual-hosted git repository.

thisisnic pushed a commit to branch maint-15.0.1-r
in repository https://gitbox.apache.org/repos/asf/arrow.git

commit 14296caf0ab715bde8001455037bbb78ec8af532
Author: Neal Richardson <ne...@gmail.com>
AuthorDate: Fri Jan 19 18:43:37 2024 -0500

    GH-39697: [R] Source build should check if offline (#39699)
    
    ### Rationale for this change
    
    CRAN.
    
    ### What changes are included in this PR?
    
    See the commit messages
    
    ### Are these changes tested?
    
    Existing CI should pass and not be affected. We should confirm that source builds get all features enabled. We should test on macbuilder with this package and with one where we've inserted `download.file <- function(...) stop()` at the top of nixlibs.R to simulate offline behavior.
    
    ### Are there any user-facing changes?
    
    I hope there is only one user affected.
    * Closes: #39697
    
    Authored-by: Neal Richardson <ne...@gmail.com>
    Signed-off-by: Jacob Wujciak-Jens <ja...@wujciak.de>
---
 dev/tasks/r/github.linux.offline.build.yml |  2 +-
 dev/tasks/tasks.yml                        |  2 +-
 r/NEWS.md                                  |  2 +-
 r/configure                                |  2 +-
 r/tools/nixlibs.R                          | 63 ++++++++++++++++++------------
 r/vignettes/developers/setup.Rmd           |  2 +-
 6 files changed, 43 insertions(+), 30 deletions(-)

diff --git a/dev/tasks/r/github.linux.offline.build.yml b/dev/tasks/r/github.linux.offline.build.yml
index b116accda8..7a747ac480 100644
--- a/dev/tasks/r/github.linux.offline.build.yml
+++ b/dev/tasks/r/github.linux.offline.build.yml
@@ -79,7 +79,7 @@ jobs:
         shell: Rscript {0}
       - name: Install
         env:
-          TEST_OFFLINE_BUILD: true
+          ARROW_OFFLINE_BUILD: true
           LIBARROW_MINIMAL: false
         {{ macros.github_set_sccache_envvars()|indent(8)}}
         run: |
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 04faef427e..5f294a3d4f 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1279,7 +1279,7 @@ tasks:
       r_org: library
       r_image: r-base
       r_tag: latest
-      flags: '-e TEST_OFFLINE_BUILD=true'
+      flags: '-e ARROW_OFFLINE_BUILD=true'
 
   test-r-dev-duckdb:
     ci: github
diff --git a/r/NEWS.md b/r/NEWS.md
index e324ed9731..3b1f083bc2 100644
--- a/r/NEWS.md
+++ b/r/NEWS.md
@@ -32,7 +32,7 @@
 
 ##  Minor improvements and fixes
 
-* Don't download cmake when TEST_OFFLINE_BUILD=true and update `SystemRequirements` (#39602).
+* Don't download cmake when ARROW_OFFLINE_BUILD=true and update `SystemRequirements` (#39602).
 * Fallback to source build gracefully if binary download fails (#39587).
 * An error is now thrown instead of warning and pulling the data into R when any
   of `sub`, `gsub`, `stringr::str_replace`, `stringr::str_replace_all` are
diff --git a/r/configure b/r/configure
index 029fc004df..0882ee6719 100755
--- a/r/configure
+++ b/r/configure
@@ -73,7 +73,7 @@ FORCE_BUNDLED_BUILD=`echo $FORCE_BUNDLED_BUILD | tr '[:upper:]' '[:lower:]'`
 ARROW_USE_PKG_CONFIG=`echo $ARROW_USE_PKG_CONFIG | tr '[:upper:]' '[:lower:]'`
 # Just used in testing: whether or not it is ok to download dependencies (in the
 # bundled build)
-TEST_OFFLINE_BUILD=`echo $TEST_OFFLINE_BUILD | tr '[:upper:]' '[:lower:]'`
+ARROW_OFFLINE_BUILD=`echo $ARROW_OFFLINE_BUILD | tr '[:upper:]' '[:lower:]'`
 
 VERSION=`grep '^Version' DESCRIPTION | sed s/Version:\ //`
 UNAME=`uname -s`
diff --git a/r/tools/nixlibs.R b/r/tools/nixlibs.R
index dfe379ebe2..17c6ab0a80 100644
--- a/r/tools/nixlibs.R
+++ b/r/tools/nixlibs.R
@@ -120,11 +120,11 @@ validate_checksum <- function(binary_url, libfile, hush = quietly) {
     # The warnings from system2 if it fails pop up later in the log and thus are
     # more confusing than they are helpful (so we suppress them)
     checksum_ok <- suppressWarnings(system2(
-        "shasum",
-        args = c("--status", "-a", "512", "-c", checksum_file),
-        stdout = ifelse(quietly, FALSE, ""),
-        stderr = ifelse(quietly, FALSE, "")
-      )) == 0
+      "shasum",
+      args = c("--status", "-a", "512", "-c", checksum_file),
+      stdout = ifelse(quietly, FALSE, ""),
+      stderr = ifelse(quietly, FALSE, "")
+    )) == 0
 
     if (!checksum_ok) {
       checksum_ok <- suppressWarnings(system2(
@@ -565,8 +565,8 @@ build_libarrow <- function(src_dir, dst_dir) {
     env_var_list <- c(env_var_list, ARROW_DEPENDENCY_SOURCE = "BUNDLED")
   }
 
-  # On macOS, if not otherwise set, let's override Boost_SOURCE to be bundled 
-  # Necessary due to #39590 for CRAN 
+  # On macOS, if not otherwise set, let's override Boost_SOURCE to be bundled
+  # Necessary due to #39590 for CRAN
   if (on_macos) {
     # Using lowercase (e.g. Boost_SOURCE) to match the cmake args we use already.
     deps_to_bundle <- c("Boost", "lz4")
@@ -906,28 +906,13 @@ on_windows <- tolower(Sys.info()[["sysname"]]) == "windows"
 # For local debugging, set ARROW_R_DEV=TRUE to make this script print more
 quietly <- !env_is("ARROW_R_DEV", "true")
 
-not_cran <- env_is("NOT_CRAN", "true")
-
-if (is_release) {
-  VERSION <- VERSION[1, 1:3]
-  arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/")
-} else {
-  # Don't override explictily set NOT_CRAN env var, as it is used in CI.
-  not_cran <- !env_is("NOT_CRAN", "false")
-  arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/")
-}
-
-if (!is_release && !test_mode) {
-  VERSION <- find_latest_nightly(VERSION)
-}
-
 # To collect dirs to rm on exit, use cleanup() to add dirs
 # we reset it to avoid errors on reruns in the same session.
 options(.arrow.cleanup = character())
 on.exit(unlink(getOption(".arrow.cleanup"), recursive = TRUE), add = TRUE)
 
-# enable full featured builds for macOS in case of CRAN source builds.
-if (not_cran || on_macos) {
+not_cran <- env_is("NOT_CRAN", "true")
+if (not_cran) {
   # Set more eager defaults
   if (env_is("LIBARROW_BINARY", "")) {
     Sys.setenv(LIBARROW_BINARY = "true")
@@ -943,9 +928,37 @@ if (not_cran || on_macos) {
 build_ok <- !env_is("LIBARROW_BUILD", "false")
 
 # Check if we're authorized to download
-download_ok <- !test_mode && !env_is("TEST_OFFLINE_BUILD", "true")
+download_ok <- !test_mode && !env_is("ARROW_OFFLINE_BUILD", "true")
+if (!download_ok) {
+  lg("Dependency downloading disabled. Unset ARROW_OFFLINE_BUILD to enable", .indent = "***")
+}
+# If not forbidden from downloading, check if we are offline and turn off downloading.
+# The default libarrow source build will download its source dependencies and fail
+# if they can't be retrieved.
+# But, don't do this if the user has requested a binary or a non-minimal build:
+# we should error rather than silently succeeding with a minimal build.
+if (download_ok && Sys.getenv("LIBARROW_BINARY") %in% c("false", "") && !env_is("LIBARROW_MINIMAL", "false")) {
+  download_ok <- try_download("https://apache.jfrog.io/artifactory/arrow/r/", tempfile())
+  if (!download_ok) {
+    lg("Network connection not available", .indent = "***")
+  }
+}
+
 download_libarrow_ok <- download_ok && !env_is("LIBARROW_DOWNLOAD", "false")
 
+# Set binary repos
+if (is_release) {
+  VERSION <- VERSION[1, 1:3]
+  arrow_repo <- paste0(getOption("arrow.repo", sprintf("https://apache.jfrog.io/artifactory/arrow/r/%s", VERSION)), "/libarrow/")
+} else {
+  arrow_repo <- paste0(getOption("arrow.dev_repo", "https://nightlies.apache.org/arrow/r"), "/libarrow/")
+}
+
+# If we're on a dev version, look for the most recent libarrow binary version
+if (download_libarrow_ok && !is_release && !test_mode) {
+  VERSION <- find_latest_nightly(VERSION)
+}
+
 # This "tools/thirdparty_dependencies" path, within the tar file, might exist if
 # create_package_with_all_dependencies() was run, or if someone has created it
 # manually before running make build.
diff --git a/r/vignettes/developers/setup.Rmd b/r/vignettes/developers/setup.Rmd
index 119bc78419..4c1eab1e69 100644
--- a/r/vignettes/developers/setup.Rmd
+++ b/r/vignettes/developers/setup.Rmd
@@ -280,7 +280,7 @@ withr::with_makevars(list(CPPFLAGS = "", LDFLAGS = ""), remotes::install_github(
 * See the user-facing [article on installation](../install.html) for a large number of
   environment variables that determine how the build works and what features
   get built.
-* `TEST_OFFLINE_BUILD`: When set to `true`, the build script will not download
+* `ARROW_OFFLINE_BUILD`: When set to `true`, the build script will not download
   prebuilt the C++ library binary or, if needed, `cmake`.
   It will turn off any features that require a download, unless they're available
   in `ARROW_THIRDPARTY_DEPENDENCY_DIR` or the `tools/thirdparty_download/` subfolder.