You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/08/13 13:23:04 UTC
[arrow] branch master updated: ARROW-6181: [R] Only allow R package
to install without libarrow on linux
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new cb9f718 ARROW-6181: [R] Only allow R package to install without libarrow on linux
cb9f718 is described below
commit cb9f718162370b8be21d30bd2c3e45197c56424e
Author: Neal Richardson <ne...@gmail.com>
AuthorDate: Tue Aug 13 08:22:54 2019 -0500
ARROW-6181: [R] Only allow R package to install without libarrow on linux
Relevant changes here:
* Ensure that on macOS and Windows, the `ARROW_R_WITH_ARROW` option is always set. This means that if the library is actually not present, some later compilation step will fail. This means we won't be able to get useless binaries on CRAN for these platforms.
* Additional debugging messages have been added to the macOS `configure` logic so that if the building on CRAN fails, we'll hopefully get some insight as to where it failed.
* Windows `rwinlib` downloading tries more major versions if one is not found for the current R package version. In the current case, if there is no windows libarrow for `0.14.1.1` or `0.14.1.9000` (there is not), it will then try `0.14.1`. If there is no binary found for that version, it will try going back to the last minor release (`0.14.0`).
* Since it is no longer possible to install the package on macOS or Windows and have `arrow_available()` be `FALSE`, the installation advice specific to these platforms has been removed from `install_arrow()`.
Closes #5045 from nealrichardson/always-with-libarrow and squashes the following commits:
25ed81e44 <Neal Richardson> Revise install_arrow since it should no longer be possible to install on macOS or Windows without libarrow
e8913d1b8 <Neal Richardson> Revise configure scripts so that Windows and macOS never install without Arrow C++
Authored-by: Neal Richardson <ne...@gmail.com>
Signed-off-by: Wes McKinney <we...@apache.org>
---
r/.Rbuildignore | 1 +
r/NAMESPACE | 1 -
r/R/install-arrow.R | 45 +++++------------------------------
r/README.md | 2 +-
r/configure | 45 +++++++++++++++++++----------------
r/configure.win | 20 ++++++++--------
r/tests/testthat/test-install-arrow.R | 24 -------------------
r/tools/winlibs.R | 14 ++++++++++-
8 files changed, 55 insertions(+), 97 deletions(-)
diff --git a/r/.Rbuildignore b/r/.Rbuildignore
index af5457a..53f63e4 100644
--- a/r/.Rbuildignore
+++ b/r/.Rbuildignore
@@ -15,3 +15,4 @@ clang_format.sh
^docs$
^pkgdown$
^Makefile$
+^.*\.orig$
diff --git a/r/NAMESPACE b/r/NAMESPACE
index 5f29a61..0c1ae8c 100644
--- a/r/NAMESPACE
+++ b/r/NAMESPACE
@@ -215,6 +215,5 @@ importFrom(tidyselect,num_range)
importFrom(tidyselect,one_of)
importFrom(tidyselect,starts_with)
importFrom(tidyselect,vars_select)
-importFrom(utils,packageDescription)
importFrom(utils,packageVersion)
useDynLib(arrow, .registration = TRUE)
diff --git a/r/R/install-arrow.R b/r/R/install-arrow.R
index 423f95a..465dba7 100644
--- a/r/R/install-arrow.R
+++ b/r/R/install-arrow.R
@@ -22,21 +22,17 @@
#' first. This function offers guidance on how to get the C++ library depending
#' on your operating system and package version.
#' @export
-#' @importFrom utils packageVersion packageDescription
+#' @importFrom utils packageVersion
#' @examples
#' install_arrow()
install_arrow <- function() {
os <- tolower(Sys.info()[["sysname"]])
# c("windows", "darwin", "linux", "sunos") # win/mac/linux/solaris
version <- packageVersion("arrow")
- # From CRAN check:
- from_cran <- identical(packageDescription("arrow")$Repository, "CRAN")
- # Is it possible to tell if was a binary install from CRAN vs. source?
-
- message(install_arrow_msg(arrow_available(), version, from_cran, os))
+ message(install_arrow_msg(arrow_available(), version, os))
}
-install_arrow_msg <- function(has_arrow, version, from_cran, os) {
+install_arrow_msg <- function(has_arrow, version, os) {
# TODO: check if there is a newer version on CRAN?
# install_arrow() sends "version" as a "package_version" class, but for
@@ -59,20 +55,10 @@ install_arrow_msg <- function(has_arrow, version, from_cran, os) {
# Suggest arrow.apache.org/install, or compilation instructions
msg <- c(paste(SEE_ARROW_INSTALL, OR_SEE_DEV_GUIDE), THEN_REINSTALL)
}
- } else if (!dev_version && !from_cran) {
- # Windows or Mac with a released version but not from CRAN
- # Recommend installing released binary package from CRAN
- msg <- INSTALL_FROM_CRAN
} else {
- # Windows or Mac, most likely a dev version
- # for each OS, recommend dev installation, refer to readme
- # TODO: if there is a newer version on CRAN, recommend CRAN
- if (os == "windows") {
- msg <- c(paste(FIND_WIN_BINARY, OR_SEE_DEV_GUIDE), THEN_REINSTALL)
- } else {
- # macOS
- msg <- c(paste(FIND_MAC_BINARY, OR_SEE_DEV_GUIDE), THEN_REINSTALL)
- }
+ # We no longer allow builds without libarrow on macOS or Windows so this
+ # case shouldn't happen
+ msg <- ""
}
# Common postscript
msg <- c(msg, SEE_README, REPORT_ISSUE)
@@ -120,22 +106,3 @@ REPORT_ISSUE <- paste(
"please report an issue here:",
"<https://issues.apache.org/jira/projects/ARROW/issues>"
)
-
-INSTALL_FROM_CRAN <- paste(
- 'Try installing the package from CRAN:',
- '`install.packages("arrow")`'
-)
-
-FIND_WIN_BINARY <- paste(
- "You may be able to download a development version of the C++ binary",
- "from the Apache Arrow project's Appveyor:",
- "<https://ci.appveyor.com/project/ApacheSoftwareFoundation/arrow>.",
- "Select an R job from a recent build,",
- 'and download the `build\arrow-*.zip` file from the "Artifacts" tab.',
- "Then, set the RWINLIB_LOCAL environment variable to point to that file."
-)
-
-FIND_MAC_BINARY <- paste(
- "You may be able to get a development version of the Arrow C++ library",
- "using Homebrew: `brew install apache-arrow --HEAD`"
-)
diff --git a/r/README.md b/r/README.md
index 0c13e0b..f487fc7 100644
--- a/r/README.md
+++ b/r/README.md
@@ -66,7 +66,7 @@ set.seed(24)
tab <- arrow::table(x = 1:10, y = rnorm(10))
tab$schema
-#> arrow::Schema
+#> arrow::Schema
#> x: int32
#> y: double
tab
diff --git a/r/configure b/r/configure
index 12d8b0b..f06078e 100755
--- a/r/configure
+++ b/r/configure
@@ -58,25 +58,22 @@ else
PKG_LIBS=${PKGCONFIG_LIBS}
else
if [[ "$OSTYPE" == "darwin"* ]]; then
- if [ "$(command -v brew)" ]; then
- if [ "$(brew ls --versions ${PKG_BREW_NAME})" == "" ]; then
- echo "Installing ${PKG_BREW_NAME} via Homebrew"
- brew install ${PKG_BREW_NAME}
- else
- # We already have it so no need to install
- echo "Using Homebrew ${PKG_BREW_NAME}"
- fi
-
+ if [ "$(command -v brew)" ] && [ "$(brew ls --versions ${PKG_BREW_NAME})" != "" ]; then
+ echo "Using Homebrew ${PKG_BREW_NAME}"
BREWDIR=$(brew --prefix)
- PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
- PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS"
else
- echo "Installing ${PKG_BREW_NAME} via Autobrew"
+ echo "Downloading ${PKG_BREW_NAME}"
curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew
+ if [ $? -ne 0 ]; then
+ echo "Failed to download manifest for ${PKG_BREW_NAME}"
+ fi
source autobrew
- PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
- PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS"
+ if [ $? -ne 0 ]; then
+ echo "Failed to retrieve binary for ${PKG_BREW_NAME}"
+ fi
fi
+ PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
+ PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS"
fi
fi
fi
@@ -93,20 +90,26 @@ if [ "$ARROW_USE_OLD_CXXABI" ]; then
fi
# Test configuration
-echo "#include $PKG_TEST_HEADER" | ${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FLAGS} ${CXX11STD} -xc++ - >/dev/null 2>&1
+TEST_CMD="${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FLAGS} ${CXX11STD} -xc++ -"
+echo "#include $PKG_TEST_HEADER" | ${TEST_CMD} >/dev/null 2>&1
-# Customize the error
-if [ $? -ne 0 ]; then
+if [ $? -eq 0 ] || [[ "$OSTYPE" == "darwin"* ]]; then
+ # Always build with arrow on macOS
+ PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_ARROW"
+ echo "PKG_CFLAGS=$PKG_CFLAGS"
+ echo "PKG_LIBS=$PKG_LIBS"
+else
+ if [[ "$OSTYPE" == "darwin"* ]]; then
+ # Just for debugging: is this possible?
+ echo "Test to load header failed. Command:"
+ echo "$TEST_CMD"
+ fi
echo "------------------------- NOTE ---------------------------"
echo "After installation, please run arrow::install_arrow()"
echo "for help installing required runtime libraries"
echo "---------------------------------------------------------"
PKG_LIBS=""
PKG_CFLAGS=""
-else
- PKG_CFLAGS="$PKG_CFLAGS -DARROW_R_WITH_ARROW"
- echo "PKG_CFLAGS=$PKG_CFLAGS"
- echo "PKG_LIBS=$PKG_LIBS"
fi
# Write to Makevars
diff --git a/r/configure.win b/r/configure.win
index 5ad2a95..b7bb0ff 100644
--- a/r/configure.win
+++ b/r/configure.win
@@ -30,18 +30,18 @@ VERSION=$(grep ^Version DESCRIPTION | sed s/Version:\ //)
"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" "tools/winlibs.R" $VERSION $RWINLIB_LOCAL
# If binary not found, script exits nonzero
if [ $? -ne 0 ]; then
- # We'll have to install without libarrow. Note to the user how to rectify.
- echo "------------------------- NOTE ---------------------------"
- echo "After installation, please run arrow::install_arrow() to install"
- echo "required runtime libraries"
- echo "---------------------------------------------------------"
-else
- # Set the right flags to point to and enable arrow/parquet
- RWINLIB="../windows/arrow-${VERSION}"
- PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_ARROW"
- PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
+ echo "Arrow C++ library was not found"
fi
+# Set the right flags to point to and enable arrow/parquet
+# It's possible that the version of the libarrow binary is not identical to the
+# R version, e.g. if the R build is a patch release, so find what the dir is
+# actually called:
+RWINLIB="../windows/$(ls windows/ | grep ^arrow-)"
+
+PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_ARROW"
+PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
+
echo "*** Writing Makevars.win"
sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars.win
# Success
diff --git a/r/tests/testthat/test-install-arrow.R b/r/tests/testthat/test-install-arrow.R
index 0e149d5..b529b01 100644
--- a/r/tests/testthat/test-install-arrow.R
+++ b/r/tests/testthat/test-install-arrow.R
@@ -55,28 +55,4 @@ r_only({
fixed = TRUE
)
})
-
- test_that("Win/mac release version get pointed to CRAN", {
- expect_match(
- install_arrow_msg(FALSE, "0.13.0", os="darwin", from_cran=FALSE),
- "install.packages",
- fixed = TRUE
- )
- expect_match(
- install_arrow_msg(FALSE, "0.13.0", os="windows", from_cran=FALSE),
- "install.packages",
- fixed = TRUE
- )
- })
-
- test_that("Win/mac dev version get recommendations", {
- expect_match(
- install_arrow_msg(FALSE, "0.13.0.9000", os="darwin", from_cran=FALSE),
- "Homebrew"
- )
- expect_match(
- install_arrow_msg(FALSE, "0.13.0.9000", os="windows", from_cran=FALSE),
- "RWINLIB_LOCAL"
- )
- })
})
diff --git a/r/tools/winlibs.R b/r/tools/winlibs.R
index b54e63c..72d817f 100644
--- a/r/tools/winlibs.R
+++ b/r/tools/winlibs.R
@@ -29,7 +29,19 @@ if(!file.exists(sprintf("windows/arrow-%s/include/arrow/api.h", VERSION))){
} else {
# Download static arrow from rwinlib
if(getRversion() < "3.3.0") setInternet2()
- download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE)
+ try(download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE), silent = TRUE)
+ if(!file.exists("lib.zip")){
+ # Try a different version
+ # First, try pruning off a dev number, i.e. go from 0.14.1.1 to 0.14.1
+ VERSION <- sub("^([0-9]+\\.[0-9]+\\.[0-9]+).*$", "\\1", VERSION)
+ try(download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE), silent = TRUE)
+ }
+ if(!file.exists("lib.zip")){
+ # Next, try without a patch release, i.e. go from 0.14.1 to 0.14.0
+ VERSION <- sub("^([0-9]+\\.[0-9]+\\.).*$", "\\10", VERSION)
+ cat(sprintf("Downloading https://github.com/rwinlib/arrow/archive/v%s.zip\n", VERSION))
+ download.file(sprintf("https://github.com/rwinlib/arrow/archive/v%s.zip", VERSION), "lib.zip", quiet = TRUE)
+ }
}
dir.create("windows", showWarnings = FALSE)
unzip("lib.zip", exdir = "windows")