You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2019/06/25 01:15:27 UTC
[arrow] branch master updated: ARROW-5683: [R] Add snappy to Rtools
Windows builds
This is an automated email from the ASF dual-hosted git repository.
wesm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new a91f78f ARROW-5683: [R] Add snappy to Rtools Windows builds
a91f78f is described below
commit a91f78fa52dc8b9c059bb08ea6972c746977b968
Author: Neal Richardson <ne...@gmail.com>
AuthorDate: Mon Jun 24 20:15:19 2019 -0500
ARROW-5683: [R] Add snappy to Rtools Windows builds
Following https://github.com/r-windows/rtools-backports/pull/7 and https://github.com/r-windows/rtools-packages/pull/19, R Windows builds can now support Snappy compression. This patch tweaks the necessary files and unskips tests, in addition to some other PKGBUILD script cleanup.
Passing build here: https://ci.appveyor.com/project/nealrichardson/arrow/builds/25507388
Author: Neal Richardson <ne...@gmail.com>
Closes #4681 from nealrichardson/r-snappy and squashes the following commits:
0996a30e2 <Neal Richardson> Add license info for rtools-backports
1af08413c <Neal Richardson> Revert "Only run mine for now"
a5d967f0c <Neal Richardson> Get snappy from backports after all
a1c6390b6 <Neal Richardson> -lsnappy
a41544914 <Neal Richardson> Only run mine for now
b911568df <Neal Richardson> More comments; unskip tests
d7a74192a <Neal Richardson> Add snappy to PKGBUILD; prune some default cmake flags and add comment
---
LICENSE.txt | 18 +++++++++++++++---
ci/PKGBUILD | 14 ++++++++++----
ci/windows-pkg-arrow-for-r.sh | 5 +++++
r/configure.win | 2 +-
r/tests/testthat/test-json.R | 2 --
r/tests/testthat/test-parquet.R | 1 -
6 files changed, 31 insertions(+), 11 deletions(-)
diff --git a/LICENSE.txt b/LICENSE.txt
index 3fe48de..1be16cc 100644
--- a/LICENSE.txt
+++ b/LICENSE.txt
@@ -890,7 +890,7 @@ This project include code from Google's Asylo project.
* cpp/src/arrow/result.h is based on status_or.h
Copyright (c) Copyright 2017 Asylo authors
-Homepage: https://asylo.dev/
+Homepage: https://asylo.dev/
License: Apache 2.0
---------------------------------------------------------------------------------
@@ -900,8 +900,8 @@ This project includes code from Google's protobuf project
* cpp/src/arrow/result.h ARROW_ASSIGN_OR_RAISE is based off ASSIGN_OR_RETURN
Copyright 2008 Google Inc. All rights reserved.
-Homepage: https://developers.google.com/protocol-buffers/
-License:
+Homepage: https://developers.google.com/protocol-buffers/
+License:
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
@@ -933,3 +933,15 @@ Code generated by the Protocol Buffer compiler is owned by the owner
of the input file used when generating it. This code is not
standalone and requires a support library to be linked with it. This
support library is itself covered by the above license.
+
+--------------------------------------------------------------------------------
+
+This project includes code from the rtools-backports project.
+
+* ci/PKGBUILD and ci/appveyor-build-r.sh are based on code
+ from the rtools-backports project.
+
+Copyright: Copyright (c) 2013 - 2019, Алексей and Jeroen Ooms.
+All rights reserved.
+Homepage: https://github.com/r-windows/rtools-backports
+License: 3-clause BSD
diff --git a/ci/PKGBUILD b/ci/PKGBUILD
index b760838..1d20bb7 100644
--- a/ci/PKGBUILD
+++ b/ci/PKGBUILD
@@ -27,11 +27,18 @@ license=("Apache-2.0")
depends=("${MINGW_PACKAGE_PREFIX}-boost"
"${MINGW_PACKAGE_PREFIX}-double-conversion"
"${MINGW_PACKAGE_PREFIX}-thrift"
+ "${MINGW_PACKAGE_PREFIX}-snappy"
"${MINGW_PACKAGE_PREFIX}-zlib")
makedepends=("${MINGW_PACKAGE_PREFIX}-cmake"
"${MINGW_PACKAGE_PREFIX}-gcc")
options=("staticlibs" "strip" "!buildflags")
source_dir=apache-${_realname}-${pkgver}
+
+# For released version:
+#source=("https://archive.apache.org/dist/arrow/arrow-${pkgver}/apache-arrow-${pkgver}.tar.gz")
+#sha256sums=("ac2a77dd9168e9892e432c474611e86ded0be6dfe15f689c948751d37f81391a")
+# For github dev version:
+# Append `#commit=54b1b2f688e5e84b4c664b1e12a95f93b94ab2f3` to the URL to select a revision
source=("${source_dir}"::"git+https://github.com/apache/arrow")
sha256sums=("SKIP")
@@ -58,6 +65,8 @@ build() {
# Workaround to fix static libparquet
export CXXFLAGS="-DARROW_STATIC"
+ # This is the difference between rtools-packages and rtools-backports
+ # Remove this when submitting to rtools-packages
export CC="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/gcc"
export CXX="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin/g++"
export PATH="/C/Rtools${MINGW_PREFIX/mingw/mingw_}/bin:$PATH"
@@ -72,13 +81,9 @@ build() {
-DCMAKE_BUILD_TYPE=${cmake_build_type} \
-DARROW_BUILD_STATIC=ON \
-DARROW_BUILD_SHARED=OFF \
- -DARROW_BUILD_TESTS=OFF \
-DARROW_PARQUET=ON \
- -DARROW_PLASMA=OFF \
-DARROW_HDFS=OFF \
- -DARROW_PYTHON=OFF \
-DARROW_BOOST_USE_SHARED=OFF \
- -DARROW_WITH_SNAPPY=OFF \
-DARROW_WITH_ZSTD=OFF \
-DARROW_WITH_LZ4=OFF \
-DARROW_JEMALLOC=OFF \
@@ -90,6 +95,7 @@ build() {
-Ddouble-conversion_ROOT="${MINGW_PREFIX}" \
-DThrift_ROOT="${MINGW_PREFIX}"
+ # Workaround for a compile error that appears with GCC 4.9.3
sed -i 's/-fPIC/ /g' flatbuffers_ep-prefix/src/flatbuffers_ep-stamp/flatbuffers_ep-configure-RELEASE.cmake
make
diff --git a/ci/windows-pkg-arrow-for-r.sh b/ci/windows-pkg-arrow-for-r.sh
index fdcf712..c70eba6 100644
--- a/ci/windows-pkg-arrow-for-r.sh
+++ b/ci/windows-pkg-arrow-for-r.sh
@@ -46,6 +46,11 @@ wget https://dl.bintray.com/rtools/backports/mingw-w64-x86_64-thrift-0.12.0-8000
# wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-thrift-0.12.0-1-any.pkg.tar.xz
# wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-thrift-0.12.0-1-any.pkg.tar.xz
+wget https://dl.bintray.com/rtools/backports/mingw-w64-i686-snappy-1.1.7-2-any.pkg.tar.xz
+wget https://dl.bintray.com/rtools/backports/mingw-w64-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
+# wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-snappy-1.1.7-2-any.pkg.tar.xz
+# wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-snappy-1.1.7-2-any.pkg.tar.xz
+
# double-conversion is only available in the Rtools4.0 builds, but apparently that's ok
wget https://dl.bintray.com/rtools/mingw64/mingw-w64-x86_64-double-conversion-3.1.2-1-any.pkg.tar.xz
wget https://dl.bintray.com/rtools/mingw32/mingw-w64-i686-double-conversion-3.1.2-1-any.pkg.tar.xz
diff --git a/r/configure.win b/r/configure.win
index b7b08a4..f3d7aad 100644
--- a/r/configure.win
+++ b/r/configure.win
@@ -39,7 +39,7 @@ else
# Set the right flags to point to and enable arrow/parquet
RWINLIB="../windows/arrow-${VERSION}"
PKG_CFLAGS="-I${RWINLIB}/include -DARROW_STATIC -DPARQUET_STATIC -DARROW_R_WITH_PARQUET -DARROW_R_WITH_ARROW"
- PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
+ PKG_LIBS="-L${RWINLIB}/lib"'$(subst gcc,,$(COMPILED_BY))$(R_ARCH) '"-L${RWINLIB}/lib"'$(R_ARCH) '"-lparquet -larrow -lthrift -lsnappy -lboost_regex-mt-s -lboost_filesystem-mt-s -lboost_system-mt-s -ldouble-conversion -lz -lws2_32"
fi
echo "*** Writing Makevars.win"
diff --git a/r/tests/testthat/test-json.R b/r/tests/testthat/test-json.R
index 26d79ca..1521627 100644
--- a/r/tests/testthat/test-json.R
+++ b/r/tests/testthat/test-json.R
@@ -41,7 +41,6 @@ test_that("Can read json file with scalars columns (ARROW-5503)", {
tib <- as.data.frame(tab1)
expect_equal(tib$hello, c(3.5, 3.25, 3.125, 0))
expect_equal(tib$world, c(FALSE, NA, NA, TRUE))
- skip_on_os("windows") # TODO: debug UTF-8 test
expect_equal(tib$yo, c("thing", NA, "\u5fcd", NA))
})
@@ -68,7 +67,6 @@ test_that("read_json_arrow() converts to tibble", {
expect_equal(tab1$hello, c(3.5, 3.25, 3.125, 0))
expect_equal(tab1$world, c(FALSE, NA, NA, TRUE))
- skip_on_os("windows") # TODO: debug UTF-8 test
expect_equal(tab1$yo, c("thing", NA, "\u5fcd", NA))
})
diff --git a/r/tests/testthat/test-parquet.R b/r/tests/testthat/test-parquet.R
index 64b2109..554744e 100644
--- a/r/tests/testthat/test-parquet.R
+++ b/r/tests/testthat/test-parquet.R
@@ -20,7 +20,6 @@ context("Parquet file reading/writing")
pq_file <- system.file("v0.7.1.parquet", package="arrow")
test_that("reading a known Parquet file to tibble", {
- skip_on_os("windows") # TODO: enable snappy in windows build
df <- read_parquet(pq_file)
expect_true(tibble::is_tibble(df))
expect_identical(dim(df), c(10L, 11L))