You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ro...@apache.org on 2019/04/11 10:20:37 UTC
[arrow] branch master updated: ARROW-4911: [R] Progress towards
completing windows support
This is an automated email from the ASF dual-hosted git repository.
romainfrancois pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 76e1bc5 ARROW-4911: [R] Progress towards completing windows support
76e1bc5 is described below
commit 76e1bc5dfb9d08e31eddd5cbcc0b1bab934da2c7
Author: Romain Francois <ro...@rstudio.com>
AuthorDate: Thu Apr 11 12:20:18 2019 +0200
ARROW-4911: [R] Progress towards completing windows support
Support for building [R] package in Windows x64/x86.
Fixes https://issues.apache.org/jira/browse/ARROW-4911 which helps us work towards releasing in CRAN: https://issues.apache.org/jira/browse/ARROW-3204.
Author: Romain Francois <ro...@rstudio.com>
Author: Javier Luraschi <ja...@hotmail.com>
Author: Romain Francois <ro...@MacBook-Pro.localdomain>
Author: Romain François <ro...@purrple.cat>
Closes #3932 from javierluraschi/bugfix/port-win-fixes and squashes the following commits:
5806d5eb <Romain Francois> cran comments
a44d1f55 <Romain Francois> using configure and Makevars.* from @jeroen
30ab2798 <Romain Francois> not yet supported
797c3b57 <Romain Francois> merge javierluraschi/bugfix/port-win-fixes
21e0d9df <Romain Francois> Revert "enable parquet on mac. from https://github.com/jeroen/arrow/commit/833a2dc7554b6dcee364ff39206ed9a7d9fe59e0"
06858d0c <Romain Francois> revert configure and Makevars.in changes so that it works fine on mac os
892af4d4 <Romain Francois> enable parquet on mac. from https://github.com/jeroen/arrow/commit/833a2dc7554b6dcee364ff39206ed9a7d9fe59e0
f008baa9 <Romain François> Merge branch 'master' into bugfix/port-win-fixes
b0fd448e <Romain Francois> Remove incorrect stuff from configure/make setup, from @jeroen. use version 0.13.0 of C++ arrow lib on windows, enable parquet
7927aa0b <Romain Francois> add @jeroen as author and use 0.13.0 as the version to prepare for cran release
1d50c303 <Romain Francois> lint
59025b6f <Romain Francois> use int64_t instead of size_t
e2e451e8 <Romain Francois> usigned comparison
d1ba6dd0 <Romain Francois> initialize value.
169918ff <Romain Francois> split int_cast in versions for signed and unsigned
ee21495d <Romain Francois> using R_xlen_t instead of size_t. causes trouble in win32
cc12f9c1 <Javier Luraschi> Merge branch 'master' into bugfix/port-win-fixes
aa91e2b5 <Javier Luraschi> modivy R's makevars.win file on release script
9dcc347b <Javier Luraschi> fix apache rat warnings
14bce9cb <Javier Luraschi> Progress towards completing windows support
---
dev/release/rat_exclude_files.txt | 1 +
r/.Rbuildignore | 1 +
r/DESCRIPTION | 3 ++-
r/configure | 32 ++++++++------------------------
r/cran-comments.md | 27 +++++++++++++++++++++++++++
r/src/Makevars.in | 5 ++---
r/src/Makevars.win | 21 ++++++---------------
r/src/array.cpp | 2 +-
r/src/array__to_vector.cpp | 12 ++++++------
r/src/array_from_vector.cpp | 22 +++++++++++++++++++---
10 files changed, 73 insertions(+), 53 deletions(-)
diff --git a/dev/release/rat_exclude_files.txt b/dev/release/rat_exclude_files.txt
index ff1d191..46ddff3 100644
--- a/dev/release/rat_exclude_files.txt
+++ b/dev/release/rat_exclude_files.txt
@@ -188,6 +188,7 @@ r/arrow.Rproj
r/README.md
r/README.Rmd
r/man/*.Rd
+r/cran-comments.md
.gitattributes
ruby/red-arrow/.yardopts
rust/arrow/test/data/*.csv
diff --git a/r/.Rbuildignore b/r/.Rbuildignore
index b51d7bc..9d39e74 100644
--- a/r/.Rbuildignore
+++ b/r/.Rbuildignore
@@ -9,3 +9,4 @@ Dockerfile
.*\.tar\.gz
^windows
clang_format.sh
+^cran-comments\.md$
diff --git a/r/DESCRIPTION b/r/DESCRIPTION
index 7264820..e3e170f 100644
--- a/r/DESCRIPTION
+++ b/r/DESCRIPTION
@@ -1,10 +1,11 @@
Package: arrow
Title: R Integration to 'Apache' 'Arrow'
-Version: 0.13.0.9000
+Version: 0.13.0
Authors@R: c(
person("Romain", "François", email = "romain@rstudio.com", role = c("aut", "cre")),
person("Javier", "Luraschi", email = "javier@rstudio.com", role = c("ctb")),
person("Jeffrey", "Wong", email = "jeffreyw@netflix.com", role = c("ctb")),
+ person("Jeroen", "Ooms", email = "jeroen@berkeley.edu", role = c("aut")),
person("Apache Arrow", email = "dev@arrow.apache.org", role = c("aut", "cph"))
)
Description: R Integration to 'Apache' 'Arrow'.
diff --git a/r/configure b/r/configure
index a3bc690..8944270 100755
--- a/r/configure
+++ b/r/configure
@@ -27,29 +27,17 @@
# Library settings
PKG_CONFIG_NAME="arrow parquet"
-PKG_DEB_NAME="arrow"
-PKG_RPM_NAME="arrow"
-PKG_CSW_NAME="arrow"
+PKG_DEB_NAME="(unsuppored)"
+PKG_RPM_NAME="(unsuppored)"
PKG_BREW_NAME="apache-arrow"
PKG_TEST_HEADER="<arrow/api.h>"
-PKG_LIBS=""
+PKG_LIBS="-larrow -lparquet"
# Use pkg-config if available
pkg-config --version >/dev/null 2>&1
if [ $? -eq 0 ]; then
- PKGCONFIG_CFLAGS=$(pkg-config --cflags --silence-errors arrow)
- if [ $? -ne 0 ]; then
- echo "Apache Arrow C++ was not found using pkg-config"
- exit 1
- fi
- PKGCONFIG_LIBS=$(pkg-config --libs arrow)
- PKGCONFIG_CFLAGS_PARQUET=$(pkg-config --cflags --silence-errors parquet)
- if [ $? -eq 0 ]; then
- PKGCONFIG_CFLAGS="${PKGCONFIG_CFLAGS} ${PKGCONFIG_CFLAGS_PARQUET} -DARROW_R_WITH_PARQUET"
- PKGCONFIG_LIBS="${PKGCONFIG_LIBS} $(pkg-config --libs parquet)"
- fi
-else
- PKG_LIBS="-larrow -lparquet"
+ PKGCONFIG_CFLAGS=`pkg-config --cflags --silence-errors ${PKG_CONFIG_NAME}`
+ PKGCONFIG_LIBS=`pkg-config --libs ${PKG_CONFIG_NAME}`
fi
# Note that cflags may be empty in case of success
@@ -68,7 +56,7 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then
curl -sfL "https://jeroen.github.io/autobrew/$PKG_BREW_NAME" > autobrew
source autobrew
fi
- PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include"
+ PKG_CFLAGS="-I$BREWDIR/opt/$PKG_BREW_NAME/include -DARROW_R_WITH_PARQUET"
PKG_LIBS="-L$BREWDIR/opt/$PKG_BREW_NAME/lib $PKG_LIBS"
fi
@@ -83,9 +71,8 @@ CXX11STD=$("${R_HOME}"/bin/R CMD config CXX11STD)
CPPFLAGS=$("${R_HOME}"/bin/R CMD config CPPFLAGS)
# If libarrow uses the old GLIBCXX ABI, so we have to use it too
-PKG_CXXFLAGS="$C_VISIBILITY"
if [ "$ARROW_USE_OLD_CXXABI" ]; then
- PKG_CXXFLAGS="$PKG_CXXFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
+ $PKG_CFLAGS="$PKG_CFLAGS -D_GLIBCXX_USE_CXX11_ABI=0"
fi
# Test configuration
@@ -95,9 +82,6 @@ echo "#include $PKG_TEST_HEADER" | ${CXXCPP} ${CPPFLAGS} ${PKG_CFLAGS} ${CXX11FL
if [ $? -ne 0 ]; then
echo "------------------------- ANTICONF ERROR ---------------------------"
echo "Configuration failed because $PKG_CONFIG_NAME was not found. Try installing:"
- echo " * deb: $PKG_DEB_NAME (Debian, Ubuntu, etc)"
- echo " * rpm: $PKG_RPM_NAME (Fedora, CentOS, RHEL)"
- echo " * csw: $PKG_CSW_NAME (Solaris)"
echo " * brew: $PKG_BREW_NAME (Mac OSX)"
echo "If $PKG_CONFIG_NAME is already installed, check that 'pkg-config' is in your"
echo "PATH and PKG_CONFIG_PATH contains a $PKG_CONFIG_NAME.pc file. If pkg-config"
@@ -108,7 +92,7 @@ if [ $? -ne 0 ]; then
fi
# Write to Makevars
-sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" -e "s|@pkgcxxflags@|$PKG_CXXFLAGS|" src/Makevars.in > src/Makevars
+sed -e "s|@cflags@|$PKG_CFLAGS|" -e "s|@libs@|$PKG_LIBS|" src/Makevars.in > src/Makevars
# Success
exit 0
diff --git a/r/cran-comments.md b/r/cran-comments.md
new file mode 100644
index 0000000..c99d06a
--- /dev/null
+++ b/r/cran-comments.md
@@ -0,0 +1,27 @@
+## Test environments
+* local OS X install, R 3.5.3
+* win-builder (devel and release)
+
+## R CMD check results
+
+0 errors | 0 warnings | 1 note
+
+* This is a new release.
+
+## Platform support
+
+This package supports Windows and macOS but not Linux.
+
+The Arrow project is cross-language development platform
+for in-memory data, it spans several languages and
+their code base is quite large (about 150K lines of C
+sources and more than 600K lines across all languages).
+
+In the future, the Apache Arrow project will release
+binaries in the official Fedora and Debian repos;
+we're working on hard on this, but due to the size,
+this is likely to be implemented until next year.
+
+In the meantime, R users can install the Linux binaries
+from custom repos or build Arrow from source when using
+Linux.
diff --git a/r/src/Makevars.in b/r/src/Makevars.in
index 0d28087..cb8576a 100644
--- a/r/src/Makevars.in
+++ b/r/src/Makevars.in
@@ -16,7 +16,6 @@
# under the License.
PKG_CPPFLAGS=@cflags@
-PKG_CXXFLAGS=@pkgcxxflags@
+PKG_CXXFLAGS=$(CXX_VISIBILITY)
CXX_STD=CXX11
-PKG_LIBS=@libs@ -Wl,-rpath,/usr/local/lib
-#CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0"
+PKG_LIBS=@libs@
diff --git a/r/src/Makevars.win b/r/src/Makevars.win
index bd5f00b..840dd1f 100644
--- a/r/src/Makevars.win
+++ b/r/src/Makevars.win
@@ -15,24 +15,15 @@
# specific language governing permissions and limitations
# under the License.
-VERSION = 0.13.0.9000
-
-ifeq "$(ARROW_PATH)" ""
- RWINLIB = ../windows/arrow-$(VERSION)
- ARROW_INCLUDE = $(RWINLIB)/include
- ARROW_LIBS = $(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH)
- OTHER_LIBS = -L$(RWINLIB)/lib$(R_ARCH)
-else
- ARROW_INCLUDE = $(ARROW_PATH)/include
- ARROW_LIBS = $(ARROW_PATH)/lib
-endif
-
-PKG_CPPFLAGS = -I$(ARROW_INCLUDE) -DARROW_STATIC -DPARQUET_STATIC
+VERSION = 0.13.0
+RWINLIB = ../windows/arrow-$(VERSION)
+PKG_CPPFLAGS = -I$(RWINLIB)/include -DARROW_STATIC -DPARQUET_STATIC \
+ -DARROW_R_WITH_PARQUET
CXX_STD = CXX11
PKG_LIBS = \
- -L$(ARROW_LIBS) \
- $(OTHER_LIBS) \
+ -L$(RWINLIB)/lib$(subst gcc,,$(COMPILED_BY))$(R_ARCH) \
+ -L$(RWINLIB)/lib$(R_ARCH) \
-lparquet -larrow -lthrift -lboost_regex-mt-s -ldouble-conversion -lz -lws2_32
#all: clean
diff --git a/r/src/array.cpp b/r/src/array.cpp
index 9ef8f5f..e4ad68d 100644
--- a/r/src/array.cpp
+++ b/r/src/array.cpp
@@ -99,7 +99,7 @@ LogicalVector Array__Mask(const std::shared_ptr<arrow::Array>& array) {
LogicalVector res(no_init(n));
arrow::internal::BitmapReader bitmap_reader(array->null_bitmap()->data(),
array->offset(), n);
- for (size_t i = 0; i < array->length(); i++, bitmap_reader.Next()) {
+ for (int64_t i = 0; i < n; i++, bitmap_reader.Next()) {
res[i] = bitmap_reader.IsSet();
}
return res;
diff --git a/r/src/array__to_vector.cpp b/r/src/array__to_vector.cpp
index f4017c0..219b19d 100644
--- a/r/src/array__to_vector.cpp
+++ b/r/src/array__to_vector.cpp
@@ -113,7 +113,7 @@ Status SomeNull_Ingest(SEXP data, R_xlen_t start, R_xlen_t n,
if (array->null_count()) {
arrow::internal::BitmapReader bitmap_reader(array->null_bitmap()->data(),
array->offset(), n);
- for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data, ++p_values) {
+ for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data, ++p_values) {
*p_data = bitmap_reader.IsSet() ? lambda(*p_values) : default_value<RTYPE>();
}
} else {
@@ -251,11 +251,11 @@ class Converter_Boolean : public Converter {
arrow::internal::BitmapReader null_reader(array->null_bitmap()->data(),
array->offset(), n);
- for (size_t i = 0; i < n; i++, data_reader.Next(), null_reader.Next(), ++p_data) {
+ for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), null_reader.Next(), ++p_data) {
*p_data = null_reader.IsSet() ? data_reader.IsSet() : NA_LOGICAL;
}
} else {
- for (size_t i = 0; i < n; i++, data_reader.Next(), ++p_data) {
+ for (R_xlen_t i = 0; i < n; i++, data_reader.Next(), ++p_data) {
*p_data = data_reader.IsSet();
}
}
@@ -472,12 +472,12 @@ class Converter_Decimal : public Converter {
internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(),
n);
- for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) {
+ for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) {
*p_data = bitmap_reader.IsSet() ? std::stod(decimals_arr.FormatValue(i).c_str())
: NA_REAL;
}
} else {
- for (size_t i = 0; i < n; i++, ++p_data) {
+ for (R_xlen_t i = 0; i < n; i++, ++p_data) {
*p_data = std::stod(decimals_arr.FormatValue(i).c_str());
}
}
@@ -514,7 +514,7 @@ class Converter_Int64 : public Converter {
if (array->null_count()) {
internal::BitmapReader bitmap_reader(array->null_bitmap()->data(), array->offset(),
n);
- for (size_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) {
+ for (R_xlen_t i = 0; i < n; i++, bitmap_reader.Next(), ++p_data) {
*p_data = bitmap_reader.IsSet() ? p_values[i] : NA_INT64;
}
} else {
diff --git a/r/src/array_from_vector.cpp b/r/src/array_from_vector.cpp
index 83d9f78..ac72a4a 100644
--- a/r/src/array_from_vector.cpp
+++ b/r/src/array_from_vector.cpp
@@ -182,7 +182,8 @@ using internal::checked_cast;
namespace internal {
-template <typename T, typename Target>
+template <typename T, typename Target,
+ typename std::enable_if<std::is_signed<Target>::value, Target>::type = 0>
Status int_cast(T x, Target* out) {
if (x < std::numeric_limits<Target>::min() || x > std::numeric_limits<Target>::max()) {
return Status::Invalid("Value is too large to fit in C integer type");
@@ -191,6 +192,21 @@ Status int_cast(T x, Target* out) {
return Status::OK();
}
+template <typename T>
+struct usigned_type;
+
+template <typename T, typename Target,
+ typename std::enable_if<std::is_unsigned<Target>::value, Target>::type = 0>
+Status int_cast(T x, Target* out) {
+ // we need to compare between unsigned integers
+ uint64_t x64 = x;
+ if (x64 < 0 || x64 > std::numeric_limits<Target>::max()) {
+ return Status::Invalid("Value is too large to fit in C integer type");
+ }
+ *out = static_cast<Target>(x);
+ return Status::OK();
+}
+
template <typename Int>
Status double_cast(Int x, double* out) {
*out = static_cast<double>(x);
@@ -297,7 +313,7 @@ struct Unbox<Type, enable_if_integer<Type>> {
if (*p == na) {
builder->UnsafeAppendNull();
} else {
- CType value;
+ CType value = 0;
RETURN_NOT_OK(internal::int_cast(*p, &value));
builder->UnsafeAppend(value);
}
@@ -375,7 +391,7 @@ struct Unbox<FloatType> {
if (*p == NA_INTEGER) {
builder->UnsafeAppendNull();
} else {
- float value;
+ float value = 0;
RETURN_NOT_OK(internal::float_cast(*p, &value));
builder->UnsafeAppend(value);
}