You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by jo...@apache.org on 2021/04/18 14:40:12 UTC
[arrow-rs] 02/14: Removed R language.
This is an automated email from the ASF dual-hosted git repository.
jorgecarleitao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow-rs.git
commit d131362e1b4ada7bf777c44a818a4bef140161a2
Author: Jorge C. Leitao <jo...@gmail.com>
AuthorDate: Sun Apr 18 14:19:30 2021 +0000
Removed R language.
---
r/.Rbuildignore | 26 -
r/.gitignore | 20 -
r/DESCRIPTION | 102 -
r/Makefile | 53 -
r/NAMESPACE | 346 -
r/NEWS.md | 358 -
r/R/array-data.R | 53 -
r/R/array.R | 301 -
r/R/arrow-datum.R | 165 -
r/R/arrow-package.R | 278 -
r/R/arrow-tabular.R | 220 -
r/R/arrowExports.R | 1668 -----
r/R/buffer.R | 72 -
r/R/chunked-array.R | 132 -
r/R/compression.R | 121 -
r/R/compute.R | 257 -
r/R/config.R | 30 -
r/R/csv.R | 587 --
r/R/dataset-factory.R | 169 -
r/R/dataset-format.R | 320 -
r/R/dataset-partition.R | 125 -
r/R/dataset-scan.R | 202 -
r/R/dataset-write.R | 99 -
r/R/dataset.R | 320 -
r/R/deprecated.R | 40 -
r/R/dictionary.R | 69 -
r/R/dplyr.R | 1101 ---
r/R/enums.R | 142 -
r/R/expression.R | 346 -
r/R/feather.R | 221 -
r/R/field.R | 82 -
r/R/filesystem.R | 510 --
r/R/flight.R | 121 -
r/R/install-arrow.R | 139 -
r/R/io.R | 290 -
r/R/ipc_stream.R | 113 -
r/R/json.R | 104 -
r/R/memory-pool.R | 61 -
r/R/message.R | 95 -
r/R/metadata.R | 132 -
r/R/parquet.R | 575 --
r/R/python.R | 159 -
r/R/record-batch-reader.R | 155 -
r/R/record-batch-writer.R | 196 -
r/R/record-batch.R | 197 -
r/R/reexports-bit64.R | 22 -
r/R/reexports-tidyselect.R | 43 -
r/R/scalar.R | 79 -
r/R/schema.R | 302 -
r/R/table.R | 179 -
r/R/type.R | 484 --
r/R/util.R | 112 -
r/README.md | 306 -
r/_pkgdown.yml | 177 -
r/arrow.Rproj | 21 -
r/cleanup | 21 -
r/configure | 277 -
r/configure.win | 73 -
r/cran-comments.md | 10 -
r/data-raw/codegen.R | 254 -
r/extra-tests/helpers.R | 36 -
r/extra-tests/test-read-files.R | 197 -
r/extra-tests/write-files.R | 42 -
r/inst/NOTICE.txt | 84 -
r/inst/build_arrow_static.sh | 79 -
r/inst/demo_flight_server.py | 120 -
r/inst/v0.7.1.parquet | Bin 4372 -> 0 bytes
r/lint.sh | 41 -
r/man/ArrayData.Rd | 27 -
r/man/ChunkedArray.Rd | 58 -
r/man/Codec.Rd | 24 -
r/man/CsvReadOptions.Rd | 100 -
r/man/CsvTableReader.Rd | 32 -
r/man/DataType.Rd | 15 -
r/man/Dataset.Rd | 81 -
r/man/DictionaryType.Rd | 15 -
r/man/Expression.Rd | 18 -
r/man/FeatherReader.Rd | 33 -
r/man/Field.Rd | 35 -
r/man/FileFormat.Rd | 56 -
r/man/FileInfo.Rd | 28 -
r/man/FileSelector.Rd | 27 -
r/man/FileSystem.Rd | 99 -
r/man/FileWriteOptions.Rd | 8 -
r/man/FixedWidthType.Rd | 15 -
r/man/FragmentScanOptions.Rd | 29 -
r/man/InputStream.Rd | 45 -
r/man/MemoryPool.Rd | 24 -
r/man/Message.Rd | 15 -
r/man/MessageReader.Rd | 15 -
r/man/OutputStream.Rd | 38 -
r/man/ParquetArrowReaderProperties.Rd | 29 -
r/man/ParquetFileReader.Rd | 58 -
r/man/ParquetFileWriter.Rd | 31 -
r/man/ParquetWriterProperties.Rd | 49 -
r/man/Partitioning.Rd | 51 -
r/man/RecordBatch.Rd | 92 -
r/man/RecordBatchReader.Rd | 86 -
r/man/RecordBatchWriter.Rd | 89 -
r/man/Scalar.Rd | 9 -
r/man/Scanner.Rd | 47 -
r/man/Schema.Rd | 85 -
r/man/Table.Rd | 92 -
r/man/array.Rd | 84 -
r/man/arrow-package.Rd | 41 -
r/man/arrow_available.Rd | 42 -
r/man/arrow_info.Rd | 17 -
r/man/buffer.Rd | 35 -
r/man/call_function.Rd | 46 -
r/man/cast_options.Rd | 22 -
r/man/codec_is_available.Rd | 20 -
r/man/compression.Rd | 31 -
r/man/copy_files.Rd | 35 -
r/man/cpu_count.Rd | 17 -
r/man/data-type.Rd | 163 -
r/man/dataset_factory.Rd | 76 -
r/man/default_memory_pool.Rd | 15 -
r/man/dictionary.Rd | 24 -
r/man/enums.Rd | 73 -
r/man/flight_connect.Rd | 21 -
r/man/flight_get.Rd | 19 -
r/man/flight_put.Rd | 25 -
r/man/hive_partition.Rd | 32 -
r/man/install_arrow.Rd | 61 -
r/man/install_pyarrow.Rd | 22 -
r/man/list_compute_functions.Rd | 39 -
r/man/list_flights.Rd | 23 -
r/man/load_flight_server.Rd | 17 -
r/man/make_readable_file.Rd | 29 -
r/man/map_batches.Rd | 30 -
r/man/match_arrow.Rd | 28 -
r/man/mmap_create.Rd | 19 -
r/man/mmap_open.Rd | 16 -
r/man/open_dataset.Rd | 79 -
r/man/read_delim_arrow.Rd | 218 -
r/man/read_feather.Rd | 50 -
r/man/read_ipc_stream.Rd | 42 -
r/man/read_json_arrow.Rd | 52 -
r/man/read_message.Rd | 14 -
r/man/read_parquet.Rd | 50 -
r/man/read_schema.Rd | 19 -
r/man/reexports.Rd | 28 -
r/man/s3_bucket.Rd | 28 -
r/man/type.Rd | 17 -
r/man/unify_schemas.Rd | 26 -
r/man/value_counts.Rd | 18 -
r/man/write_dataset.Rd | 66 -
r/man/write_feather.Rd | 61 -
r/man/write_ipc_stream.Rd | 38 -
r/man/write_parquet.Rd | 108 -
r/man/write_to_raw.Rd | 22 -
r/pkgdown/extra.js | 65 -
r/src/.clang-format | 20 -
r/src/.gitignore | 3 -
r/src/Makevars.in | 29 -
r/src/array.cpp | 286 -
r/src/array_to_vector.cpp | 1330 ----
r/src/arraydata.cpp | 49 -
r/src/arrowExports.cpp | 7032 --------------------
r/src/arrow_cpp11.h | 377 --
r/src/arrow_types.h | 237 -
r/src/arrow_vctrs.h | 22 -
r/src/buffer.cpp | 71 -
r/src/chunkedarray.cpp | 139 -
r/src/compression.cpp | 56 -
r/src/compute.cpp | 301 -
r/src/csv.cpp | 177 -
r/src/dataset.cpp | 486 --
r/src/datatype.cpp | 426 --
r/src/expression.cpp | 71 -
r/src/feather.cpp | 87 -
r/src/field.cpp | 56 -
r/src/filesystem.cpp | 329 -
r/src/imports.cpp | 43 -
r/src/io.cpp | 181 -
r/src/json.cpp | 67 -
r/src/memorypool.cpp | 92 -
r/src/message.cpp | 105 -
r/src/nameof.h | 93 -
r/src/parquet.cpp | 326 -
r/src/py-to-r.cpp | 81 -
r/src/r_to_arrow.cpp | 1046 ---
r/src/recordbatch.cpp | 326 -
r/src/recordbatchreader.cpp | 130 -
r/src/recordbatchwriter.cpp | 67 -
r/src/runtimeinfo.cpp | 30 -
r/src/scalar.cpp | 97 -
r/src/schema.cpp | 146 -
r/src/symbols.cpp | 86 -
r/src/table.cpp | 350 -
r/src/threadpool.cpp | 51 -
r/src/type_infer.cpp | 202 -
r/tests/testthat.R | 27 -
.../data-arrow-extra-meta_3.0.0.parquet | Bin 7862 -> 0 bytes
.../golden-files/data-arrow_0.17.0_lz4.feather | Bin 1650 -> 0 bytes
.../data-arrow_0.17.0_uncompressed.feather | Bin 1354 -> 0 bytes
.../golden-files/data-arrow_0.17.0_zstd.feather | Bin 1626 -> 0 bytes
.../testthat/golden-files/data-arrow_1.0.1.parquet | Bin 3603 -> 0 bytes
.../golden-files/data-arrow_1.0.1_lz4.feather | Bin 2858 -> 0 bytes
.../data-arrow_1.0.1_uncompressed.feather | Bin 2626 -> 0 bytes
.../golden-files/data-arrow_1.0.1_zstd.feather | Bin 2842 -> 0 bytes
.../testthat/golden-files/data-arrow_2.0.0.parquet | Bin 3965 -> 0 bytes
.../golden-files/data-arrow_2.0.0_lz4.feather | Bin 3162 -> 0 bytes
.../data-arrow_2.0.0_uncompressed.feather | Bin 2930 -> 0 bytes
.../golden-files/data-arrow_2.0.0_zstd.feather | Bin 3146 -> 0 bytes
r/tests/testthat/helper-arrow.R | 69 -
r/tests/testthat/helper-data.R | 169 -
r/tests/testthat/helper-expectation.R | 191 -
r/tests/testthat/helper-parquet.R | 29 -
r/tests/testthat/helper-roundtrip.R | 43 -
r/tests/testthat/helper-skip.R | 55 -
r/tests/testthat/latin1.R | 76 -
r/tests/testthat/test-Array.R | 820 ---
r/tests/testthat/test-RecordBatch.R | 501 --
r/tests/testthat/test-Table.R | 477 --
r/tests/testthat/test-array-data.R | 35 -
r/tests/testthat/test-arrow-info.R | 23 -
r/tests/testthat/test-arrow.R | 74 -
r/tests/testthat/test-backwards-compatibility.R | 117 -
r/tests/testthat/test-buffer-reader.R | 40 -
r/tests/testthat/test-buffer.R | 99 -
r/tests/testthat/test-chunked-array.R | 413 --
r/tests/testthat/test-chunked-array.txt | 103 -
r/tests/testthat/test-compressed.R | 75 -
r/tests/testthat/test-compute-aggregate.R | 353 -
r/tests/testthat/test-compute-arith.R | 116 -
r/tests/testthat/test-compute-sort.R | 165 -
r/tests/testthat/test-compute-vector.R | 133 -
r/tests/testthat/test-csv.R | 258 -
r/tests/testthat/test-data-type.R | 413 --
r/tests/testthat/test-dataset.R | 1786 -----
r/tests/testthat/test-dplyr-arrange.R | 211 -
r/tests/testthat/test-dplyr-filter.R | 417 --
r/tests/testthat/test-dplyr-group-by.R | 135 -
r/tests/testthat/test-dplyr-mutate.R | 417 --
r/tests/testthat/test-dplyr-string-functions.R | 347 -
r/tests/testthat/test-dplyr.R | 591 --
r/tests/testthat/test-expression.R | 99 -
r/tests/testthat/test-feather.R | 242 -
r/tests/testthat/test-field.R | 38 -
r/tests/testthat/test-filesystem.R | 175 -
r/tests/testthat/test-install-arrow.R | 39 -
r/tests/testthat/test-json.R | 250 -
r/tests/testthat/test-memory-pool.R | 26 -
r/tests/testthat/test-message-reader.R | 86 -
r/tests/testthat/test-message.R | 64 -
r/tests/testthat/test-metadata.R | 207 -
r/tests/testthat/test-parquet.R | 254 -
r/tests/testthat/test-python-flight.R | 63 -
r/tests/testthat/test-python.R | 102 -
r/tests/testthat/test-read-record-batch.R | 79 -
r/tests/testthat/test-read-write.R | 126 -
r/tests/testthat/test-record-batch-reader.R | 142 -
r/tests/testthat/test-s3-minio.R | 229 -
r/tests/testthat/test-s3.R | 53 -
r/tests/testthat/test-scalar.R | 104 -
r/tests/testthat/test-schema.R | 176 -
r/tests/testthat/test-thread-pool.R | 26 -
r/tests/testthat/test-type.R | 108 -
r/tests/testthat/test-utf.R | 25 -
r/tools/autobrew | 66 -
r/tools/nixlibs.R | 506 --
r/tools/ubsan.supp | 18 -
r/tools/winlibs.R | 65 -
r/vignettes/arrow.Rmd | 206 -
r/vignettes/dataset.Rmd | 397 --
r/vignettes/developing.Rmd | 520 --
r/vignettes/flight.Rmd | 78 -
r/vignettes/fs.Rmd | 130 -
r/vignettes/install.Rmd | 366 -
r/vignettes/python.Rmd | 131 -
271 files changed, 46586 deletions(-)
diff --git a/r/.Rbuildignore b/r/.Rbuildignore
deleted file mode 100644
index cf4b7ce..0000000
--- a/r/.Rbuildignore
+++ /dev/null
@@ -1,26 +0,0 @@
-^.*\.Rproj$
-^\.Rproj\.user$
-^README\.Rmd$
-src/.clang-format
-LICENSE.md
-^data-raw$
-lint.sh
-Dockerfile
-.*\.tar\.gz
-^windows
-^libarrow
-^revdep
-clang_format.sh
-^cran-comments\.md$
-^arrow_.*.tar.gz$
-^arrow_.*.tgz$
-^_pkgdown\.yml$
-^docs$
-^pkgdown$
-^Makefile$
-^.*\.orig$
-^.*\.cmd$
-^autobrew$
-^apache-arrow.rb$
-^.*\.Rhistory$
-^extra-tests
diff --git a/r/.gitignore b/r/.gitignore
deleted file mode 100644
index 76e8a8d..0000000
--- a/r/.gitignore
+++ /dev/null
@@ -1,20 +0,0 @@
-Meta
-docs/
-inst/doc
-*.o
-*.o-*
-*.d
-*.so
-*.dll
-.RData
-.Rproj.user
-.Rhistory
-src/Makevars
-src/Makevars.win
-windows/
-libarrow/
-revdep/
-vignettes/nyc-taxi/
-arrow_*.tar.gz
-arrow_*.tgz
-extra-tests/files
diff --git a/r/DESCRIPTION b/r/DESCRIPTION
deleted file mode 100644
index 7a63b9e..0000000
--- a/r/DESCRIPTION
+++ /dev/null
@@ -1,102 +0,0 @@
-Package: arrow
-Title: Integration to 'Apache' 'Arrow'
-Version: 3.0.0.9000
-Authors@R: c(
- person("Neal", "Richardson", email = "neal@ursalabs.org", role = c("aut", "cre")),
- person("Ian", "Cook", email = "ianmcook@gmail.com", role = c("aut")),
- person("Jonathan", "Keane", email = "jkeane@gmail.com", role = c("aut")),
- person("Romain", "Fran\u00e7ois", email = "romain@rstudio.com", role = c("aut"), comment = c(ORCID = "0000-0002-2444-4226")),
- person("Jeroen", "Ooms", email = "jeroen@berkeley.edu", role = c("aut")),
- person("Javier", "Luraschi", email = "javier@rstudio.com", role = c("ctb")),
- person("Jeffrey", "Wong", email = "jeffreyw@netflix.com", role = c("ctb")),
- person("Apache Arrow", email = "dev@arrow.apache.org", role = c("aut", "cph"))
- )
-Description: 'Apache' 'Arrow' <https://arrow.apache.org/> is a cross-language
- development platform for in-memory data. It specifies a standardized
- language-independent columnar memory format for flat and hierarchical data,
- organized for efficient analytic operations on modern hardware. This
- package provides an interface to the 'Arrow C++' library.
-Depends: R (>= 3.3)
-License: Apache License (>= 2.0)
-URL: https://github.com/apache/arrow/, https://arrow.apache.org/docs/r/
-BugReports: https://issues.apache.org/jira/projects/ARROW/issues
-Encoding: UTF-8
-Language: en-US
-SystemRequirements: C++11; for AWS S3 support on Linux, libcurl and openssl (optional)
-Biarch: true
-Imports:
- assertthat,
- bit64 (>= 0.9-7),
- methods,
- purrr,
- R6,
- rlang,
- stats,
- tidyselect,
- utils,
- vctrs
-Roxygen: list(markdown = TRUE, r6 = FALSE, load = "source")
-RoxygenNote: 7.1.1
-VignetteBuilder: knitr
-Suggests:
- decor,
- distro,
- dplyr,
- hms,
- knitr,
- lubridate,
- pkgload,
- reticulate,
- rmarkdown,
- stringr,
- testthat,
- tibble,
- withr
-LinkingTo: cpp11 (>= 0.2.0)
-Collate:
- 'enums.R'
- 'arrow-package.R'
- 'type.R'
- 'array-data.R'
- 'arrow-datum.R'
- 'array.R'
- 'arrow-tabular.R'
- 'arrowExports.R'
- 'buffer.R'
- 'chunked-array.R'
- 'io.R'
- 'compression.R'
- 'scalar.R'
- 'compute.R'
- 'config.R'
- 'csv.R'
- 'dataset.R'
- 'dataset-factory.R'
- 'dataset-format.R'
- 'dataset-partition.R'
- 'dataset-scan.R'
- 'dataset-write.R'
- 'deprecated.R'
- 'dictionary.R'
- 'record-batch.R'
- 'table.R'
- 'expression.R'
- 'dplyr.R'
- 'feather.R'
- 'field.R'
- 'filesystem.R'
- 'flight.R'
- 'install-arrow.R'
- 'ipc_stream.R'
- 'json.R'
- 'memory-pool.R'
- 'message.R'
- 'metadata.R'
- 'parquet.R'
- 'python.R'
- 'record-batch-reader.R'
- 'record-batch-writer.R'
- 'reexports-bit64.R'
- 'reexports-tidyselect.R'
- 'schema.R'
- 'util.R'
diff --git a/r/Makefile b/r/Makefile
deleted file mode 100644
index efc55ab..0000000
--- a/r/Makefile
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-VERSION=$(shell grep ^Version DESCRIPTION | sed s/Version:\ //)
-ARROW_R_DEV="TRUE"
-ARROW_LARGE_MEMORY_TESTS=$(ARROW_R_DEV)
-
-doc:
- R -s -e 'roxygen2::roxygenize()'
- -git add --all man/*.Rd
-
-test:
- export ARROW_R_DEV=$(ARROW_R_DEV) && R CMD INSTALL --install-tests --no-test-load --no-docs --no-help --no-byte-compile .
- export NOT_CRAN=true && export ARROW_R_DEV=$(ARROW_R_DEV) && export AWS_EC2_METADATA_DISABLED=TRUE && export ARROW_LARGE_MEMORY_TESTS=$(ARROW_LARGE_MEMORY_TESTS) && R -s -e 'library(testthat); setwd(file.path(.libPaths()[1], "arrow", "tests")); system.time(test_check("arrow", filter="${file}", reporter=ifelse(nchar("${r}"), "${r}", "summary")))'
-
-deps:
- R -s -e 'lib <- Sys.getenv("R_LIB", .libPaths()[1]); install.packages("devtools", repo="https://cloud.r-project.org", lib=lib); devtools::install_dev_deps(lib=lib)'
-
-build: doc
- cp ../NOTICE.txt inst/NOTICE.txt
- R CMD build .
-
-check: build
- -export _R_CHECK_CRAN_INCOMING_REMOTE_=FALSE && export ARROW_R_DEV=$(ARROW_R_DEV) && export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz
- rm -rf arrow.Rcheck/
-
-release: build
- -export _R_CHECK_TESTS_NLINES_=0 && R CMD check --as-cran --run-donttest arrow_$(VERSION).tar.gz
- rm -rf arrow.Rcheck/
-
-clean:
- -rm src/*.o
- -rm src/*.so
- -rm src/*.dll
- -rm src/Makevars
- -rm src/Makevars.win
- -rm -rf arrow.Rcheck/
- -rm -rf libarrow/
- -find . -name "*.orig" -delete
diff --git a/r/NAMESPACE b/r/NAMESPACE
deleted file mode 100644
index 5673538..0000000
--- a/r/NAMESPACE
+++ /dev/null
@@ -1,346 +0,0 @@
-# Generated by roxygen2: do not edit by hand
-
-S3method("!=",ArrowObject)
-S3method("$",ArrowTabular)
-S3method("$",Schema)
-S3method("$",StructArray)
-S3method("$",SubTreeFileSystem)
-S3method("$<-",ArrowTabular)
-S3method("$<-",Schema)
-S3method("==",ArrowObject)
-S3method("[",ArrowDatum)
-S3method("[",ArrowTabular)
-S3method("[",Dataset)
-S3method("[",Schema)
-S3method("[",arrow_dplyr_query)
-S3method("[[",ArrowTabular)
-S3method("[[",Schema)
-S3method("[[",StructArray)
-S3method("[[<-",ArrowTabular)
-S3method("[[<-",Schema)
-S3method("names<-",ArrowTabular)
-S3method(Ops,ArrowDatum)
-S3method(Ops,Expression)
-S3method(Ops,array_expression)
-S3method(all,equal.ArrowObject)
-S3method(as.character,ArrowDatum)
-S3method(as.character,FileFormat)
-S3method(as.character,FragmentScanOptions)
-S3method(as.data.frame,ArrowTabular)
-S3method(as.data.frame,StructArray)
-S3method(as.data.frame,arrow_dplyr_query)
-S3method(as.double,ArrowDatum)
-S3method(as.integer,ArrowDatum)
-S3method(as.list,ArrowTabular)
-S3method(as.list,Schema)
-S3method(as.raw,Buffer)
-S3method(as.vector,ArrowDatum)
-S3method(as.vector,array_expression)
-S3method(c,Dataset)
-S3method(dim,ArrowTabular)
-S3method(dim,Dataset)
-S3method(dim,StructArray)
-S3method(dim,arrow_dplyr_query)
-S3method(dimnames,ArrowTabular)
-S3method(head,ArrowDatum)
-S3method(head,ArrowTabular)
-S3method(head,Dataset)
-S3method(head,arrow_dplyr_query)
-S3method(is.na,ArrowDatum)
-S3method(is.na,Expression)
-S3method(is.na,Scalar)
-S3method(is.na,array_expression)
-S3method(is.nan,ArrowDatum)
-S3method(is_in,ArrowDatum)
-S3method(is_in,default)
-S3method(length,ArrowDatum)
-S3method(length,Scalar)
-S3method(length,Schema)
-S3method(match_arrow,ArrowDatum)
-S3method(match_arrow,default)
-S3method(max,ArrowDatum)
-S3method(mean,ArrowDatum)
-S3method(median,ArrowDatum)
-S3method(min,ArrowDatum)
-S3method(names,Dataset)
-S3method(names,FeatherReader)
-S3method(names,RecordBatch)
-S3method(names,Scanner)
-S3method(names,ScannerBuilder)
-S3method(names,Schema)
-S3method(names,StructArray)
-S3method(names,Table)
-S3method(names,arrow_dplyr_query)
-S3method(print,"arrow-enum")
-S3method(print,array_expression)
-S3method(print,arrow_dplyr_query)
-S3method(print,arrow_info)
-S3method(print,arrow_r_metadata)
-S3method(quantile,ArrowDatum)
-S3method(read_message,InputStream)
-S3method(read_message,MessageReader)
-S3method(read_message,default)
-S3method(row.names,ArrowTabular)
-S3method(sort,ArrowDatum)
-S3method(sort,Scalar)
-S3method(sum,ArrowDatum)
-S3method(tail,ArrowDatum)
-S3method(tail,ArrowTabular)
-S3method(tail,Dataset)
-S3method(tail,arrow_dplyr_query)
-S3method(type,ArrowDatum)
-S3method(type,default)
-S3method(unique,ArrowDatum)
-S3method(vec_ptype_abbr,arrow_fixed_size_binary)
-S3method(vec_ptype_abbr,arrow_fixed_size_list)
-S3method(vec_ptype_abbr,arrow_large_list)
-S3method(vec_ptype_abbr,arrow_list)
-S3method(vec_ptype_full,arrow_fixed_size_binary)
-S3method(vec_ptype_full,arrow_fixed_size_list)
-S3method(vec_ptype_full,arrow_large_list)
-S3method(vec_ptype_full,arrow_list)
-export(Array)
-export(Buffer)
-export(BufferOutputStream)
-export(BufferReader)
-export(ChunkedArray)
-export(Codec)
-export(CompressedInputStream)
-export(CompressedOutputStream)
-export(CompressionType)
-export(CsvConvertOptions)
-export(CsvFileFormat)
-export(CsvFragmentScanOptions)
-export(CsvParseOptions)
-export(CsvReadOptions)
-export(CsvTableReader)
-export(Dataset)
-export(DatasetFactory)
-export(DateUnit)
-export(DictionaryArray)
-export(DirectoryPartitioning)
-export(DirectoryPartitioningFactory)
-export(Expression)
-export(FeatherReader)
-export(Field)
-export(FileFormat)
-export(FileInfo)
-export(FileMode)
-export(FileOutputStream)
-export(FileSelector)
-export(FileSystem)
-export(FileSystemDataset)
-export(FileSystemDatasetFactory)
-export(FileType)
-export(FixedSizeListArray)
-export(FixedSizeListType)
-export(FragmentScanOptions)
-export(HivePartitioning)
-export(HivePartitioningFactory)
-export(InMemoryDataset)
-export(IpcFileFormat)
-export(JsonParseOptions)
-export(JsonReadOptions)
-export(JsonTableReader)
-export(LargeListArray)
-export(ListArray)
-export(LocalFileSystem)
-export(MemoryMappedFile)
-export(MessageReader)
-export(MessageType)
-export(MetadataVersion)
-export(NullEncodingBehavior)
-export(ParquetArrowReaderProperties)
-export(ParquetFileFormat)
-export(ParquetFileReader)
-export(ParquetFileWriter)
-export(ParquetVersionType)
-export(ParquetWriterProperties)
-export(Partitioning)
-export(QuantileInterpolation)
-export(RandomAccessFile)
-export(ReadableFile)
-export(RecordBatch)
-export(RecordBatchFileReader)
-export(RecordBatchFileWriter)
-export(RecordBatchStreamReader)
-export(RecordBatchStreamWriter)
-export(S3FileSystem)
-export(Scalar)
-export(Scanner)
-export(ScannerBuilder)
-export(Schema)
-export(StatusCode)
-export(StructArray)
-export(StructScalar)
-export(SubTreeFileSystem)
-export(Table)
-export(TimeUnit)
-export(TimestampParser)
-export(Type)
-export(UnionDataset)
-export(arrow_available)
-export(arrow_info)
-export(arrow_with_dataset)
-export(arrow_with_parquet)
-export(arrow_with_s3)
-export(binary)
-export(bool)
-export(boolean)
-export(buffer)
-export(call_function)
-export(cast_options)
-export(chunked_array)
-export(codec_is_available)
-export(contains)
-export(copy_files)
-export(cpu_count)
-export(dataset_factory)
-export(date32)
-export(date64)
-export(decimal)
-export(default_memory_pool)
-export(dictionary)
-export(ends_with)
-export(everything)
-export(field)
-export(fixed_size_binary)
-export(fixed_size_list_of)
-export(flight_connect)
-export(flight_get)
-export(flight_path_exists)
-export(flight_put)
-export(float)
-export(float16)
-export(float32)
-export(float64)
-export(halffloat)
-export(hive_partition)
-export(install_arrow)
-export(install_pyarrow)
-export(int16)
-export(int32)
-export(int64)
-export(int8)
-export(is_in)
-export(large_binary)
-export(large_list_of)
-export(large_utf8)
-export(last_col)
-export(list_compute_functions)
-export(list_flights)
-export(list_of)
-export(load_flight_server)
-export(map_batches)
-export(match_arrow)
-export(matches)
-export(mmap_create)
-export(mmap_open)
-export(null)
-export(num_range)
-export(one_of)
-export(open_dataset)
-export(read_arrow)
-export(read_csv_arrow)
-export(read_delim_arrow)
-export(read_feather)
-export(read_ipc_stream)
-export(read_json_arrow)
-export(read_message)
-export(read_parquet)
-export(read_schema)
-export(read_tsv_arrow)
-export(record_batch)
-export(s3_bucket)
-export(schema)
-export(set_cpu_count)
-export(starts_with)
-export(string)
-export(struct)
-export(time32)
-export(time64)
-export(timestamp)
-export(type)
-export(uint16)
-export(uint32)
-export(uint64)
-export(uint8)
-export(unify_schemas)
-export(utf8)
-export(value_counts)
-export(write_arrow)
-export(write_dataset)
-export(write_feather)
-export(write_ipc_stream)
-export(write_parquet)
-export(write_to_raw)
-importFrom(R6,R6Class)
-importFrom(assertthat,assert_that)
-importFrom(assertthat,is.string)
-importFrom(bit64,print.integer64)
-importFrom(bit64,str.integer64)
-importFrom(methods,as)
-importFrom(purrr,as_mapper)
-importFrom(purrr,keep)
-importFrom(purrr,map)
-importFrom(purrr,map2)
-importFrom(purrr,map_chr)
-importFrom(purrr,map_dfr)
-importFrom(purrr,map_int)
-importFrom(purrr,map_lgl)
-importFrom(rlang,"%||%")
-importFrom(rlang,.data)
-importFrom(rlang,abort)
-importFrom(rlang,as_label)
-importFrom(rlang,dots_n)
-importFrom(rlang,enexpr)
-importFrom(rlang,enexprs)
-importFrom(rlang,enquo)
-importFrom(rlang,enquos)
-importFrom(rlang,env)
-importFrom(rlang,env_bind)
-importFrom(rlang,eval_tidy)
-importFrom(rlang,exec)
-importFrom(rlang,expr)
-importFrom(rlang,is_bare_character)
-importFrom(rlang,is_false)
-importFrom(rlang,is_integerish)
-importFrom(rlang,is_quosure)
-importFrom(rlang,list2)
-importFrom(rlang,new_data_mask)
-importFrom(rlang,new_environment)
-importFrom(rlang,quo_get_expr)
-importFrom(rlang,quo_is_null)
-importFrom(rlang,quo_set_expr)
-importFrom(rlang,quos)
-importFrom(rlang,seq2)
-importFrom(rlang,set_names)
-importFrom(rlang,syms)
-importFrom(rlang,warn)
-importFrom(stats,median)
-importFrom(stats,quantile)
-importFrom(tidyselect,contains)
-importFrom(tidyselect,ends_with)
-importFrom(tidyselect,eval_select)
-importFrom(tidyselect,everything)
-importFrom(tidyselect,last_col)
-importFrom(tidyselect,matches)
-importFrom(tidyselect,num_range)
-importFrom(tidyselect,one_of)
-importFrom(tidyselect,starts_with)
-importFrom(tidyselect,vars_pull)
-importFrom(tidyselect,vars_rename)
-importFrom(tidyselect,vars_select)
-importFrom(utils,head)
-importFrom(utils,install.packages)
-importFrom(utils,modifyList)
-importFrom(utils,object.size)
-importFrom(utils,packageVersion)
-importFrom(utils,tail)
-importFrom(vctrs,s3_register)
-importFrom(vctrs,vec_cast)
-importFrom(vctrs,vec_ptype_abbr)
-importFrom(vctrs,vec_ptype_full)
-importFrom(vctrs,vec_size)
-importFrom(vctrs,vec_unique)
-useDynLib(arrow, .registration = TRUE)
diff --git a/r/NEWS.md b/r/NEWS.md
deleted file mode 100644
index c064ac7..0000000
--- a/r/NEWS.md
+++ /dev/null
@@ -1,358 +0,0 @@
-<!---
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied. See the License for the
- specific language governing permissions and limitations
- under the License.
--->
-
-# arrow 3.0.0.9000
-
-## dplyr methods
-
-Many more `dplyr` verbs are supported on Arrow objects:
-
-* `dplyr::mutate()` is now supported in Arrow for many applications. For queries on `Table` and `RecordBatch` that are not yet supported in Arrow, the implementation falls back to pulling data into an in-memory R `data.frame` first, as in the previous release. For queries on `Dataset` (which can be larger than memory), it raises an error if the function is not implemented. The main `mutate()` features that cannot yet be called on Arrow objects are (1) `mutate()` after `group_by()` (which [...]
-* `dplyr::transmute()` (which calls `mutate()`)
-* `dplyr::group_by()` now preserves the `.drop()` argument and supports on-the-fly definition of columns
-* `dplyr::relocate()` to reorder columns
-* `dplyr::arrange()` to sort rows
-* `dplyr::compute()` to evaluate the lazy expressions and return an Arrow Table. This is equivalent to `dplyr::collect(as_data_frame = FALSE)`, which was added in 2.0.0.
-
-Over 100 functions can now be called on Arrow objects inside a `dplyr` verb:
-
-* String functions `nchar()`, `tolower()`, and `toupper()`, along with their `stringr` spellings `str_length()`, `str_to_lower()`, and `str_to_upper()`, are supported in Arrow `dplyr` calls. `str_trim()` is also supported.
-* Regular expression functions `sub()`, `gsub()`, and `grepl()`, along with `str_replace()`, `str_replace_all()`, and `str_detect()`, are supported.
-* `cast(x, type)` and `dictionary_encode()` allow changing the type of columns in Arrow objects; `as.numeric()`, `as.character()`, etc. are exposed as similar type-altering conveniences
-* `dplyr::between()`; the Arrow version also allows the `left` and `right` arguments to be columns in the data and not just scalars
-* Additionally, any Arrow C++ compute function can be called inside a `dplyr` verb. This enables you to access Arrow functions that don't have a direct R mapping. See `list_compute_functions()` for all available functions, which are available in `dplyr` prefixed by `arrow_`.
-* Arrow C++ compute functions now do more systematic type promotion when called on data with different types (e.g. int32 and float64). Previously, Scalars in an expressions were always cast to match the type of the corresponding Array, so this new type promotion enables, among other things, operations on two columns (Arrays) in a dataset. As a side effect, some comparisons that worked in prior versions are no longer supported: for example, `dplyr::filter(arrow_dataset, string_column == 3 [...]
-
-## Datasets
-
-* `open_dataset()` now accepts a vector of file paths (or even a single file path). Among other things, this enables you to open a single very large file and use `write_dataset()` to partition it without having to read the whole file into memory.
-* Datasets can now detect and read a directory of compressed CSVs
-* `write_dataset()` now defaults to `format = "parquet"` and better validates the `format` argument
-* Invalid input for `schema` in `open_dataset()` is now correctly handled
-* Collecting 0 columns from a Dataset now no longer returns all of the columns
-* The `Scanner$Scan()` method has been removed; use `Scanner$ScanBatches()`
-
-## Other improvements
-
-* `value_counts()` to tabulate values in an `Array` or `ChunkedArray`, similar to `base::table()`.
-* `StructArray` objects gain data.frame-like methods, including `names()`, `$`, `[[`, and `dim()`.
-* RecordBatch columns can now be added, replaced, or removed by assigning (`<-`) with either `$` or `[[`
-* Similarly, `Schema` can now be edited by assigning in new types. This enables using the CSV reader to detect the schema of a file, modify the `Schema` object for any columns that you want to read in as a different type, and then use that `Schema` to read the data.
-* Better validation when creating a `Table` with a schema, with columns of different lengths, and with scalar value recycling
-* Reading Parquet files in Japanese or other multi-byte locales on Windows no longer hangs (workaround for a [bug in libstdc++](https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98723); thanks @yutannihilation for the persistence in discovering this!)
-* If you attempt to read string data that has embedded nul (`\0`) characters, the error message now informs you that you can set `options(arrow.skip_nul = TRUE)` to strip them out. It is not recommended to set this option by default since this code path is significantly slower, and most string data does not contain nuls.
-* `read_json_arrow()` now accepts a schema: `read_json_arrow("file.json", schema = schema(col_a = float64(), col_b = string()))`
-
-## Installation and configuration
-
-* The R package can now support working with an Arrow C++ library that has additional features (such as dataset, parquet, string libraries) disabled, and the bundled build script enables setting environment variables to disable them. See `vignette("install", package = "arrow")` for details. This allows a faster, smaller package build in cases where that is useful, and it enables a minimal, functioning R package build on Solaris.
-* On macOS, it is now possible to use the same bundled C++ build that is used by default on Linux, along with all of its customization parameters, by setting the environment variable `FORCE_BUNDLED_BUILD=true`.
-* `arrow` now uses the `mimalloc` memory allocator by default on macOS, if available (as it is in CRAN binaries), instead of `jemalloc`. There are [configuration issues](https://issues.apache.org/jira/browse/ARROW-6994) with `jemalloc` on macOS, and [benchmark analysis](https://ursalabs.org/blog/2021-r-benchmarks-part-1/) shows that this has negative effects on performance, especially on memory-intensive workflows. `jemalloc` remains the default on Linux; `mimalloc` is default on Windows.
-* Setting the `ARROW_DEFAULT_MEMORY_POOL` environment variable to switch memory allocators now works correctly when the Arrow C++ library has been statically linked (as is usually the case when installing from CRAN).
-* The `arrow_info()` function now reports on the additional optional features, as well as the detected SIMD level. If key features or compression libraries are not enabled in the build, `arrow_info()` will refer to the installation vignette for guidance on how to install a more complete build, if desired.
-* If you attempt to read a file that was compressed with a codec that your Arrow build does not contain support for, the error message now will tell you how to reinstall Arrow with that feature enabled.
-* A new vignette about developer environment setup `vignette("developing", package = "arrow")`.
-* When building from source, you can use the environment variable `ARROW_HOME` to point to a specific directory where the Arrow libraries are. This is similar to passing `INCLUDE_DIR` and `LIB_DIR`.
-
-# arrow 3.0.0
-
-## Python and Flight
-
-* Flight methods `flight_get()` and `flight_put()` (renamed from `push_data()` in this release) can handle both Tables and RecordBatches
-* `flight_put()` gains an `overwrite` argument to optionally check for the existence of a resource with the the same name
-* `list_flights()` and `flight_path_exists()` enable you to see available resources on a Flight server
-* `Schema` objects now have `r_to_py` and `py_to_r` methods
-* Schema metadata is correctly preserved when converting Tables to/from Python
-
-## Enhancements
-
-* Arithmetic operations (`+`, `*`, etc.) are supported on Arrays and ChunkedArrays and can be used in filter expressions in Arrow `dplyr` pipelines
-* Table columns can now be added, replaced, or removed by assigning (`<-`) with either `$` or `[[`
-* Column names of Tables and RecordBatches can be renamed by assigning `names()`
-* Large string types can now be written to Parquet files
-* The [pronouns `.data` and `.env`](https://rlang.r-lib.org/reference/tidyeval-data.html) are now fully supported in Arrow `dplyr` pipelines.
-* Option `arrow.skip_nul` (default `FALSE`, as in `base::scan()`) allows conversion of Arrow string (`utf8()`) type data containing embedded nul `\0` characters to R. If set to `TRUE`, nuls will be stripped and a warning is emitted if any are found.
-* `arrow_info()` for an overview of various run-time and build-time Arrow configurations, useful for debugging
-* Set environment variable `ARROW_DEFAULT_MEMORY_POOL` before loading the Arrow package to change memory allocators. Windows packages are built with `mimalloc`; most others are built with both `jemalloc` (used by default) and `mimalloc`. These alternative memory allocators are generally much faster than the system memory allocator, so they are used by default when available, but sometimes it is useful to turn them off for debugging purposes. To disable them, set `ARROW_DEFAULT_MEMORY_POO [...]
-* List columns that have attributes on each element are now also included with the metadata that is saved when creating Arrow tables. This allows `sf` tibbles to faithfully preserved and roundtripped (ARROW-10386).
-* R metadata that exceeds 100Kb is now compressed before being written to a table; see `schema()` for more details.
-
-## Bug fixes
-
-* Fixed a performance regression in converting Arrow string types to R that was present in the 2.0.0 release
-* C++ functions now trigger garbage collection when needed
-* `write_parquet()` can now write RecordBatches
-* Reading a Table from a RecordBatchStreamReader containing 0 batches no longer crashes
-* `readr`'s `problems` attribute is removed when converting to Arrow RecordBatch and table to prevent large amounts of metadata from accumulating inadvertently (ARROW-10624)
-* Fixed reading of compressed Feather files written with Arrow 0.17 (ARROW-10850)
-* `SubTreeFileSystem` gains a useful print method and no longer errors when printing
-
-## Packaging and installation
-
-* Nightly development versions of the conda `r-arrow` package are available with `conda install -c arrow-nightlies -c conda-forge --strict-channel-priority r-arrow`
-* Linux installation now safely supports older `cmake` versions
-* Compiler version checking for enabling S3 support correctly identifies the active compiler
-* Updated guidance and troubleshooting in `vignette("install", package = "arrow")`, especially for known CentOS issues
-* Operating system detection on Linux uses the [`distro`](https://enpiar.com/distro/) package. If your OS isn't correctly identified, please report an issue there.
-
-# arrow 2.0.0
-
-## Datasets
-
-* `write_dataset()` to Feather or Parquet files with partitioning. See the end of `vignette("dataset", package = "arrow")` for discussion and examples.
-* Datasets now have `head()`, `tail()`, and take (`[`) methods. `head()` is optimized but the others may not be performant.
-* `collect()` gains an `as_data_frame` argument, default `TRUE` but when `FALSE` allows you to evaluate the accumulated `select` and `filter` query but keep the result in Arrow, not an R `data.frame`
-* `read_csv_arrow()` supports specifying column types, both with a `Schema` and with the compact string representation for types used in the `readr` package. It also has gained a `timestamp_parsers` argument that lets you express a set of `strptime` parse strings that will be tried to convert columns designated as `Timestamp` type.
-
-## AWS S3 support
-
-* S3 support is now enabled in binary macOS and Windows (Rtools40 only, i.e. R >= 4.0) packages. To enable it on Linux, you need the additional system dependencies `libcurl` and `openssl`, as well as a sufficiently modern compiler. See `vignette("install", package = "arrow")` for details.
-* File readers and writers (`read_parquet()`, `write_feather()`, et al.), as well as `open_dataset()` and `write_dataset()`, allow you to access resources on S3 (or on file systems that emulate S3) either by providing an `s3://` URI or by providing a `FileSystem$path()`. See `vignette("fs", package = "arrow")` for examples.
-* `copy_files()` allows you to recursively copy directories of files from one file system to another, such as from S3 to your local machine.
-
-## Flight RPC
-
-[Flight](https://arrow.apache.org/blog/2019/10/13/introducing-arrow-flight/)
-is a general-purpose client-server framework for high performance
-transport of large datasets over network interfaces.
-The `arrow` R package now provides methods for connecting to Flight RPC servers
-to send and receive data. See `vignette("flight", package = "arrow")` for an overview.
-
-## Computation
-
-* Comparison (`==`, `>`, etc.) and boolean (`&`, `|`, `!`) operations, along with `is.na`, `%in%` and `match` (called `match_arrow()`), on Arrow Arrays and ChunkedArrays are now implemented in the C++ library.
-* Aggregation methods `min()`, `max()`, and `unique()` are implemented for Arrays and ChunkedArrays.
-* `dplyr` filter expressions on Arrow Tables and RecordBatches are now evaluated in the C++ library, rather than by pulling data into R and evaluating. This yields significant performance improvements.
-* `dim()` (`nrow`) for dplyr queries on Table/RecordBatch is now supported
-
-## Packaging and installation
-
-* `arrow` now depends on [`cpp11`](https://cpp11.r-lib.org/), which brings more robust UTF-8 handling and faster compilation
-* The Linux build script now succeeds on older versions of R
-* MacOS binary packages now ship with zstandard compression enabled
-
-## Bug fixes and other enhancements
-
-* Automatic conversion of Arrow `Int64` type when all values fit with an R 32-bit integer now correctly inspects all chunks in a ChunkedArray, and this conversion can be disabled (so that `Int64` always yields a `bit64::integer64` vector) by setting `options(arrow.int64_downcast = FALSE)`.
-* In addition to the data.frame column metadata preserved in round trip, added in 1.0.0, now attributes of the data.frame itself are also preserved in Arrow schema metadata.
-* File writers now respect the system umask setting
-* `ParquetFileReader` has additional methods for accessing individual columns or row groups from the file
-* Various segfaults fixed: invalid input in `ParquetFileWriter`; invalid `ArrowObject` pointer from a saved R object; converting deeply nested structs from Arrow to R
-* The `properties` and `arrow_properties` arguments to `write_parquet()` are deprecated
-
-# arrow 1.0.1
-
-## Bug fixes
-
-* Filtering a Dataset that has multiple partition keys using an `%in%` expression now faithfully returns all relevant rows
-* Datasets can now have path segments in the root directory that start with `.` or `_`; files and subdirectories starting with those prefixes are still ignored
-* `open_dataset("~/path")` now correctly expands the path
-* The `version` option to `write_parquet()` is now correctly implemented
-* An UBSAN failure in the `parquet-cpp` library has been fixed
-* For bundled Linux builds, the logic for finding `cmake` is more robust, and you can now specify a `/path/to/cmake` by setting the `CMAKE` environment variable
-
-# arrow 1.0.0
-
-## Arrow format conversion
-
-* `vignette("arrow", package = "arrow")` includes tables that explain how R types are converted to Arrow types and vice versa.
-* Support added for converting to/from more Arrow types: `uint64`, `binary`, `fixed_size_binary`, `large_binary`, `large_utf8`, `large_list`, `list` of `structs`.
-* `character` vectors that exceed 2GB are converted to Arrow `large_utf8` type
-* `POSIXlt` objects can now be converted to Arrow (`struct`)
-* R `attributes()` are preserved in Arrow metadata when converting to Arrow RecordBatch and table and are restored when converting from Arrow. This means that custom subclasses, such as `haven::labelled`, are preserved in round trip through Arrow.
-* Schema metadata is now exposed as a named list, and it can be modified by assignment like `batch$metadata$new_key <- "new value"`
-* Arrow types `int64`, `uint32`, and `uint64` now are converted to R `integer` if all values fit in bounds
-* Arrow `date32` is now converted to R `Date` with `double` underlying storage. Even though the data values themselves are integers, this provides more strict round-trip fidelity
-* When converting to R `factor`, `dictionary` ChunkedArrays that do not have identical dictionaries are properly unified
-* In the 1.0 release, the Arrow IPC metadata version is increased from V4 to V5. By default, `RecordBatch{File,Stream}Writer` will write V5, but you can specify an alternate `metadata_version`. For convenience, if you know the consumer you're writing to cannot read V5, you can set the environment variable `ARROW_PRE_1_0_METADATA_VERSION=1` to write V4 without changing any other code.
-
-## Datasets
-
-* CSV and other text-delimited datasets are now supported
-* With a custom C++ build, it is possible to read datasets directly on S3 by passing a URL like `ds <- open_dataset("s3://...")`. Note that this currently requires a special C++ library build with additional dependencies--this is not yet available in CRAN releases or in nightly packages.
-* When reading individual CSV and JSON files, compression is automatically detected from the file extension
-
-## Other enhancements
-
-* Initial support for C++ aggregation methods: `sum()` and `mean()` are implemented for `Array` and `ChunkedArray`
-* Tables and RecordBatches have additional data.frame-like methods, including `dimnames()` and `as.list()`
-* Tables and ChunkedArrays can now be moved to/from Python via `reticulate`
-
-## Bug fixes and deprecations
-
-* Non-UTF-8 strings (common on Windows) are correctly coerced to UTF-8 when passing to Arrow memory and appropriately re-localized when converting to R
-* The `coerce_timestamps` option to `write_parquet()` is now correctly implemented.
-* Creating a Dictionary array respects the `type` definition if provided by the user
-* `read_arrow` and `write_arrow` are now deprecated; use the `read/write_feather()` and `read/write_ipc_stream()` functions depending on whether you're working with the Arrow IPC file or stream format, respectively.
-* Previously deprecated `FileStats`, `read_record_batch`, and `read_table` have been removed.
-
-## Installation and packaging
-
-* For improved performance in memory allocation, macOS and Linux binaries now have `jemalloc` included, and Windows packages use `mimalloc`
-* Linux installation: some tweaks to OS detection for binaries, some updates to known installation issues in the vignette
-* The bundled libarrow is built with the same `CC` and `CXX` values that R uses
-* Failure to build the bundled libarrow yields a clear message
-* Various streamlining efforts to reduce library size and compile time
-
-# arrow 0.17.1
-
-* Updates for compatibility with `dplyr` 1.0
-* `reticulate::r_to_py()` conversion now correctly works automatically, without having to call the method yourself
-* Assorted bug fixes in the C++ library around Parquet reading
-
-# arrow 0.17.0
-
-## Feather v2
-
-This release includes support for version 2 of the Feather file format.
-Feather v2 features full support for all Arrow data types,
-fixes the 2GB per-column limitation for large amounts of string data,
-and it allows files to be compressed using either `lz4` or `zstd`.
-`write_feather()` can write either version 2 or
-[version 1](https://github.com/wesm/feather) Feather files, and `read_feather()`
-automatically detects which file version it is reading.
-
-Related to this change, several functions around reading and writing data
-have been reworked. `read_ipc_stream()` and `write_ipc_stream()` have been
-added to facilitate writing data to the Arrow IPC stream format, which is
-slightly different from the IPC file format (Feather v2 *is* the IPC file format).
-
-Behavior has been standardized: all `read_<format>()` return an R `data.frame`
-(default) or a `Table` if the argument `as_data_frame = FALSE`;
-all `write_<format>()` functions return the data object, invisibly.
-To facilitate some workflows, a special `write_to_raw()` function is added
-to wrap `write_ipc_stream()` and return the `raw` vector containing the buffer
-that was written.
-
-To achieve this standardization, `read_table()`, `read_record_batch()`,
-`read_arrow()`, and `write_arrow()` have been deprecated.
-
-## Python interoperability
-
-The 0.17 Apache Arrow release includes a C data interface that allows
-exchanging Arrow data in-process at the C level without copying
-and without libraries having a build or runtime dependency on each other. This enables
-us to use `reticulate` to share data between R and Python (`pyarrow`) efficiently.
-
-See `vignette("python", package = "arrow")` for details.
-
-## Datasets
-
-* Dataset reading benefits from many speedups and fixes in the C++ library
-* Datasets have a `dim()` method, which sums rows across all files (#6635, @boshek)
-* Combine multiple datasets into a single queryable `UnionDataset` with the `c()` method
-* Dataset filtering now treats `NA` as `FALSE`, consistent with `dplyr::filter()`
-* Dataset filtering is now correctly supported for all Arrow date/time/timestamp column types
-* `vignette("dataset", package = "arrow")` now has correct, executable code
-
-## Installation
-
-* Installation on Linux now builds C++ the library from source by default, with some compression libraries disabled. For a faster, richer build, set the environment variable `NOT_CRAN=true`. See `vignette("install", package = "arrow")` for details and more options.
-* Source installation is faster and more reliable on more Linux distributions.
-
-## Other bug fixes and enhancements
-
-* `unify_schemas()` to create a `Schema` containing the union of fields in multiple schemas
-* Timezones are faithfully preserved in roundtrip between R and Arrow
-* `read_feather()` and other reader functions close any file connections they open
-* Arrow R6 objects no longer have namespace collisions when the `R.oo` package is also loaded
-* `FileStats` is renamed to `FileInfo`, and the original spelling has been deprecated
-
-# arrow 0.16.0.2
-
-* `install_arrow()` now installs the latest release of `arrow`, including Linux dependencies, either for CRAN releases or for development builds (if `nightly = TRUE`)
-* Package installation on Linux no longer downloads C++ dependencies unless the `LIBARROW_DOWNLOAD` or `NOT_CRAN` environment variable is set
-* `write_feather()`, `write_arrow()` and `write_parquet()` now return their input,
-similar to the `write_*` functions in the `readr` package (#6387, @boshek)
-* Can now infer the type of an R `list` and create a ListArray when all list elements are the same type (#6275, @michaelchirico)
-
-# arrow 0.16.0
-
-## Multi-file datasets
-
-This release includes a `dplyr` interface to Arrow Datasets,
-which let you work efficiently with large, multi-file datasets as a single entity.
-Explore a directory of data files with `open_dataset()` and then use `dplyr` methods to `select()`, `filter()`, etc. Work will be done where possible in Arrow memory. When necessary, data is pulled into R for further computation. `dplyr` methods are conditionally loaded if you have `dplyr` available; it is not a hard dependency.
-
-See `vignette("dataset", package = "arrow")` for details.
-
-## Linux installation
-
-A source package installation (as from CRAN) will now handle its C++ dependencies automatically.
-For common Linux distributions and versions, installation will retrieve a prebuilt static
-C++ library for inclusion in the package; where this binary is not available,
-the package executes a bundled script that should build the Arrow C++ library with
-no system dependencies beyond what R requires.
-
-See `vignette("install", package = "arrow")` for details.
-
-## Data exploration
-
-* `Table`s and `RecordBatch`es also have `dplyr` methods.
-* For exploration without `dplyr`, `[` methods for Tables, RecordBatches, Arrays, and ChunkedArrays now support natural row extraction operations. These use the C++ `Filter`, `Slice`, and `Take` methods for efficient access, depending on the type of selection vector.
-* An experimental, lazily evaluated `array_expression` class has also been added, enabling among other things the ability to filter a Table with some function of Arrays, such as `arrow_table[arrow_table$var1 > 5, ]` without having to pull everything into R first.
-
-## Compression
-
-* `write_parquet()` now supports compression
-* `codec_is_available()` returns `TRUE` or `FALSE` whether the Arrow C++ library was built with support for a given compression library (e.g. gzip, lz4, snappy)
-* Windows builds now include support for zstd and lz4 compression (#5814, @gnguy)
-
-## Other fixes and improvements
-
-* Arrow null type is now supported
-* Factor types are now preserved in round trip through Parquet format (#6135, @yutannihilation)
-* Reading an Arrow dictionary type coerces dictionary values to `character` (as R `factor` levels are required to be) instead of raising an error
-* Many improvements to Parquet function documentation (@karldw, @khughitt)
-
-# arrow 0.15.1
-
-* This patch release includes bugfixes in the C++ library around dictionary types and Parquet reading.
-
-# arrow 0.15.0
-
-## Breaking changes
-
-* The R6 classes that wrap the C++ classes are now documented and exported and have been renamed to be more R-friendly. Users of the high-level R interface in this package are not affected. Those who want to interact with the Arrow C++ API more directly should work with these objects and methods. As part of this change, many functions that instantiated these R6 objects have been removed in favor of `Class$create()` methods. Notably, `arrow::array()` and `arrow::table()` have been removed [...]
-* Due to a subtle change in the Arrow message format, data written by the 0.15 version libraries may not be readable by older versions. If you need to send data to a process that uses an older version of Arrow (for example, an Apache Spark server that hasn't yet updated to Arrow 0.15), you can set the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`.
-* The `as_tibble` argument in the `read_*()` functions has been renamed to `as_data_frame` (ARROW-6337, @jameslamb)
-* The `arrow::Column` class has been removed, as it was removed from the C++ library
-
-## New features
-
-* `Table` and `RecordBatch` objects have S3 methods that enable you to work with them more like `data.frame`s. Extract columns, subset, and so on. See `?Table` and `?RecordBatch` for examples.
-* Initial implementation of bindings for the C++ File System API. (ARROW-6348)
-* Compressed streams are now supported on Windows (ARROW-6360), and you can also specify a compression level (ARROW-6533)
-
-## Other upgrades
-
-* Parquet file reading is much, much faster, thanks to improvements in the Arrow C++ library.
-* `read_csv_arrow()` supports more parsing options, including `col_names`, `na`, `quoted_na`, and `skip`
-* `read_parquet()` and `read_feather()` can ingest data from a `raw` vector (ARROW-6278)
-* File readers now properly handle paths that need expanding, such as `~/file.parquet` (ARROW-6323)
-* Improved support for creating types in a schema: the types' printed names (e.g. "double") are guaranteed to be valid to use in instantiating a schema (e.g. `double()`), and time types can be created with human-friendly resolution strings ("ms", "s", etc.). (ARROW-6338, ARROW-6364)
-
-
-# arrow 0.14.1
-
-Initial CRAN release of the `arrow` package. Key features include:
-
-* Read and write support for various file formats, including Parquet, Feather/Arrow, CSV, and JSON.
-* API bindings to the C++ library for Arrow data types and objects, as well as mapping between Arrow types and R data types.
-* Tools for helping with C++ library configuration and installation.
diff --git a/r/R/array-data.R b/r/R/array-data.R
deleted file mode 100644
index 08b0913..0000000
--- a/r/R/array-data.R
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @title ArrayData class
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @description The `ArrayData` class allows you to get and inspect the data
-#' inside an `arrow::Array`.
-#'
-#' @section Usage:
-#'
-#' ```
-#' data <- Array$create(x)$data()
-#'
-#' data$type()
-#' data$length()
-#' data$null_count()
-#' data$offset()
-#' data$buffers()
-#' ```
-#'
-#' @section Methods:
-#'
-#' ...
-#'
-#' @rdname ArrayData
-#' @name ArrayData
-#' @include type.R
-ArrayData <- R6Class("ArrayData",
- inherit = ArrowObject,
- active = list(
- type = function() ArrayData__get_type(self),
- length = function() ArrayData__get_length(self),
- null_count = function() ArrayData__get_null_count(self),
- offset = function() ArrayData__get_offset(self),
- buffers = function() ArrayData__buffers(self)
- )
-)
diff --git a/r/R/array.R b/r/R/array.R
deleted file mode 100644
index 1d63c57..0000000
--- a/r/R/array.R
+++ /dev/null
@@ -1,301 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-datum.R
-
-#' @title Arrow Arrays
-#' @description An `Array` is an immutable data array with some logical type
-#' and some length. Most logical types are contained in the base
-#' `Array` class; there are also subclasses for `DictionaryArray`, `ListArray`,
-#' and `StructArray`.
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @section Factory:
-#' The `Array$create()` factory method instantiates an `Array` and
-#' takes the following arguments:
-#' * `x`: an R vector, list, or `data.frame`
-#' * `type`: an optional [data type][data-type] for `x`. If omitted, the type
-#' will be inferred from the data.
-#'
-#' `Array$create()` will return the appropriate subclass of `Array`, such as
-#' `DictionaryArray` when given an R factor.
-#'
-#' To compose a `DictionaryArray` directly, call `DictionaryArray$create()`,
-#' which takes two arguments:
-#' * `x`: an R vector or `Array` of integers for the dictionary indices
-#' * `dict`: an R vector or `Array` of dictionary values (like R factor levels
-#' but not limited to strings only)
-#' @section Usage:
-#'
-#' ```
-#' a <- Array$create(x)
-#' length(a)
-#'
-#' print(a)
-#' a == a
-#' ```
-#'
-#' @section Methods:
-#'
-#' - `$IsNull(i)`: Return true if value at index is null. Does not boundscheck
-#' - `$IsValid(i)`: Return true if value at index is valid. Does not boundscheck
-#' - `$length()`: Size in the number of elements this array contains
-#' - `$offset()`: A relative position into another array's data, to enable zero-copy slicing
-#' - `$null_count()`: The number of null entries in the array
-#' - `$type()`: logical type of data
-#' - `$type_id()`: type id
-#' - `$Equals(other)` : is this array equal to `other`
-#' - `$ApproxEquals(other)` :
-#' - `$Diff(other)` : return a string expressing the difference between two arrays
-#' - `$data()`: return the underlying [ArrayData][ArrayData]
-#' - `$as_vector()`: convert to an R vector
-#' - `$ToString()`: string representation of the array
-#' - `$Slice(offset, length = NULL)`: Construct a zero-copy slice of the array
-#' with the indicated offset and length. If length is `NULL`, the slice goes
-#' until the end of the array.
-#' - `$Take(i)`: return an `Array` with values at positions given by integers
-#' (R vector or Array Array) `i`.
-#' - `$Filter(i, keep_na = TRUE)`: return an `Array` with values at positions where logical
-#' vector (or Arrow boolean Array) `i` is `TRUE`.
-#' - `$SortIndices(descending = FALSE)`: return an `Array` of integer positions that can be
-#' used to rearrange the `Array` in ascending or descending order
-#' - `$RangeEquals(other, start_idx, end_idx, other_start_idx)` :
-#' - `$cast(target_type, safe = TRUE, options = cast_options(safe))`: Alter the
-#' data in the array to change its type.
-#' - `$View(type)`: Construct a zero-copy view of this array with the given type.
-#' - `$Validate()` : Perform any validation checks to determine obvious inconsistencies
-#' within the array's internal data. This can be an expensive check, potentially `O(length)`
-#'
-#' @rdname array
-#' @name array
-#' @export
-Array <- R6Class("Array",
- inherit = ArrowDatum,
- public = list(
- IsNull = function(i) Array__IsNull(self, i),
- IsValid = function(i) Array__IsValid(self, i),
- length = function() Array__length(self),
- type_id = function() Array__type_id(self),
- Equals = function(other, ...) {
- inherits(other, "Array") && Array__Equals(self, other)
- },
- ApproxEquals = function(other) {
- inherits(other, "Array") && Array__ApproxEquals(self, other)
- },
- Diff = function(other) {
- if (!inherits(other, "Array")) {
- other <- Array$create(other)
- }
- Array__Diff(self, other)
- },
- data = function() Array__data(self),
- as_vector = function() Array__as_vector(self),
- ToString = function() {
- typ <- paste0("<", self$type$ToString(), ">")
- paste(typ, Array__ToString(self), sep = "\n")
- },
- Slice = function(offset, length = NULL) {
- if (is.null(length)) {
- Array__Slice1(self, offset)
- } else {
- Array__Slice2(self, offset, length)
- }
- },
- Take = function(i) {
- if (is.numeric(i)) {
- i <- as.integer(i)
- }
- if (is.integer(i)) {
- i <- Array$create(i)
- }
- call_function("take", self, i)
- },
- Filter = function(i, keep_na = TRUE) {
- if (is.logical(i)) {
- i <- Array$create(i)
- }
- assert_is(i, "Array")
- call_function("filter", self, i, options = list(keep_na = keep_na))
- },
- SortIndices = function(descending = FALSE) {
- assert_that(is.logical(descending))
- assert_that(length(descending) == 1L)
- assert_that(!is.na(descending))
- call_function("array_sort_indices", self, options = list(order = descending))
- },
- RangeEquals = function(other, start_idx, end_idx, other_start_idx = 0L) {
- assert_is(other, "Array")
- Array__RangeEquals(self, other, start_idx, end_idx, other_start_idx)
- },
- View = function(type) {
- Array$create(Array__View(self, as_type(type)))
- },
- Validate = function() Array__Validate(self)
- ),
- active = list(
- null_count = function() Array__null_count(self),
- offset = function() Array__offset(self),
- type = function() Array__type(self)
- )
-)
-Array$create <- function(x, type = NULL) {
- if (!is.null(type)) {
- type <- as_type(type)
- }
- if (inherits(x, "Scalar")) {
- out <- x$as_array()
- if (!is.null(type)) {
- out <- out$cast(type)
- }
- return(out)
- }
- vec_to_arrow(x, type)
-}
-
-#' @rdname array
-#' @usage NULL
-#' @format NULL
-#' @export
-DictionaryArray <- R6Class("DictionaryArray", inherit = Array,
- public = list(
- indices = function() DictionaryArray__indices(self),
- dictionary = function() DictionaryArray__dictionary(self)
- ),
- active = list(
- ordered = function() self$type$ordered
- )
-)
-DictionaryArray$create <- function(x, dict = NULL) {
- if (is.factor(x)) {
- # The simple case: converting a factor.
- # Ignoring `dict`; should probably error if dict is not NULL
- return(Array$create(x))
- }
-
- assert_that(!is.null(dict))
- if (!is.Array(x)) {
- x <- Array$create(x)
- }
- if (!is.Array(dict)) {
- dict <- Array$create(dict)
- }
- type <- DictionaryType$create(x$type, dict$type)
- DictionaryArray__FromArrays(type, x, dict)
-}
-
-#' @rdname array
-#' @usage NULL
-#' @format NULL
-#' @export
-StructArray <- R6Class("StructArray", inherit = Array,
- public = list(
- field = function(i) StructArray__field(self, i),
- GetFieldByName = function(name) StructArray__GetFieldByName(self, name),
- Flatten = function() StructArray__Flatten(self)
- )
-)
-
-
-#' @export
-`[[.StructArray` <- function(x, i, ...) {
- if (is.character(i)) {
- x$GetFieldByName(i)
- } else if (is.numeric(i)) {
- x$field(i - 1)
- } else {
- stop("'i' must be character or numeric, not ", class(i), call. = FALSE)
- }
-}
-
-#' @export
-`$.StructArray` <- function(x, name, ...) {
- assert_that(is.string(name))
- if (name %in% ls(x)) {
- get(name, x)
- } else {
- x$GetFieldByName(name)
- }
-}
-
-#' @export
-names.StructArray <- function(x, ...) StructType__field_names(x$type)
-
-#' @export
-dim.StructArray <- function(x, ...) c(length(x), x$type$num_fields)
-
-#' @export
-as.data.frame.StructArray <- function(x, row.names = NULL, optional = FALSE, ...) {
- as.vector(x)
-}
-
-#' @rdname array
-#' @usage NULL
-#' @format NULL
-#' @export
-ListArray <- R6Class("ListArray", inherit = Array,
- public = list(
- values = function() ListArray__values(self),
- value_length = function(i) ListArray__value_length(self, i),
- value_offset = function(i) ListArray__value_offset(self, i),
- raw_value_offsets = function() ListArray__raw_value_offsets(self)
- ),
- active = list(
- value_type = function() ListArray__value_type(self)
- )
-)
-
-#' @rdname array
-#' @usage NULL
-#' @format NULL
-#' @export
-LargeListArray <- R6Class("LargeListArray", inherit = Array,
- public = list(
- values = function() LargeListArray__values(self),
- value_length = function(i) LargeListArray__value_length(self, i),
- value_offset = function(i) LargeListArray__value_offset(self, i),
- raw_value_offsets = function() LargeListArray__raw_value_offsets(self)
- ),
- active = list(
- value_type = function() LargeListArray__value_type(self)
- )
-)
-
-#' @rdname array
-#' @usage NULL
-#' @format NULL
-#' @export
-FixedSizeListArray <- R6Class("FixedSizeListArray", inherit = Array,
- public = list(
- values = function() FixedSizeListArray__values(self),
- value_length = function(i) FixedSizeListArray__value_length(self, i),
- value_offset = function(i) FixedSizeListArray__value_offset(self, i)
- ),
- active = list(
- value_type = function() FixedSizeListArray__value_type(self),
- list_size = function() self$type$list_size
- )
-)
-
-is.Array <- function(x, type = NULL) {
- is_it <- inherits(x, c("Array", "ChunkedArray"))
- if (is_it && !is.null(type)) {
- is_it <- x$type$ToString() %in% type
- }
- is_it
-}
diff --git a/r/R/arrow-datum.R b/r/R/arrow-datum.R
deleted file mode 100644
index dd43307..0000000
--- a/r/R/arrow-datum.R
+++ /dev/null
@@ -1,165 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-
-# Base class for Array, ChunkedArray, and Scalar, for S3 method dispatch only.
-# Does not exist in C++ class hierarchy
-ArrowDatum <- R6Class("ArrowDatum", inherit = ArrowObject,
- public = list(
- cast = function(target_type, safe = TRUE, ...) {
- opts <- cast_options(safe, ...)
- opts$to_type <- as_type(target_type)
- call_function("cast", self, options = opts)
- }
- )
-)
-
-#' @export
-length.ArrowDatum <- function(x) x$length()
-
-#' @export
-is.na.ArrowDatum <- function(x) call_function("is_null", x)
-
-#' @export
-is.nan.ArrowDatum <- function(x) call_function("is_nan", x)
-
-#' @export
-as.vector.ArrowDatum <- function(x, mode) {
- tryCatch(
- x$as_vector(),
- error = handle_embedded_nul_error
- )
-}
-
-filter_rows <- function(x, i, keep_na = TRUE, ...) {
- # General purpose function for [ row subsetting with R semantics
- # Based on the input for `i`, calls x$Filter, x$Slice, or x$Take
- nrows <- x$num_rows %||% x$length() # Depends on whether Array or Table-like
- if (inherits(i, "array_expression")) {
- # Evaluate it
- i <- eval_array_expression(i)
- }
- if (is.logical(i)) {
- if (isTRUE(i)) {
- # Shortcut without doing any work
- x
- } else {
- i <- rep_len(i, nrows) # For R recycling behavior; consider vctrs::vec_recycle()
- x$Filter(i, keep_na)
- }
- } else if (is.numeric(i)) {
- if (all(i < 0)) {
- # in R, negative i means "everything but i"
- i <- setdiff(seq_len(nrows), -1 * i)
- }
- if (is.sliceable(i)) {
- x$Slice(i[1] - 1, length(i))
- } else if (all(i > 0)) {
- x$Take(i - 1)
- } else {
- stop("Cannot mix positive and negative indices", call. = FALSE)
- }
- } else if (is.Array(i, INTEGER_TYPES)) {
- # NOTE: this doesn't do the - 1 offset
- x$Take(i)
- } else if (is.Array(i, "bool")) {
- x$Filter(i, keep_na)
- } else {
- # Unsupported cases
- if (is.Array(i)) {
- stop("Cannot extract rows with an Array of type ", i$type$ToString(), call. = FALSE)
- }
- stop("Cannot extract rows with an object of class ", class(i), call.=FALSE)
- }
-}
-
-#' @export
-`[.ArrowDatum` <- filter_rows
-
-#' @importFrom utils head
-#' @export
-head.ArrowDatum <- function(x, n = 6L, ...) {
- assert_is(n, c("numeric", "integer"))
- assert_that(length(n) == 1)
- len <- NROW(x)
- if (n < 0) {
- # head(x, negative) means all but the last n rows
- n <- max(len + n, 0)
- } else {
- n <- min(len, n)
- }
- if (n == len) {
- return(x)
- }
- x$Slice(0, n)
-}
-
-#' @importFrom utils tail
-#' @export
-tail.ArrowDatum <- function(x, n = 6L, ...) {
- assert_is(n, c("numeric", "integer"))
- assert_that(length(n) == 1)
- len <- NROW(x)
- if (n < 0) {
- # tail(x, negative) means all but the first n rows
- n <- min(-n, len)
- } else {
- n <- max(len - n, 0)
- }
- if (n == 0) {
- return(x)
- }
- x$Slice(n)
-}
-
-is.sliceable <- function(i) {
- # Determine whether `i` can be expressed as a $Slice() command
- is.numeric(i) &&
- length(i) > 0 &&
- all(i > 0) &&
- identical(as.integer(i), i[1]:i[length(i)])
-}
-
-#' @export
-as.double.ArrowDatum <- function(x, ...) as.double(as.vector(x), ...)
-
-#' @export
-as.integer.ArrowDatum <- function(x, ...) as.integer(as.vector(x), ...)
-
-#' @export
-as.character.ArrowDatum <- function(x, ...) as.character(as.vector(x), ...)
-
-#' @export
-sort.ArrowDatum <- function(x, decreasing = FALSE, na.last = NA, ...) {
- # Arrow always sorts nulls at the end of the array. This corresponds to
- # sort(na.last = TRUE). For the other two cases (na.last = NA and
- # na.last = FALSE) we need to use workarounds.
- # TODO: Implement this more cleanly after ARROW-12063
- if (is.na(na.last)) {
- # Filter out NAs before sorting
- x <- x$Filter(!is.na(x))
- x$Take(x$SortIndices(descending = decreasing))
- } else if (na.last) {
- x$Take(x$SortIndices(descending = decreasing))
- } else {
- # Create a new array that encodes missing values as 1 and non-missing values
- # as 0. Sort descending by that array first to get the NAs at the beginning
- tbl <- Table$create(x = x, `is_na` = as.integer(is.na(x)))
- tbl$x$Take(tbl$SortIndices(names = c("is_na", "x"), descending = c(TRUE, decreasing)))
- }
-}
diff --git a/r/R/arrow-package.R b/r/R/arrow-package.R
deleted file mode 100644
index 30d5949..0000000
--- a/r/R/arrow-package.R
+++ /dev/null
@@ -1,278 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @importFrom stats quantile median
-#' @importFrom R6 R6Class
-#' @importFrom purrr as_mapper map map2 map_chr map_dfr map_int map_lgl keep
-#' @importFrom assertthat assert_that is.string
-#' @importFrom rlang list2 %||% is_false abort dots_n warn enquo quo_is_null enquos is_integerish quos eval_tidy new_data_mask syms env new_environment env_bind as_label set_names exec is_bare_character quo_get_expr quo_set_expr .data seq2 is_quosure enexpr enexprs expr
-#' @importFrom tidyselect vars_pull vars_rename vars_select eval_select
-#' @useDynLib arrow, .registration = TRUE
-#' @keywords internal
-"_PACKAGE"
-
-#' @importFrom vctrs s3_register vec_size vec_cast vec_unique
-.onLoad <- function(...) {
- dplyr_methods <- paste0(
- "dplyr::",
- c(
- "select", "filter", "collect", "summarise", "group_by", "groups",
- "group_vars", "group_by_drop_default", "ungroup", "mutate", "transmute",
- "arrange", "rename", "pull", "relocate", "compute"
- )
- )
- for (cl in c("Dataset", "ArrowTabular", "arrow_dplyr_query")) {
- for (m in dplyr_methods) {
- s3_register(m, cl)
- }
- }
- s3_register("dplyr::tbl_vars", "arrow_dplyr_query")
-
- for (cl in c("Array", "RecordBatch", "ChunkedArray", "Table", "Schema")) {
- s3_register("reticulate::py_to_r", paste0("pyarrow.lib.", cl))
- s3_register("reticulate::r_to_py", cl)
- }
-
- # Create these once, at package build time
- if (arrow_available()) {
- dplyr_functions$dataset <- build_function_list(build_dataset_expression)
- dplyr_functions$array <- build_function_list(build_array_expression)
- }
- invisible()
-}
-
-.onAttach <- function(libname, pkgname) {
- if (!arrow_available()) {
- msg <- paste(
- "The Arrow C++ library is not available. To retry installation with debug output, run:",
- " install_arrow(verbose = TRUE)",
- "See https://arrow.apache.org/docs/r/articles/install.html for more guidance and troubleshooting.",
- sep = "\n"
- )
- packageStartupMessage(msg)
- } else {
- # Just to be extra safe, let's wrap this in a try();
- # we don't a failed startup message to prevent the package from loading
- try({
- features <- arrow_info()$capabilities
- # That has all of the #ifdef features, plus the compression libs and the
- # string libraries (but not the memory allocators, they're added elsewhere)
- #
- # Let's print a message if some are off
- if (some_features_are_off(features)) {
- packageStartupMessage("See arrow_info() for available features")
- }
- })
- }
-}
-
-#' Is the C++ Arrow library available?
-#'
-#' You won't generally need to call these function, but they're made available
-#' for diagnostic purposes.
-#' @return `TRUE` or `FALSE` depending on whether the package was installed
-#' with:
-#' * The Arrow C++ library (check with `arrow_available()`)
-#' * Arrow Dataset support enabled (check with `arrow_with_dataset()`)
-#' * Parquet support enabled (check with `arrow_with_parquet()`)
-#' * Amazon S3 support enabled (check with `arrow_with_s3()`)
-#' @export
-#' @examples
-#' arrow_available()
-#' arrow_with_dataset()
-#' arrow_with_parquet()
-#' arrow_with_s3()
-#' @seealso If any of these are `FALSE`, see
-#' `vignette("install", package = "arrow")` for guidance on reinstalling the
-#' package.
-arrow_available <- function() {
- tryCatch(.Call(`_arrow_available`), error = function(e) return(FALSE))
-}
-
-#' @rdname arrow_available
-#' @export
-arrow_with_dataset <- function() {
- tryCatch(.Call(`_dataset_available`), error = function(e) return(FALSE))
-}
-
-#' @rdname arrow_available
-#' @export
-arrow_with_parquet <- function() {
- tryCatch(.Call(`_parquet_available`), error = function(e) return(FALSE))
-}
-
-#' @rdname arrow_available
-#' @export
-arrow_with_s3 <- function() {
- tryCatch(.Call(`_s3_available`), error = function(e) return(FALSE))
-}
-
-option_use_threads <- function() {
- !is_false(getOption("arrow.use_threads"))
-}
-
-#' Report information on the package's capabilities
-#'
-#' This function summarizes a number of build-time configurations and run-time
-#' settings for the Arrow package. It may be useful for diagnostics.
-#' @return A list including version information, boolean "capabilities", and
-#' statistics from Arrow's memory allocator, and also Arrow's run-time
-#' information.
-#' @export
-#' @importFrom utils packageVersion
-arrow_info <- function() {
- opts <- options()
- out <- list(
- version = packageVersion("arrow"),
- libarrow = arrow_available(),
- options = opts[grep("^arrow\\.", names(opts))]
- )
- if (out$libarrow) {
- pool <- default_memory_pool()
- runtimeinfo <- runtime_info()
- compute_funcs <- list_compute_functions()
- out <- c(out, list(
- capabilities = c(
- dataset = arrow_with_dataset(),
- parquet = arrow_with_parquet(),
- s3 = arrow_with_s3(),
- utf8proc = "utf8_upper" %in% compute_funcs,
- re2 = "replace_substring_regex" %in% compute_funcs,
- vapply(tolower(names(CompressionType)[-1]), codec_is_available, logical(1))
- ),
- memory_pool = list(
- backend_name = pool$backend_name,
- bytes_allocated = pool$bytes_allocated,
- max_memory = pool$max_memory,
- available_backends = supported_memory_backends()
- ),
- runtime_info = list(
- simd_level = runtimeinfo[1],
- detected_simd_level = runtimeinfo[2]
- )
- ))
- }
- structure(out, class = "arrow_info")
-}
-
-some_features_are_off <- function(features) {
- # `features` is a named logical vector (as in arrow_info()$capabilities)
- # Let's exclude some less relevant ones
- blocklist <- c("lzo", "bz2", "brotli")
- # Return TRUE if any of the other features are FALSE
- !all(features[setdiff(names(features), blocklist)])
-}
-
-#' @export
-print.arrow_info <- function(x, ...) {
- print_key_values <- function(title, vals, ...) {
- # Make a key-value table for printing, no column names
- df <- data.frame(vals, stringsAsFactors = FALSE, ...)
- names(df) <- ""
-
- cat(title, ":\n", sep = "")
- print(df)
- cat("\n")
- }
- cat("Arrow package version: ", format(x$version), "\n\n", sep = "")
- if (x$libarrow) {
- print_key_values("Capabilities", c(
- x$capabilities,
- jemalloc = "jemalloc" %in% x$memory_pool$available_backends,
- mimalloc = "mimalloc" %in% x$memory_pool$available_backends
- ))
- if (some_features_are_off(x$capabilities) && identical(tolower(Sys.info()[["sysname"]]), "linux")) {
- # Only on linux because (e.g.) we disable certain features on purpose on rtools35 and solaris
- cat("To reinstall with more optional capabilities enabled, see\n https://arrow.apache.org/docs/r/articles/install.html\n\n")
- }
-
- if (length(x$options)) {
- print_key_values("Arrow options()", map_chr(x$options, format))
- }
-
- format_bytes <- function(b, units = "auto", digits = 2L, ...) {
- format(structure(b, class = "object_size"), units = units, digits = digits, ...)
- }
- print_key_values("Memory", c(
- Allocator = x$memory_pool$backend_name,
- # utils:::format.object_size is not properly vectorized
- Current = format_bytes(x$memory_pool$bytes_allocated, ...),
- Max = format_bytes(x$memory_pool$max_memory, ...)
- ))
- print_key_values("Runtime", c(
- `SIMD Level` = x$runtime_info$simd_level,
- `Detected SIMD Level` = x$runtime_info$detected_simd_level
- ))
- } else {
- cat("Arrow C++ library not available. See https://arrow.apache.org/docs/r/articles/install.html for troubleshooting.\n")
- }
- invisible(x)
-}
-
-option_compress_metadata <- function() {
- !is_false(getOption("arrow.compress_metadata"))
-}
-
-#' @include enums.R
-ArrowObject <- R6Class("ArrowObject",
- public = list(
- initialize = function(xp) self$set_pointer(xp),
-
- pointer = function() get(".:xp:.", envir = self),
- `.:xp:.` = NULL,
- set_pointer = function(xp) {
- if (!inherits(xp, "externalptr")) {
- stop(
- class(self)[1], "$new() requires a pointer as input: ",
- "did you mean $create() instead?",
- call. = FALSE
- )
- }
- assign(".:xp:.", xp, envir = self)
- },
- print = function(...) {
- if (!is.null(self$.class_title)) {
- # Allow subclasses to override just printing the class name first
- class_title <- self$.class_title()
- } else {
- class_title <- class(self)[[1]]
- }
- cat(class_title, "\n", sep = "")
- if (!is.null(self$ToString)){
- cat(self$ToString(), "\n", sep = "")
- }
- invisible(self)
- },
-
- invalidate = function() {
- assign(".:xp:.", NULL, envir = self)
- }
- )
-)
-
-#' @export
-`!=.ArrowObject` <- function(lhs, rhs) !(lhs == rhs)
-
-#' @export
-`==.ArrowObject` <- function(x, y) {
- x$Equals(y)
-}
-
-#' @export
-all.equal.ArrowObject <- function(target, current, ..., check.attributes = TRUE) {
- target$Equals(current, check_metadata = check.attributes)
-}
diff --git a/r/R/arrow-tabular.R b/r/R/arrow-tabular.R
deleted file mode 100644
index f321116..0000000
--- a/r/R/arrow-tabular.R
+++ /dev/null
@@ -1,220 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-datum.R
-
-# Base class for RecordBatch and Table for S3 method dispatch only.
-# Does not exist in C++ class hierarchy
-ArrowTabular <- R6Class("ArrowTabular", inherit = ArrowObject,
- public = list(
- ToString = function() ToString_tabular(self),
- Take = function(i) {
- if (is.numeric(i)) {
- i <- as.integer(i)
- }
- if (is.integer(i)) {
- i <- Array$create(i)
- }
- assert_that(is.Array(i))
- call_function("take", self, i)
- },
- Filter = function(i, keep_na = TRUE) {
- if (is.logical(i)) {
- i <- Array$create(i)
- }
- assert_that(is.Array(i, "bool"))
- call_function("filter", self, i, options = list(keep_na = keep_na))
- },
- SortIndices = function(names, descending = FALSE) {
- assert_that(is.character(names))
- assert_that(length(names) > 0)
- assert_that(!any(is.na(names)))
- if (length(descending) == 1L) {
- descending <- rep_len(descending, length(names))
- }
- assert_that(is.logical(descending))
- assert_that(identical(length(names), length(descending)))
- assert_that(!any(is.na(descending)))
- call_function(
- "sort_indices",
- self,
- # cpp11 does not support logical vectors so convert to integer
- options = list(names = names, orders = as.integer(descending))
- )
- }
- )
-)
-
-#' @export
-as.data.frame.ArrowTabular <- function(x, row.names = NULL, optional = FALSE, ...) {
- tryCatch(
- df <- x$to_data_frame(),
- error = handle_embedded_nul_error
- )
- if (!is.null(r_metadata <- x$metadata$r)) {
- df <- apply_arrow_r_metadata(df, .unserialize_arrow_r_metadata(r_metadata))
- }
- df
-}
-
-#' @export
-`names<-.ArrowTabular` <- function(x, value) x$RenameColumns(value)
-
-#' @importFrom methods as
-#' @export
-`[.ArrowTabular` <- function(x, i, j, ..., drop = FALSE) {
- if (nargs() == 2L) {
- # List-like column extraction (x[i])
- return(x[, i])
- }
- if (!missing(j)) {
- # Selecting columns is cheaper than filtering rows, so do it first.
- # That way, if we're filtering too, we have fewer arrays to filter/slice/take
- if (is.character(j)) {
- j_new <- match(j, names(x))
- if (any(is.na(j_new))) {
- stop("Column not found: ", oxford_paste(j[is.na(j_new)]), call. = FALSE)
- }
- j <- j_new
- }
- if (is_integerish(j)) {
- if (any(is.na(j))) {
- stop("Column indices cannot be NA", call. = FALSE)
- }
- if (length(j) && all(j < 0)) {
- # in R, negative j means "everything but j"
- j <- setdiff(seq_len(x$num_columns), -1 * j)
- }
- x <- x$SelectColumns(as.integer(j) - 1L)
- }
-
- if (drop && ncol(x) == 1L) {
- x <- x$column(0)
- }
- }
- if (!missing(i)) {
- x <- filter_rows(x, i, ...)
- }
- x
-}
-
-#' @export
-`[[.ArrowTabular` <- function(x, i, ...) {
- if (is.character(i)) {
- x$GetColumnByName(i)
- } else if (is.numeric(i)) {
- x$column(i - 1)
- } else {
- stop("'i' must be character or numeric, not ", class(i), call. = FALSE)
- }
-}
-
-#' @export
-`$.ArrowTabular` <- function(x, name, ...) {
- assert_that(is.string(name))
- if (name %in% ls(x)) {
- get(name, x)
- } else {
- x$GetColumnByName(name)
- }
-}
-
-#' @export
-`[[<-.ArrowTabular` <- function(x, i, value) {
- if (!is.character(i) & !is.numeric(i)) {
- stop("'i' must be character or numeric, not ", class(i), call. = FALSE)
- }
- assert_that(length(i) == 1, !is.na(i))
-
- if (is.null(value)) {
- if (is.character(i)) {
- i <- match(i, names(x))
- }
- x <- x$RemoveColumn(i - 1L)
- } else {
- if (!is.character(i)) {
- # get or create a/the column name
- if (i <= x$num_columns) {
- i <- names(x)[i]
- } else {
- i <- as.character(i)
- }
- }
-
- # auto-magic recycling on non-ArrowObjects
- if (!inherits(value, "ArrowObject")) {
- value <- vctrs::vec_recycle(value, x$num_rows)
- }
-
- # construct the field
- if (inherits(x, "RecordBatch") && !inherits(value, "Array")) {
- value <- Array$create(value)
- } else if (inherits(x, "Table") && !inherits(value, "ChunkedArray")) {
- value <- ChunkedArray$create(value)
- }
- new_field <- field(i, value$type)
-
- if (i %in% names(x)) {
- i <- match(i, names(x)) - 1L
- x <- x$SetColumn(i, new_field, value)
- } else {
- i <- x$num_columns
- x <- x$AddColumn(i, new_field, value)
- }
- }
- x
-}
-
-#' @export
-`$<-.ArrowTabular` <- function(x, i, value) {
- assert_that(is.string(i))
- # We need to check if `i` is in names in case it is an active binding (e.g.
- # `metadata`, in which case we use assign to change the active binding instead
- # of the column in the table)
- if (i %in% ls(x)) {
- assign(i, value, x)
- } else {
- x[[i]] <- value
- }
- x
-}
-
-#' @export
-dim.ArrowTabular <- function(x) c(x$num_rows, x$num_columns)
-
-#' @export
-as.list.ArrowTabular <- function(x, ...) as.list(as.data.frame(x, ...))
-
-#' @export
-row.names.ArrowTabular <- function(x) as.character(seq_len(nrow(x)))
-
-#' @export
-dimnames.ArrowTabular <- function(x) list(row.names(x), names(x))
-
-#' @export
-head.ArrowTabular <- head.ArrowDatum
-
-#' @export
-tail.ArrowTabular <- tail.ArrowDatum
-
-ToString_tabular <- function(x, ...) {
- # Generic to work with both RecordBatch and Table
- sch <- unlist(strsplit(x$schema$ToString(), "\n"))
- sch <- sub("(.*): (.*)", "$\\1 <\\2>", sch)
- dims <- sprintf("%s rows x %s columns", nrow(x), ncol(x))
- paste(c(dims, sch), collapse = "\n")
-}
diff --git a/r/R/arrowExports.R b/r/R/arrowExports.R
deleted file mode 100644
index 51cdcf8..0000000
--- a/r/R/arrowExports.R
+++ /dev/null
@@ -1,1668 +0,0 @@
-# Generated by using data-raw/codegen.R -> do not edit by hand
-
-Array__Slice1 <- function(array, offset){
- .Call(`_arrow_Array__Slice1`, array, offset)
-}
-
-Array__Slice2 <- function(array, offset, length){
- .Call(`_arrow_Array__Slice2`, array, offset, length)
-}
-
-Array__IsNull <- function(x, i){
- .Call(`_arrow_Array__IsNull`, x, i)
-}
-
-Array__IsValid <- function(x, i){
- .Call(`_arrow_Array__IsValid`, x, i)
-}
-
-Array__length <- function(x){
- .Call(`_arrow_Array__length`, x)
-}
-
-Array__offset <- function(x){
- .Call(`_arrow_Array__offset`, x)
-}
-
-Array__null_count <- function(x){
- .Call(`_arrow_Array__null_count`, x)
-}
-
-Array__type <- function(x){
- .Call(`_arrow_Array__type`, x)
-}
-
-Array__ToString <- function(x){
- .Call(`_arrow_Array__ToString`, x)
-}
-
-Array__type_id <- function(x){
- .Call(`_arrow_Array__type_id`, x)
-}
-
-Array__Equals <- function(lhs, rhs){
- .Call(`_arrow_Array__Equals`, lhs, rhs)
-}
-
-Array__ApproxEquals <- function(lhs, rhs){
- .Call(`_arrow_Array__ApproxEquals`, lhs, rhs)
-}
-
-Array__Diff <- function(lhs, rhs){
- .Call(`_arrow_Array__Diff`, lhs, rhs)
-}
-
-Array__data <- function(array){
- .Call(`_arrow_Array__data`, array)
-}
-
-Array__RangeEquals <- function(self, other, start_idx, end_idx, other_start_idx){
- .Call(`_arrow_Array__RangeEquals`, self, other, start_idx, end_idx, other_start_idx)
-}
-
-Array__View <- function(array, type){
- .Call(`_arrow_Array__View`, array, type)
-}
-
-Array__Validate <- function(array){
- invisible(.Call(`_arrow_Array__Validate`, array))
-}
-
-DictionaryArray__indices <- function(array){
- .Call(`_arrow_DictionaryArray__indices`, array)
-}
-
-DictionaryArray__dictionary <- function(array){
- .Call(`_arrow_DictionaryArray__dictionary`, array)
-}
-
-StructArray__field <- function(array, i){
- .Call(`_arrow_StructArray__field`, array, i)
-}
-
-StructArray__GetFieldByName <- function(array, name){
- .Call(`_arrow_StructArray__GetFieldByName`, array, name)
-}
-
-StructArray__Flatten <- function(array){
- .Call(`_arrow_StructArray__Flatten`, array)
-}
-
-ListArray__value_type <- function(array){
- .Call(`_arrow_ListArray__value_type`, array)
-}
-
-LargeListArray__value_type <- function(array){
- .Call(`_arrow_LargeListArray__value_type`, array)
-}
-
-ListArray__values <- function(array){
- .Call(`_arrow_ListArray__values`, array)
-}
-
-LargeListArray__values <- function(array){
- .Call(`_arrow_LargeListArray__values`, array)
-}
-
-ListArray__value_length <- function(array, i){
- .Call(`_arrow_ListArray__value_length`, array, i)
-}
-
-LargeListArray__value_length <- function(array, i){
- .Call(`_arrow_LargeListArray__value_length`, array, i)
-}
-
-FixedSizeListArray__value_length <- function(array, i){
- .Call(`_arrow_FixedSizeListArray__value_length`, array, i)
-}
-
-ListArray__value_offset <- function(array, i){
- .Call(`_arrow_ListArray__value_offset`, array, i)
-}
-
-LargeListArray__value_offset <- function(array, i){
- .Call(`_arrow_LargeListArray__value_offset`, array, i)
-}
-
-FixedSizeListArray__value_offset <- function(array, i){
- .Call(`_arrow_FixedSizeListArray__value_offset`, array, i)
-}
-
-ListArray__raw_value_offsets <- function(array){
- .Call(`_arrow_ListArray__raw_value_offsets`, array)
-}
-
-LargeListArray__raw_value_offsets <- function(array){
- .Call(`_arrow_LargeListArray__raw_value_offsets`, array)
-}
-
-Array__as_vector <- function(array){
- .Call(`_arrow_Array__as_vector`, array)
-}
-
-ChunkedArray__as_vector <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__as_vector`, chunked_array)
-}
-
-RecordBatch__to_dataframe <- function(batch, use_threads){
- .Call(`_arrow_RecordBatch__to_dataframe`, batch, use_threads)
-}
-
-Table__to_dataframe <- function(table, use_threads){
- .Call(`_arrow_Table__to_dataframe`, table, use_threads)
-}
-
-ArrayData__get_type <- function(x){
- .Call(`_arrow_ArrayData__get_type`, x)
-}
-
-ArrayData__get_length <- function(x){
- .Call(`_arrow_ArrayData__get_length`, x)
-}
-
-ArrayData__get_null_count <- function(x){
- .Call(`_arrow_ArrayData__get_null_count`, x)
-}
-
-ArrayData__get_offset <- function(x){
- .Call(`_arrow_ArrayData__get_offset`, x)
-}
-
-ArrayData__buffers <- function(x){
- .Call(`_arrow_ArrayData__buffers`, x)
-}
-
-Buffer__is_mutable <- function(buffer){
- .Call(`_arrow_Buffer__is_mutable`, buffer)
-}
-
-Buffer__ZeroPadding <- function(buffer){
- invisible(.Call(`_arrow_Buffer__ZeroPadding`, buffer))
-}
-
-Buffer__capacity <- function(buffer){
- .Call(`_arrow_Buffer__capacity`, buffer)
-}
-
-Buffer__size <- function(buffer){
- .Call(`_arrow_Buffer__size`, buffer)
-}
-
-r___RBuffer__initialize <- function(x){
- .Call(`_arrow_r___RBuffer__initialize`, x)
-}
-
-Buffer__data <- function(buffer){
- .Call(`_arrow_Buffer__data`, buffer)
-}
-
-Buffer__Equals <- function(x, y){
- .Call(`_arrow_Buffer__Equals`, x, y)
-}
-
-ChunkedArray__length <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__length`, chunked_array)
-}
-
-ChunkedArray__null_count <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__null_count`, chunked_array)
-}
-
-ChunkedArray__num_chunks <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__num_chunks`, chunked_array)
-}
-
-ChunkedArray__chunk <- function(chunked_array, i){
- .Call(`_arrow_ChunkedArray__chunk`, chunked_array, i)
-}
-
-ChunkedArray__chunks <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__chunks`, chunked_array)
-}
-
-ChunkedArray__type <- function(chunked_array){
- .Call(`_arrow_ChunkedArray__type`, chunked_array)
-}
-
-ChunkedArray__Slice1 <- function(chunked_array, offset){
- .Call(`_arrow_ChunkedArray__Slice1`, chunked_array, offset)
-}
-
-ChunkedArray__Slice2 <- function(chunked_array, offset, length){
- .Call(`_arrow_ChunkedArray__Slice2`, chunked_array, offset, length)
-}
-
-ChunkedArray__View <- function(array, type){
- .Call(`_arrow_ChunkedArray__View`, array, type)
-}
-
-ChunkedArray__Validate <- function(chunked_array){
- invisible(.Call(`_arrow_ChunkedArray__Validate`, chunked_array))
-}
-
-ChunkedArray__Equals <- function(x, y){
- .Call(`_arrow_ChunkedArray__Equals`, x, y)
-}
-
-ChunkedArray__ToString <- function(x){
- .Call(`_arrow_ChunkedArray__ToString`, x)
-}
-
-ChunkedArray__from_list <- function(chunks, s_type){
- .Call(`_arrow_ChunkedArray__from_list`, chunks, s_type)
-}
-
-util___Codec__Create <- function(codec, compression_level){
- .Call(`_arrow_util___Codec__Create`, codec, compression_level)
-}
-
-util___Codec__name <- function(codec){
- .Call(`_arrow_util___Codec__name`, codec)
-}
-
-util___Codec__IsAvailable <- function(codec){
- .Call(`_arrow_util___Codec__IsAvailable`, codec)
-}
-
-io___CompressedOutputStream__Make <- function(codec, raw){
- .Call(`_arrow_io___CompressedOutputStream__Make`, codec, raw)
-}
-
-io___CompressedInputStream__Make <- function(codec, raw){
- .Call(`_arrow_io___CompressedInputStream__Make`, codec, raw)
-}
-
-RecordBatch__cast <- function(batch, schema, options){
- .Call(`_arrow_RecordBatch__cast`, batch, schema, options)
-}
-
-Table__cast <- function(table, schema, options){
- .Call(`_arrow_Table__cast`, table, schema, options)
-}
-
-compute__CallFunction <- function(func_name, args, options){
- .Call(`_arrow_compute__CallFunction`, func_name, args, options)
-}
-
-compute__GroupBy <- function(arguments, keys, options){
- .Call(`_arrow_compute__GroupBy`, arguments, keys, options)
-}
-
-compute__GetFunctionNames <- function(){
- .Call(`_arrow_compute__GetFunctionNames`)
-}
-
-csv___ReadOptions__initialize <- function(options){
- .Call(`_arrow_csv___ReadOptions__initialize`, options)
-}
-
-csv___ParseOptions__initialize <- function(options){
- .Call(`_arrow_csv___ParseOptions__initialize`, options)
-}
-
-csv___ReadOptions__column_names <- function(options){
- .Call(`_arrow_csv___ReadOptions__column_names`, options)
-}
-
-csv___ConvertOptions__initialize <- function(options){
- .Call(`_arrow_csv___ConvertOptions__initialize`, options)
-}
-
-csv___TableReader__Make <- function(input, read_options, parse_options, convert_options){
- .Call(`_arrow_csv___TableReader__Make`, input, read_options, parse_options, convert_options)
-}
-
-csv___TableReader__Read <- function(table_reader){
- .Call(`_arrow_csv___TableReader__Read`, table_reader)
-}
-
-TimestampParser__kind <- function(parser){
- .Call(`_arrow_TimestampParser__kind`, parser)
-}
-
-TimestampParser__format <- function(parser){
- .Call(`_arrow_TimestampParser__format`, parser)
-}
-
-TimestampParser__MakeStrptime <- function(format){
- .Call(`_arrow_TimestampParser__MakeStrptime`, format)
-}
-
-TimestampParser__MakeISO8601 <- function(){
- .Call(`_arrow_TimestampParser__MakeISO8601`)
-}
-
-dataset___Dataset__NewScan <- function(ds){
- .Call(`_arrow_dataset___Dataset__NewScan`, ds)
-}
-
-dataset___Dataset__schema <- function(dataset){
- .Call(`_arrow_dataset___Dataset__schema`, dataset)
-}
-
-dataset___Dataset__type_name <- function(dataset){
- .Call(`_arrow_dataset___Dataset__type_name`, dataset)
-}
-
-dataset___Dataset__ReplaceSchema <- function(dataset, schm){
- .Call(`_arrow_dataset___Dataset__ReplaceSchema`, dataset, schm)
-}
-
-dataset___UnionDataset__create <- function(datasets, schm){
- .Call(`_arrow_dataset___UnionDataset__create`, datasets, schm)
-}
-
-dataset___InMemoryDataset__create <- function(table){
- .Call(`_arrow_dataset___InMemoryDataset__create`, table)
-}
-
-dataset___UnionDataset__children <- function(ds){
- .Call(`_arrow_dataset___UnionDataset__children`, ds)
-}
-
-dataset___FileSystemDataset__format <- function(dataset){
- .Call(`_arrow_dataset___FileSystemDataset__format`, dataset)
-}
-
-dataset___FileSystemDataset__filesystem <- function(dataset){
- .Call(`_arrow_dataset___FileSystemDataset__filesystem`, dataset)
-}
-
-dataset___FileSystemDataset__files <- function(dataset){
- .Call(`_arrow_dataset___FileSystemDataset__files`, dataset)
-}
-
-dataset___DatasetFactory__Finish1 <- function(factory, unify_schemas){
- .Call(`_arrow_dataset___DatasetFactory__Finish1`, factory, unify_schemas)
-}
-
-dataset___DatasetFactory__Finish2 <- function(factory, schema){
- .Call(`_arrow_dataset___DatasetFactory__Finish2`, factory, schema)
-}
-
-dataset___DatasetFactory__Inspect <- function(factory, unify_schemas){
- .Call(`_arrow_dataset___DatasetFactory__Inspect`, factory, unify_schemas)
-}
-
-dataset___UnionDatasetFactory__Make <- function(children){
- .Call(`_arrow_dataset___UnionDatasetFactory__Make`, children)
-}
-
-dataset___FileSystemDatasetFactory__Make0 <- function(fs, paths, format){
- .Call(`_arrow_dataset___FileSystemDatasetFactory__Make0`, fs, paths, format)
-}
-
-dataset___FileSystemDatasetFactory__Make2 <- function(fs, selector, format, partitioning){
- .Call(`_arrow_dataset___FileSystemDatasetFactory__Make2`, fs, selector, format, partitioning)
-}
-
-dataset___FileSystemDatasetFactory__Make1 <- function(fs, selector, format){
- .Call(`_arrow_dataset___FileSystemDatasetFactory__Make1`, fs, selector, format)
-}
-
-dataset___FileSystemDatasetFactory__Make3 <- function(fs, selector, format, factory){
- .Call(`_arrow_dataset___FileSystemDatasetFactory__Make3`, fs, selector, format, factory)
-}
-
-dataset___FileFormat__type_name <- function(format){
- .Call(`_arrow_dataset___FileFormat__type_name`, format)
-}
-
-dataset___FileFormat__DefaultWriteOptions <- function(fmt){
- .Call(`_arrow_dataset___FileFormat__DefaultWriteOptions`, fmt)
-}
-
-dataset___ParquetFileFormat__Make <- function(options, dict_columns){
- .Call(`_arrow_dataset___ParquetFileFormat__Make`, options, dict_columns)
-}
-
-dataset___FileWriteOptions__type_name <- function(options){
- .Call(`_arrow_dataset___FileWriteOptions__type_name`, options)
-}
-
-dataset___ParquetFileWriteOptions__update <- function(options, writer_props, arrow_writer_props){
- invisible(.Call(`_arrow_dataset___ParquetFileWriteOptions__update`, options, writer_props, arrow_writer_props))
-}
-
-dataset___IpcFileWriteOptions__update2 <- function(ipc_options, use_legacy_format, codec, metadata_version){
- invisible(.Call(`_arrow_dataset___IpcFileWriteOptions__update2`, ipc_options, use_legacy_format, codec, metadata_version))
-}
-
-dataset___IpcFileWriteOptions__update1 <- function(ipc_options, use_legacy_format, metadata_version){
- invisible(.Call(`_arrow_dataset___IpcFileWriteOptions__update1`, ipc_options, use_legacy_format, metadata_version))
-}
-
-dataset___IpcFileFormat__Make <- function(){
- .Call(`_arrow_dataset___IpcFileFormat__Make`)
-}
-
-dataset___CsvFileFormat__Make <- function(parse_options, convert_options, read_options){
- .Call(`_arrow_dataset___CsvFileFormat__Make`, parse_options, convert_options, read_options)
-}
-
-dataset___FragmentScanOptions__type_name <- function(fragment_scan_options){
- .Call(`_arrow_dataset___FragmentScanOptions__type_name`, fragment_scan_options)
-}
-
-dataset___CsvFragmentScanOptions__Make <- function(convert_options, read_options){
- .Call(`_arrow_dataset___CsvFragmentScanOptions__Make`, convert_options, read_options)
-}
-
-dataset___ParquetFragmentScanOptions__Make <- function(use_buffered_stream, buffer_size, pre_buffer){
- .Call(`_arrow_dataset___ParquetFragmentScanOptions__Make`, use_buffered_stream, buffer_size, pre_buffer)
-}
-
-dataset___DirectoryPartitioning <- function(schm){
- .Call(`_arrow_dataset___DirectoryPartitioning`, schm)
-}
-
-dataset___DirectoryPartitioning__MakeFactory <- function(field_names){
- .Call(`_arrow_dataset___DirectoryPartitioning__MakeFactory`, field_names)
-}
-
-dataset___HivePartitioning <- function(schm, null_fallback){
- .Call(`_arrow_dataset___HivePartitioning`, schm, null_fallback)
-}
-
-dataset___HivePartitioning__MakeFactory <- function(null_fallback){
- .Call(`_arrow_dataset___HivePartitioning__MakeFactory`, null_fallback)
-}
-
-dataset___ScannerBuilder__ProjectNames <- function(sb, cols){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__ProjectNames`, sb, cols))
-}
-
-dataset___ScannerBuilder__ProjectExprs <- function(sb, exprs, names){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__ProjectExprs`, sb, exprs, names))
-}
-
-dataset___ScannerBuilder__Filter <- function(sb, expr){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__Filter`, sb, expr))
-}
-
-dataset___ScannerBuilder__UseThreads <- function(sb, threads){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__UseThreads`, sb, threads))
-}
-
-dataset___ScannerBuilder__BatchSize <- function(sb, batch_size){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__BatchSize`, sb, batch_size))
-}
-
-dataset___ScannerBuilder__FragmentScanOptions <- function(sb, options){
- invisible(.Call(`_arrow_dataset___ScannerBuilder__FragmentScanOptions`, sb, options))
-}
-
-dataset___ScannerBuilder__schema <- function(sb){
- .Call(`_arrow_dataset___ScannerBuilder__schema`, sb)
-}
-
-dataset___ScannerBuilder__Finish <- function(sb){
- .Call(`_arrow_dataset___ScannerBuilder__Finish`, sb)
-}
-
-dataset___Scanner__ToTable <- function(scanner){
- .Call(`_arrow_dataset___Scanner__ToTable`, scanner)
-}
-
-dataset___Scanner__ScanBatches <- function(scanner){
- .Call(`_arrow_dataset___Scanner__ScanBatches`, scanner)
-}
-
-dataset___Scanner__head <- function(scanner, n){
- .Call(`_arrow_dataset___Scanner__head`, scanner, n)
-}
-
-dataset___Scanner__schema <- function(sc){
- .Call(`_arrow_dataset___Scanner__schema`, sc)
-}
-
-dataset___ScanTask__get_batches <- function(scan_task){
- .Call(`_arrow_dataset___ScanTask__get_batches`, scan_task)
-}
-
-dataset___Dataset__Write <- function(file_write_options, filesystem, base_dir, partitioning, basename_template, scanner){
- invisible(.Call(`_arrow_dataset___Dataset__Write`, file_write_options, filesystem, base_dir, partitioning, basename_template, scanner))
-}
-
-dataset___Scanner__TakeRows <- function(scanner, indices){
- .Call(`_arrow_dataset___Scanner__TakeRows`, scanner, indices)
-}
-
-Int8__initialize <- function(){
- .Call(`_arrow_Int8__initialize`)
-}
-
-Int16__initialize <- function(){
- .Call(`_arrow_Int16__initialize`)
-}
-
-Int32__initialize <- function(){
- .Call(`_arrow_Int32__initialize`)
-}
-
-Int64__initialize <- function(){
- .Call(`_arrow_Int64__initialize`)
-}
-
-UInt8__initialize <- function(){
- .Call(`_arrow_UInt8__initialize`)
-}
-
-UInt16__initialize <- function(){
- .Call(`_arrow_UInt16__initialize`)
-}
-
-UInt32__initialize <- function(){
- .Call(`_arrow_UInt32__initialize`)
-}
-
-UInt64__initialize <- function(){
- .Call(`_arrow_UInt64__initialize`)
-}
-
-Float16__initialize <- function(){
- .Call(`_arrow_Float16__initialize`)
-}
-
-Float32__initialize <- function(){
- .Call(`_arrow_Float32__initialize`)
-}
-
-Float64__initialize <- function(){
- .Call(`_arrow_Float64__initialize`)
-}
-
-Boolean__initialize <- function(){
- .Call(`_arrow_Boolean__initialize`)
-}
-
-Utf8__initialize <- function(){
- .Call(`_arrow_Utf8__initialize`)
-}
-
-LargeUtf8__initialize <- function(){
- .Call(`_arrow_LargeUtf8__initialize`)
-}
-
-Binary__initialize <- function(){
- .Call(`_arrow_Binary__initialize`)
-}
-
-LargeBinary__initialize <- function(){
- .Call(`_arrow_LargeBinary__initialize`)
-}
-
-Date32__initialize <- function(){
- .Call(`_arrow_Date32__initialize`)
-}
-
-Date64__initialize <- function(){
- .Call(`_arrow_Date64__initialize`)
-}
-
-Null__initialize <- function(){
- .Call(`_arrow_Null__initialize`)
-}
-
-Decimal128Type__initialize <- function(precision, scale){
- .Call(`_arrow_Decimal128Type__initialize`, precision, scale)
-}
-
-FixedSizeBinary__initialize <- function(byte_width){
- .Call(`_arrow_FixedSizeBinary__initialize`, byte_width)
-}
-
-Timestamp__initialize <- function(unit, timezone){
- .Call(`_arrow_Timestamp__initialize`, unit, timezone)
-}
-
-Time32__initialize <- function(unit){
- .Call(`_arrow_Time32__initialize`, unit)
-}
-
-Time64__initialize <- function(unit){
- .Call(`_arrow_Time64__initialize`, unit)
-}
-
-list__ <- function(x){
- .Call(`_arrow_list__`, x)
-}
-
-large_list__ <- function(x){
- .Call(`_arrow_large_list__`, x)
-}
-
-fixed_size_list__ <- function(x, list_size){
- .Call(`_arrow_fixed_size_list__`, x, list_size)
-}
-
-struct__ <- function(fields){
- .Call(`_arrow_struct__`, fields)
-}
-
-DataType__ToString <- function(type){
- .Call(`_arrow_DataType__ToString`, type)
-}
-
-DataType__name <- function(type){
- .Call(`_arrow_DataType__name`, type)
-}
-
-DataType__Equals <- function(lhs, rhs){
- .Call(`_arrow_DataType__Equals`, lhs, rhs)
-}
-
-DataType__num_fields <- function(type){
- .Call(`_arrow_DataType__num_fields`, type)
-}
-
-DataType__fields <- function(type){
- .Call(`_arrow_DataType__fields`, type)
-}
-
-DataType__id <- function(type){
- .Call(`_arrow_DataType__id`, type)
-}
-
-ListType__ToString <- function(type){
- .Call(`_arrow_ListType__ToString`, type)
-}
-
-FixedWidthType__bit_width <- function(type){
- .Call(`_arrow_FixedWidthType__bit_width`, type)
-}
-
-DateType__unit <- function(type){
- .Call(`_arrow_DateType__unit`, type)
-}
-
-TimeType__unit <- function(type){
- .Call(`_arrow_TimeType__unit`, type)
-}
-
-DecimalType__precision <- function(type){
- .Call(`_arrow_DecimalType__precision`, type)
-}
-
-DecimalType__scale <- function(type){
- .Call(`_arrow_DecimalType__scale`, type)
-}
-
-TimestampType__timezone <- function(type){
- .Call(`_arrow_TimestampType__timezone`, type)
-}
-
-TimestampType__unit <- function(type){
- .Call(`_arrow_TimestampType__unit`, type)
-}
-
-DictionaryType__initialize <- function(index_type, value_type, ordered){
- .Call(`_arrow_DictionaryType__initialize`, index_type, value_type, ordered)
-}
-
-DictionaryType__index_type <- function(type){
- .Call(`_arrow_DictionaryType__index_type`, type)
-}
-
-DictionaryType__value_type <- function(type){
- .Call(`_arrow_DictionaryType__value_type`, type)
-}
-
-DictionaryType__name <- function(type){
- .Call(`_arrow_DictionaryType__name`, type)
-}
-
-DictionaryType__ordered <- function(type){
- .Call(`_arrow_DictionaryType__ordered`, type)
-}
-
-StructType__GetFieldByName <- function(type, name){
- .Call(`_arrow_StructType__GetFieldByName`, type, name)
-}
-
-StructType__GetFieldIndex <- function(type, name){
- .Call(`_arrow_StructType__GetFieldIndex`, type, name)
-}
-
-StructType__field_names <- function(type){
- .Call(`_arrow_StructType__field_names`, type)
-}
-
-ListType__value_field <- function(type){
- .Call(`_arrow_ListType__value_field`, type)
-}
-
-ListType__value_type <- function(type){
- .Call(`_arrow_ListType__value_type`, type)
-}
-
-LargeListType__value_field <- function(type){
- .Call(`_arrow_LargeListType__value_field`, type)
-}
-
-LargeListType__value_type <- function(type){
- .Call(`_arrow_LargeListType__value_type`, type)
-}
-
-FixedSizeListType__value_field <- function(type){
- .Call(`_arrow_FixedSizeListType__value_field`, type)
-}
-
-FixedSizeListType__value_type <- function(type){
- .Call(`_arrow_FixedSizeListType__value_type`, type)
-}
-
-FixedSizeListType__list_size <- function(type){
- .Call(`_arrow_FixedSizeListType__list_size`, type)
-}
-
-dataset___expr__call <- function(func_name, argument_list, options){
- .Call(`_arrow_dataset___expr__call`, func_name, argument_list, options)
-}
-
-dataset___expr__field_ref <- function(name){
- .Call(`_arrow_dataset___expr__field_ref`, name)
-}
-
-dataset___expr__get_field_ref_name <- function(ref){
- .Call(`_arrow_dataset___expr__get_field_ref_name`, ref)
-}
-
-dataset___expr__scalar <- function(x){
- .Call(`_arrow_dataset___expr__scalar`, x)
-}
-
-dataset___expr__ToString <- function(x){
- .Call(`_arrow_dataset___expr__ToString`, x)
-}
-
-ipc___WriteFeather__Table <- function(stream, table, version, chunk_size, compression, compression_level){
- invisible(.Call(`_arrow_ipc___WriteFeather__Table`, stream, table, version, chunk_size, compression, compression_level))
-}
-
-ipc___feather___Reader__version <- function(reader){
- .Call(`_arrow_ipc___feather___Reader__version`, reader)
-}
-
-ipc___feather___Reader__Read <- function(reader, columns){
- .Call(`_arrow_ipc___feather___Reader__Read`, reader, columns)
-}
-
-ipc___feather___Reader__Open <- function(stream){
- .Call(`_arrow_ipc___feather___Reader__Open`, stream)
-}
-
-ipc___feather___Reader__schema <- function(reader){
- .Call(`_arrow_ipc___feather___Reader__schema`, reader)
-}
-
-Field__initialize <- function(name, field, nullable){
- .Call(`_arrow_Field__initialize`, name, field, nullable)
-}
-
-Field__ToString <- function(field){
- .Call(`_arrow_Field__ToString`, field)
-}
-
-Field__name <- function(field){
- .Call(`_arrow_Field__name`, field)
-}
-
-Field__Equals <- function(field, other){
- .Call(`_arrow_Field__Equals`, field, other)
-}
-
-Field__nullable <- function(field){
- .Call(`_arrow_Field__nullable`, field)
-}
-
-Field__type <- function(field){
- .Call(`_arrow_Field__type`, field)
-}
-
-fs___FileInfo__type <- function(x){
- .Call(`_arrow_fs___FileInfo__type`, x)
-}
-
-fs___FileInfo__set_type <- function(x, type){
- invisible(.Call(`_arrow_fs___FileInfo__set_type`, x, type))
-}
-
-fs___FileInfo__path <- function(x){
- .Call(`_arrow_fs___FileInfo__path`, x)
-}
-
-fs___FileInfo__set_path <- function(x, path){
- invisible(.Call(`_arrow_fs___FileInfo__set_path`, x, path))
-}
-
-fs___FileInfo__size <- function(x){
- .Call(`_arrow_fs___FileInfo__size`, x)
-}
-
-fs___FileInfo__set_size <- function(x, size){
- invisible(.Call(`_arrow_fs___FileInfo__set_size`, x, size))
-}
-
-fs___FileInfo__base_name <- function(x){
- .Call(`_arrow_fs___FileInfo__base_name`, x)
-}
-
-fs___FileInfo__extension <- function(x){
- .Call(`_arrow_fs___FileInfo__extension`, x)
-}
-
-fs___FileInfo__mtime <- function(x){
- .Call(`_arrow_fs___FileInfo__mtime`, x)
-}
-
-fs___FileInfo__set_mtime <- function(x, time){
- invisible(.Call(`_arrow_fs___FileInfo__set_mtime`, x, time))
-}
-
-fs___FileSelector__base_dir <- function(selector){
- .Call(`_arrow_fs___FileSelector__base_dir`, selector)
-}
-
-fs___FileSelector__allow_not_found <- function(selector){
- .Call(`_arrow_fs___FileSelector__allow_not_found`, selector)
-}
-
-fs___FileSelector__recursive <- function(selector){
- .Call(`_arrow_fs___FileSelector__recursive`, selector)
-}
-
-fs___FileSelector__create <- function(base_dir, allow_not_found, recursive){
- .Call(`_arrow_fs___FileSelector__create`, base_dir, allow_not_found, recursive)
-}
-
-fs___FileSystem__GetTargetInfos_Paths <- function(file_system, paths){
- .Call(`_arrow_fs___FileSystem__GetTargetInfos_Paths`, file_system, paths)
-}
-
-fs___FileSystem__GetTargetInfos_FileSelector <- function(file_system, selector){
- .Call(`_arrow_fs___FileSystem__GetTargetInfos_FileSelector`, file_system, selector)
-}
-
-fs___FileSystem__CreateDir <- function(file_system, path, recursive){
- invisible(.Call(`_arrow_fs___FileSystem__CreateDir`, file_system, path, recursive))
-}
-
-fs___FileSystem__DeleteDir <- function(file_system, path){
- invisible(.Call(`_arrow_fs___FileSystem__DeleteDir`, file_system, path))
-}
-
-fs___FileSystem__DeleteDirContents <- function(file_system, path){
- invisible(.Call(`_arrow_fs___FileSystem__DeleteDirContents`, file_system, path))
-}
-
-fs___FileSystem__DeleteFile <- function(file_system, path){
- invisible(.Call(`_arrow_fs___FileSystem__DeleteFile`, file_system, path))
-}
-
-fs___FileSystem__DeleteFiles <- function(file_system, paths){
- invisible(.Call(`_arrow_fs___FileSystem__DeleteFiles`, file_system, paths))
-}
-
-fs___FileSystem__Move <- function(file_system, src, dest){
- invisible(.Call(`_arrow_fs___FileSystem__Move`, file_system, src, dest))
-}
-
-fs___FileSystem__CopyFile <- function(file_system, src, dest){
- invisible(.Call(`_arrow_fs___FileSystem__CopyFile`, file_system, src, dest))
-}
-
-fs___FileSystem__OpenInputStream <- function(file_system, path){
- .Call(`_arrow_fs___FileSystem__OpenInputStream`, file_system, path)
-}
-
-fs___FileSystem__OpenInputFile <- function(file_system, path){
- .Call(`_arrow_fs___FileSystem__OpenInputFile`, file_system, path)
-}
-
-fs___FileSystem__OpenOutputStream <- function(file_system, path){
- .Call(`_arrow_fs___FileSystem__OpenOutputStream`, file_system, path)
-}
-
-fs___FileSystem__OpenAppendStream <- function(file_system, path){
- .Call(`_arrow_fs___FileSystem__OpenAppendStream`, file_system, path)
-}
-
-fs___FileSystem__type_name <- function(file_system){
- .Call(`_arrow_fs___FileSystem__type_name`, file_system)
-}
-
-fs___LocalFileSystem__create <- function(){
- .Call(`_arrow_fs___LocalFileSystem__create`)
-}
-
-fs___SubTreeFileSystem__create <- function(base_path, base_fs){
- .Call(`_arrow_fs___SubTreeFileSystem__create`, base_path, base_fs)
-}
-
-fs___SubTreeFileSystem__base_fs <- function(file_system){
- .Call(`_arrow_fs___SubTreeFileSystem__base_fs`, file_system)
-}
-
-fs___SubTreeFileSystem__base_path <- function(file_system){
- .Call(`_arrow_fs___SubTreeFileSystem__base_path`, file_system)
-}
-
-fs___FileSystemFromUri <- function(path){
- .Call(`_arrow_fs___FileSystemFromUri`, path)
-}
-
-fs___CopyFiles <- function(source_fs, source_sel, destination_fs, destination_base_dir, chunk_size, use_threads){
- invisible(.Call(`_arrow_fs___CopyFiles`, source_fs, source_sel, destination_fs, destination_base_dir, chunk_size, use_threads))
-}
-
-fs___S3FileSystem__create <- function(anonymous, access_key, secret_key, session_token, role_arn, session_name, external_id, load_frequency, region, endpoint_override, scheme, background_writes){
- .Call(`_arrow_fs___S3FileSystem__create`, anonymous, access_key, secret_key, session_token, role_arn, session_name, external_id, load_frequency, region, endpoint_override, scheme, background_writes)
-}
-
-fs___S3FileSystem__region <- function(fs){
- .Call(`_arrow_fs___S3FileSystem__region`, fs)
-}
-
-io___Readable__Read <- function(x, nbytes){
- .Call(`_arrow_io___Readable__Read`, x, nbytes)
-}
-
-io___InputStream__Close <- function(x){
- invisible(.Call(`_arrow_io___InputStream__Close`, x))
-}
-
-io___OutputStream__Close <- function(x){
- invisible(.Call(`_arrow_io___OutputStream__Close`, x))
-}
-
-io___RandomAccessFile__GetSize <- function(x){
- .Call(`_arrow_io___RandomAccessFile__GetSize`, x)
-}
-
-io___RandomAccessFile__supports_zero_copy <- function(x){
- .Call(`_arrow_io___RandomAccessFile__supports_zero_copy`, x)
-}
-
-io___RandomAccessFile__Seek <- function(x, position){
- invisible(.Call(`_arrow_io___RandomAccessFile__Seek`, x, position))
-}
-
-io___RandomAccessFile__Tell <- function(x){
- .Call(`_arrow_io___RandomAccessFile__Tell`, x)
-}
-
-io___RandomAccessFile__Read0 <- function(x){
- .Call(`_arrow_io___RandomAccessFile__Read0`, x)
-}
-
-io___RandomAccessFile__ReadAt <- function(x, position, nbytes){
- .Call(`_arrow_io___RandomAccessFile__ReadAt`, x, position, nbytes)
-}
-
-io___MemoryMappedFile__Create <- function(path, size){
- .Call(`_arrow_io___MemoryMappedFile__Create`, path, size)
-}
-
-io___MemoryMappedFile__Open <- function(path, mode){
- .Call(`_arrow_io___MemoryMappedFile__Open`, path, mode)
-}
-
-io___MemoryMappedFile__Resize <- function(x, size){
- invisible(.Call(`_arrow_io___MemoryMappedFile__Resize`, x, size))
-}
-
-io___ReadableFile__Open <- function(path){
- .Call(`_arrow_io___ReadableFile__Open`, path)
-}
-
-io___BufferReader__initialize <- function(buffer){
- .Call(`_arrow_io___BufferReader__initialize`, buffer)
-}
-
-io___Writable__write <- function(stream, buf){
- invisible(.Call(`_arrow_io___Writable__write`, stream, buf))
-}
-
-io___OutputStream__Tell <- function(stream){
- .Call(`_arrow_io___OutputStream__Tell`, stream)
-}
-
-io___FileOutputStream__Open <- function(path){
- .Call(`_arrow_io___FileOutputStream__Open`, path)
-}
-
-io___BufferOutputStream__Create <- function(initial_capacity){
- .Call(`_arrow_io___BufferOutputStream__Create`, initial_capacity)
-}
-
-io___BufferOutputStream__capacity <- function(stream){
- .Call(`_arrow_io___BufferOutputStream__capacity`, stream)
-}
-
-io___BufferOutputStream__Finish <- function(stream){
- .Call(`_arrow_io___BufferOutputStream__Finish`, stream)
-}
-
-io___BufferOutputStream__Tell <- function(stream){
- .Call(`_arrow_io___BufferOutputStream__Tell`, stream)
-}
-
-io___BufferOutputStream__Write <- function(stream, bytes){
- invisible(.Call(`_arrow_io___BufferOutputStream__Write`, stream, bytes))
-}
-
-json___ReadOptions__initialize <- function(use_threads, block_size){
- .Call(`_arrow_json___ReadOptions__initialize`, use_threads, block_size)
-}
-
-json___ParseOptions__initialize1 <- function(newlines_in_values){
- .Call(`_arrow_json___ParseOptions__initialize1`, newlines_in_values)
-}
-
-json___ParseOptions__initialize2 <- function(newlines_in_values, explicit_schema){
- .Call(`_arrow_json___ParseOptions__initialize2`, newlines_in_values, explicit_schema)
-}
-
-json___TableReader__Make <- function(input, read_options, parse_options){
- .Call(`_arrow_json___TableReader__Make`, input, read_options, parse_options)
-}
-
-json___TableReader__Read <- function(table_reader){
- .Call(`_arrow_json___TableReader__Read`, table_reader)
-}
-
-MemoryPool__default <- function(){
- .Call(`_arrow_MemoryPool__default`)
-}
-
-MemoryPool__bytes_allocated <- function(pool){
- .Call(`_arrow_MemoryPool__bytes_allocated`, pool)
-}
-
-MemoryPool__max_memory <- function(pool){
- .Call(`_arrow_MemoryPool__max_memory`, pool)
-}
-
-MemoryPool__backend_name <- function(pool){
- .Call(`_arrow_MemoryPool__backend_name`, pool)
-}
-
-supported_memory_backends <- function(){
- .Call(`_arrow_supported_memory_backends`)
-}
-
-ipc___Message__body_length <- function(message){
- .Call(`_arrow_ipc___Message__body_length`, message)
-}
-
-ipc___Message__metadata <- function(message){
- .Call(`_arrow_ipc___Message__metadata`, message)
-}
-
-ipc___Message__body <- function(message){
- .Call(`_arrow_ipc___Message__body`, message)
-}
-
-ipc___Message__Verify <- function(message){
- .Call(`_arrow_ipc___Message__Verify`, message)
-}
-
-ipc___Message__type <- function(message){
- .Call(`_arrow_ipc___Message__type`, message)
-}
-
-ipc___Message__Equals <- function(x, y){
- .Call(`_arrow_ipc___Message__Equals`, x, y)
-}
-
-ipc___ReadRecordBatch__Message__Schema <- function(message, schema){
- .Call(`_arrow_ipc___ReadRecordBatch__Message__Schema`, message, schema)
-}
-
-ipc___ReadSchema_InputStream <- function(stream){
- .Call(`_arrow_ipc___ReadSchema_InputStream`, stream)
-}
-
-ipc___ReadSchema_Message <- function(message){
- .Call(`_arrow_ipc___ReadSchema_Message`, message)
-}
-
-ipc___MessageReader__Open <- function(stream){
- .Call(`_arrow_ipc___MessageReader__Open`, stream)
-}
-
-ipc___MessageReader__ReadNextMessage <- function(reader){
- .Call(`_arrow_ipc___MessageReader__ReadNextMessage`, reader)
-}
-
-ipc___ReadMessage <- function(stream){
- .Call(`_arrow_ipc___ReadMessage`, stream)
-}
-
-parquet___arrow___ArrowReaderProperties__Make <- function(use_threads){
- .Call(`_arrow_parquet___arrow___ArrowReaderProperties__Make`, use_threads)
-}
-
-parquet___arrow___ArrowReaderProperties__set_use_threads <- function(properties, use_threads){
- invisible(.Call(`_arrow_parquet___arrow___ArrowReaderProperties__set_use_threads`, properties, use_threads))
-}
-
-parquet___arrow___ArrowReaderProperties__get_use_threads <- function(properties, use_threads){
- .Call(`_arrow_parquet___arrow___ArrowReaderProperties__get_use_threads`, properties, use_threads)
-}
-
-parquet___arrow___ArrowReaderProperties__get_read_dictionary <- function(properties, column_index){
- .Call(`_arrow_parquet___arrow___ArrowReaderProperties__get_read_dictionary`, properties, column_index)
-}
-
-parquet___arrow___ArrowReaderProperties__set_read_dictionary <- function(properties, column_index, read_dict){
- invisible(.Call(`_arrow_parquet___arrow___ArrowReaderProperties__set_read_dictionary`, properties, column_index, read_dict))
-}
-
-parquet___arrow___FileReader__OpenFile <- function(file, props){
- .Call(`_arrow_parquet___arrow___FileReader__OpenFile`, file, props)
-}
-
-parquet___arrow___FileReader__ReadTable1 <- function(reader){
- .Call(`_arrow_parquet___arrow___FileReader__ReadTable1`, reader)
-}
-
-parquet___arrow___FileReader__ReadTable2 <- function(reader, column_indices){
- .Call(`_arrow_parquet___arrow___FileReader__ReadTable2`, reader, column_indices)
-}
-
-parquet___arrow___FileReader__ReadRowGroup1 <- function(reader, i){
- .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroup1`, reader, i)
-}
-
-parquet___arrow___FileReader__ReadRowGroup2 <- function(reader, i, column_indices){
- .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroup2`, reader, i, column_indices)
-}
-
-parquet___arrow___FileReader__ReadRowGroups1 <- function(reader, row_groups){
- .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroups1`, reader, row_groups)
-}
-
-parquet___arrow___FileReader__ReadRowGroups2 <- function(reader, row_groups, column_indices){
- .Call(`_arrow_parquet___arrow___FileReader__ReadRowGroups2`, reader, row_groups, column_indices)
-}
-
-parquet___arrow___FileReader__num_rows <- function(reader){
- .Call(`_arrow_parquet___arrow___FileReader__num_rows`, reader)
-}
-
-parquet___arrow___FileReader__num_columns <- function(reader){
- .Call(`_arrow_parquet___arrow___FileReader__num_columns`, reader)
-}
-
-parquet___arrow___FileReader__num_row_groups <- function(reader){
- .Call(`_arrow_parquet___arrow___FileReader__num_row_groups`, reader)
-}
-
-parquet___arrow___FileReader__ReadColumn <- function(reader, i){
- .Call(`_arrow_parquet___arrow___FileReader__ReadColumn`, reader, i)
-}
-
-parquet___ArrowWriterProperties___create <- function(allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit){
- .Call(`_arrow_parquet___ArrowWriterProperties___create`, allow_truncated_timestamps, use_deprecated_int96_timestamps, timestamp_unit)
-}
-
-parquet___WriterProperties___Builder__create <- function(){
- .Call(`_arrow_parquet___WriterProperties___Builder__create`)
-}
-
-parquet___WriterProperties___Builder__version <- function(builder, version){
- invisible(.Call(`_arrow_parquet___WriterProperties___Builder__version`, builder, version))
-}
-
-parquet___ArrowWriterProperties___Builder__set_compressions <- function(builder, paths, types){
- invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_compressions`, builder, paths, types))
-}
-
-parquet___ArrowWriterProperties___Builder__set_compression_levels <- function(builder, paths, levels){
- invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_compression_levels`, builder, paths, levels))
-}
-
-parquet___ArrowWriterProperties___Builder__set_use_dictionary <- function(builder, paths, use_dictionary){
- invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_use_dictionary`, builder, paths, use_dictionary))
-}
-
-parquet___ArrowWriterProperties___Builder__set_write_statistics <- function(builder, paths, write_statistics){
- invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__set_write_statistics`, builder, paths, write_statistics))
-}
-
-parquet___ArrowWriterProperties___Builder__data_page_size <- function(builder, data_page_size){
- invisible(.Call(`_arrow_parquet___ArrowWriterProperties___Builder__data_page_size`, builder, data_page_size))
-}
-
-parquet___WriterProperties___Builder__build <- function(builder){
- .Call(`_arrow_parquet___WriterProperties___Builder__build`, builder)
-}
-
-parquet___arrow___ParquetFileWriter__Open <- function(schema, sink, properties, arrow_properties){
- .Call(`_arrow_parquet___arrow___ParquetFileWriter__Open`, schema, sink, properties, arrow_properties)
-}
-
-parquet___arrow___FileWriter__WriteTable <- function(writer, table, chunk_size){
- invisible(.Call(`_arrow_parquet___arrow___FileWriter__WriteTable`, writer, table, chunk_size))
-}
-
-parquet___arrow___FileWriter__Close <- function(writer){
- invisible(.Call(`_arrow_parquet___arrow___FileWriter__Close`, writer))
-}
-
-parquet___arrow___WriteTable <- function(table, sink, properties, arrow_properties){
- invisible(.Call(`_arrow_parquet___arrow___WriteTable`, table, sink, properties, arrow_properties))
-}
-
-parquet___arrow___FileReader__GetSchema <- function(reader){
- .Call(`_arrow_parquet___arrow___FileReader__GetSchema`, reader)
-}
-
-ImportArray <- function(array, schema){
- .Call(`_arrow_ImportArray`, array, schema)
-}
-
-ImportRecordBatch <- function(array, schema){
- .Call(`_arrow_ImportRecordBatch`, array, schema)
-}
-
-ImportSchema <- function(schema){
- .Call(`_arrow_ImportSchema`, schema)
-}
-
-allocate_arrow_schema <- function(){
- .Call(`_arrow_allocate_arrow_schema`)
-}
-
-delete_arrow_schema <- function(ptr){
- invisible(.Call(`_arrow_delete_arrow_schema`, ptr))
-}
-
-allocate_arrow_array <- function(){
- .Call(`_arrow_allocate_arrow_array`)
-}
-
-delete_arrow_array <- function(ptr){
- invisible(.Call(`_arrow_delete_arrow_array`, ptr))
-}
-
-ExportType <- function(type, ptr){
- invisible(.Call(`_arrow_ExportType`, type, ptr))
-}
-
-ExportSchema <- function(schema, ptr){
- invisible(.Call(`_arrow_ExportSchema`, schema, ptr))
-}
-
-ExportArray <- function(array, array_ptr, schema_ptr){
- invisible(.Call(`_arrow_ExportArray`, array, array_ptr, schema_ptr))
-}
-
-ExportRecordBatch <- function(batch, array_ptr, schema_ptr){
- invisible(.Call(`_arrow_ExportRecordBatch`, batch, array_ptr, schema_ptr))
-}
-
-vec_to_arrow <- function(x, s_type){
- .Call(`_arrow_vec_to_arrow`, x, s_type)
-}
-
-DictionaryArray__FromArrays <- function(type, indices, dict){
- .Call(`_arrow_DictionaryArray__FromArrays`, type, indices, dict)
-}
-
-RecordBatch__num_columns <- function(x){
- .Call(`_arrow_RecordBatch__num_columns`, x)
-}
-
-RecordBatch__num_rows <- function(x){
- .Call(`_arrow_RecordBatch__num_rows`, x)
-}
-
-RecordBatch__schema <- function(x){
- .Call(`_arrow_RecordBatch__schema`, x)
-}
-
-RecordBatch__RenameColumns <- function(batch, names){
- .Call(`_arrow_RecordBatch__RenameColumns`, batch, names)
-}
-
-RecordBatch__ReplaceSchemaMetadata <- function(x, metadata){
- .Call(`_arrow_RecordBatch__ReplaceSchemaMetadata`, x, metadata)
-}
-
-RecordBatch__columns <- function(batch){
- .Call(`_arrow_RecordBatch__columns`, batch)
-}
-
-RecordBatch__column <- function(batch, i){
- .Call(`_arrow_RecordBatch__column`, batch, i)
-}
-
-RecordBatch__GetColumnByName <- function(batch, name){
- .Call(`_arrow_RecordBatch__GetColumnByName`, batch, name)
-}
-
-RecordBatch__SelectColumns <- function(batch, indices){
- .Call(`_arrow_RecordBatch__SelectColumns`, batch, indices)
-}
-
-RecordBatch__Equals <- function(self, other, check_metadata){
- .Call(`_arrow_RecordBatch__Equals`, self, other, check_metadata)
-}
-
-RecordBatch__AddColumn <- function(batch, i, field, column){
- .Call(`_arrow_RecordBatch__AddColumn`, batch, i, field, column)
-}
-
-RecordBatch__SetColumn <- function(batch, i, field, column){
- .Call(`_arrow_RecordBatch__SetColumn`, batch, i, field, column)
-}
-
-RecordBatch__RemoveColumn <- function(batch, i){
- .Call(`_arrow_RecordBatch__RemoveColumn`, batch, i)
-}
-
-RecordBatch__column_name <- function(batch, i){
- .Call(`_arrow_RecordBatch__column_name`, batch, i)
-}
-
-RecordBatch__names <- function(batch){
- .Call(`_arrow_RecordBatch__names`, batch)
-}
-
-RecordBatch__Slice1 <- function(self, offset){
- .Call(`_arrow_RecordBatch__Slice1`, self, offset)
-}
-
-RecordBatch__Slice2 <- function(self, offset, length){
- .Call(`_arrow_RecordBatch__Slice2`, self, offset, length)
-}
-
-ipc___SerializeRecordBatch__Raw <- function(batch){
- .Call(`_arrow_ipc___SerializeRecordBatch__Raw`, batch)
-}
-
-ipc___ReadRecordBatch__InputStream__Schema <- function(stream, schema){
- .Call(`_arrow_ipc___ReadRecordBatch__InputStream__Schema`, stream, schema)
-}
-
-RecordBatch__from_arrays <- function(schema_sxp, lst){
- .Call(`_arrow_RecordBatch__from_arrays`, schema_sxp, lst)
-}
-
-RecordBatchReader__schema <- function(reader){
- .Call(`_arrow_RecordBatchReader__schema`, reader)
-}
-
-RecordBatchReader__ReadNext <- function(reader){
- .Call(`_arrow_RecordBatchReader__ReadNext`, reader)
-}
-
-ipc___RecordBatchStreamReader__Open <- function(stream){
- .Call(`_arrow_ipc___RecordBatchStreamReader__Open`, stream)
-}
-
-ipc___RecordBatchStreamReader__batches <- function(reader){
- .Call(`_arrow_ipc___RecordBatchStreamReader__batches`, reader)
-}
-
-ipc___RecordBatchFileReader__schema <- function(reader){
- .Call(`_arrow_ipc___RecordBatchFileReader__schema`, reader)
-}
-
-ipc___RecordBatchFileReader__num_record_batches <- function(reader){
- .Call(`_arrow_ipc___RecordBatchFileReader__num_record_batches`, reader)
-}
-
-ipc___RecordBatchFileReader__ReadRecordBatch <- function(reader, i){
- .Call(`_arrow_ipc___RecordBatchFileReader__ReadRecordBatch`, reader, i)
-}
-
-ipc___RecordBatchFileReader__Open <- function(file){
- .Call(`_arrow_ipc___RecordBatchFileReader__Open`, file)
-}
-
-Table__from_RecordBatchReader <- function(reader){
- .Call(`_arrow_Table__from_RecordBatchReader`, reader)
-}
-
-Table__from_RecordBatchFileReader <- function(reader){
- .Call(`_arrow_Table__from_RecordBatchFileReader`, reader)
-}
-
-ipc___RecordBatchFileReader__batches <- function(reader){
- .Call(`_arrow_ipc___RecordBatchFileReader__batches`, reader)
-}
-
-ipc___RecordBatchWriter__WriteRecordBatch <- function(batch_writer, batch){
- invisible(.Call(`_arrow_ipc___RecordBatchWriter__WriteRecordBatch`, batch_writer, batch))
-}
-
-ipc___RecordBatchWriter__WriteTable <- function(batch_writer, table){
- invisible(.Call(`_arrow_ipc___RecordBatchWriter__WriteTable`, batch_writer, table))
-}
-
-ipc___RecordBatchWriter__Close <- function(batch_writer){
- invisible(.Call(`_arrow_ipc___RecordBatchWriter__Close`, batch_writer))
-}
-
-ipc___RecordBatchFileWriter__Open <- function(stream, schema, use_legacy_format, metadata_version){
- .Call(`_arrow_ipc___RecordBatchFileWriter__Open`, stream, schema, use_legacy_format, metadata_version)
-}
-
-ipc___RecordBatchStreamWriter__Open <- function(stream, schema, use_legacy_format, metadata_version){
- .Call(`_arrow_ipc___RecordBatchStreamWriter__Open`, stream, schema, use_legacy_format, metadata_version)
-}
-
-runtime_info <- function(){
- .Call(`_arrow_runtime_info`)
-}
-
-Array__GetScalar <- function(x, i){
- .Call(`_arrow_Array__GetScalar`, x, i)
-}
-
-Scalar__ToString <- function(s){
- .Call(`_arrow_Scalar__ToString`, s)
-}
-
-StructScalar__field <- function(s, i){
- .Call(`_arrow_StructScalar__field`, s, i)
-}
-
-StructScalar__GetFieldByName <- function(s, name){
- .Call(`_arrow_StructScalar__GetFieldByName`, s, name)
-}
-
-Scalar__as_vector <- function(scalar){
- .Call(`_arrow_Scalar__as_vector`, scalar)
-}
-
-MakeArrayFromScalar <- function(scalar){
- .Call(`_arrow_MakeArrayFromScalar`, scalar)
-}
-
-Scalar__is_valid <- function(s){
- .Call(`_arrow_Scalar__is_valid`, s)
-}
-
-Scalar__type <- function(s){
- .Call(`_arrow_Scalar__type`, s)
-}
-
-Scalar__Equals <- function(lhs, rhs){
- .Call(`_arrow_Scalar__Equals`, lhs, rhs)
-}
-
-Scalar__ApproxEquals <- function(lhs, rhs){
- .Call(`_arrow_Scalar__ApproxEquals`, lhs, rhs)
-}
-
-schema_ <- function(fields){
- .Call(`_arrow_schema_`, fields)
-}
-
-Schema__ToString <- function(s){
- .Call(`_arrow_Schema__ToString`, s)
-}
-
-Schema__num_fields <- function(s){
- .Call(`_arrow_Schema__num_fields`, s)
-}
-
-Schema__field <- function(s, i){
- .Call(`_arrow_Schema__field`, s, i)
-}
-
-Schema__AddField <- function(s, i, field){
- .Call(`_arrow_Schema__AddField`, s, i, field)
-}
-
-Schema__SetField <- function(s, i, field){
- .Call(`_arrow_Schema__SetField`, s, i, field)
-}
-
-Schema__RemoveField <- function(s, i){
- .Call(`_arrow_Schema__RemoveField`, s, i)
-}
-
-Schema__GetFieldByName <- function(s, x){
- .Call(`_arrow_Schema__GetFieldByName`, s, x)
-}
-
-Schema__fields <- function(schema){
- .Call(`_arrow_Schema__fields`, schema)
-}
-
-Schema__field_names <- function(schema){
- .Call(`_arrow_Schema__field_names`, schema)
-}
-
-Schema__HasMetadata <- function(schema){
- .Call(`_arrow_Schema__HasMetadata`, schema)
-}
-
-Schema__metadata <- function(schema){
- .Call(`_arrow_Schema__metadata`, schema)
-}
-
-Schema__WithMetadata <- function(schema, metadata){
- .Call(`_arrow_Schema__WithMetadata`, schema, metadata)
-}
-
-Schema__serialize <- function(schema){
- .Call(`_arrow_Schema__serialize`, schema)
-}
-
-Schema__Equals <- function(schema, other, check_metadata){
- .Call(`_arrow_Schema__Equals`, schema, other, check_metadata)
-}
-
-arrow__UnifySchemas <- function(schemas){
- .Call(`_arrow_arrow__UnifySchemas`, schemas)
-}
-
-Table__num_columns <- function(x){
- .Call(`_arrow_Table__num_columns`, x)
-}
-
-Table__num_rows <- function(x){
- .Call(`_arrow_Table__num_rows`, x)
-}
-
-Table__schema <- function(x){
- .Call(`_arrow_Table__schema`, x)
-}
-
-Table__ReplaceSchemaMetadata <- function(x, metadata){
- .Call(`_arrow_Table__ReplaceSchemaMetadata`, x, metadata)
-}
-
-Table__column <- function(table, i){
- .Call(`_arrow_Table__column`, table, i)
-}
-
-Table__field <- function(table, i){
- .Call(`_arrow_Table__field`, table, i)
-}
-
-Table__columns <- function(table){
- .Call(`_arrow_Table__columns`, table)
-}
-
-Table__ColumnNames <- function(table){
- .Call(`_arrow_Table__ColumnNames`, table)
-}
-
-Table__RenameColumns <- function(table, names){
- .Call(`_arrow_Table__RenameColumns`, table, names)
-}
-
-Table__Slice1 <- function(table, offset){
- .Call(`_arrow_Table__Slice1`, table, offset)
-}
-
-Table__Slice2 <- function(table, offset, length){
- .Call(`_arrow_Table__Slice2`, table, offset, length)
-}
-
-Table__Equals <- function(lhs, rhs, check_metadata){
- .Call(`_arrow_Table__Equals`, lhs, rhs, check_metadata)
-}
-
-Table__Validate <- function(table){
- .Call(`_arrow_Table__Validate`, table)
-}
-
-Table__ValidateFull <- function(table){
- .Call(`_arrow_Table__ValidateFull`, table)
-}
-
-Table__GetColumnByName <- function(table, name){
- .Call(`_arrow_Table__GetColumnByName`, table, name)
-}
-
-Table__RemoveColumn <- function(table, i){
- .Call(`_arrow_Table__RemoveColumn`, table, i)
-}
-
-Table__AddColumn <- function(table, i, field, column){
- .Call(`_arrow_Table__AddColumn`, table, i, field, column)
-}
-
-Table__SetColumn <- function(table, i, field, column){
- .Call(`_arrow_Table__SetColumn`, table, i, field, column)
-}
-
-Table__SelectColumns <- function(table, indices){
- .Call(`_arrow_Table__SelectColumns`, table, indices)
-}
-
-all_record_batches <- function(lst){
- .Call(`_arrow_all_record_batches`, lst)
-}
-
-Table__from_record_batches <- function(batches, schema_sxp){
- .Call(`_arrow_Table__from_record_batches`, batches, schema_sxp)
-}
-
-Table__from_dots <- function(lst, schema_sxp){
- .Call(`_arrow_Table__from_dots`, lst, schema_sxp)
-}
-
-GetCpuThreadPoolCapacity <- function(){
- .Call(`_arrow_GetCpuThreadPoolCapacity`)
-}
-
-SetCpuThreadPoolCapacity <- function(threads){
- invisible(.Call(`_arrow_SetCpuThreadPoolCapacity`, threads))
-}
-
-Array__infer_type <- function(x){
- .Call(`_arrow_Array__infer_type`, x)
-}
-
-
-
diff --git a/r/R/buffer.R b/r/R/buffer.R
deleted file mode 100644
index db61ed3..0000000
--- a/r/R/buffer.R
+++ /dev/null
@@ -1,72 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @title Buffer class
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @description A Buffer is an object containing a pointer to a piece of
-#' contiguous memory with a particular size.
-#' @section Factory:
-#' `buffer()` lets you create an `arrow::Buffer` from an R object
-#' @section Methods:
-#'
-#' - `$is_mutable()` :
-#' - `$ZeroPadding()` :
-#' - `$size()` :
-#' - `$capacity()`:
-#'
-#' @rdname buffer
-#' @name buffer
-#' @export
-#' @include arrow-package.R
-#' @include enums.R
-Buffer <- R6Class("Buffer", inherit = ArrowObject,
- public = list(
- ZeroPadding = function() Buffer__ZeroPadding(self),
- data = function() Buffer__data(self),
- Equals = function(other, ...) {
- inherits(other, "Buffer") && Buffer__Equals(self, other)
- }
- ),
-
- active = list(
- is_mutable = function() Buffer__is_mutable(self),
- size = function() Buffer__size(self),
- capacity = function() Buffer__capacity(self)
- )
-)
-
-Buffer$create <- function(x) {
- if (inherits(x, "Buffer")) {
- x
- } else if (inherits(x, c("raw", "numeric", "integer", "complex"))) {
- r___RBuffer__initialize(x)
- } else if (inherits(x, "BufferOutputStream")) {
- x$finish()
- } else {
- stop("Cannot convert object of class ", class(x), " to arrow::Buffer")
- }
-}
-
-#' @param x R object. Only raw, numeric and integer vectors are currently supported
-#' @return an instance of `Buffer` that borrows memory from `x`
-#' @export
-buffer <- Buffer$create
-
-#' @export
-as.raw.Buffer <- function(x) x$data()
diff --git a/r/R/chunked-array.R b/r/R/chunked-array.R
deleted file mode 100644
index a7f9c8f..0000000
--- a/r/R/chunked-array.R
+++ /dev/null
@@ -1,132 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-datum.R
-
-#' @title ChunkedArray class
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @description A `ChunkedArray` is a data structure managing a list of
-#' primitive Arrow [Arrays][Array] logically as one large array. Chunked arrays
-#' may be grouped together in a [Table].
-#' @section Factory:
-#' The `ChunkedArray$create()` factory method instantiates the object from
-#' various Arrays or R vectors. `chunked_array()` is an alias for it.
-#'
-#' @section Methods:
-#'
-#' - `$length()`: Size in the number of elements this array contains
-#' - `$chunk(i)`: Extract an `Array` chunk by integer position
-#' - `$as_vector()`: convert to an R vector
-#' - `$Slice(offset, length = NULL)`: Construct a zero-copy slice of the array
-#' with the indicated offset and length. If length is `NULL`, the slice goes
-#' until the end of the array.
-#' - `$Take(i)`: return a `ChunkedArray` with values at positions given by
-#' integers `i`. If `i` is an Arrow `Array` or `ChunkedArray`, it will be
-#' coerced to an R vector before taking.
-#' - `$Filter(i, keep_na = TRUE)`: return a `ChunkedArray` with values at positions where
-#' logical vector or Arrow boolean-type `(Chunked)Array` `i` is `TRUE`.
-#' - `$SortIndices(descending = FALSE)`: return an `Array` of integer positions that can be
-#' used to rearrange the `ChunkedArray` in ascending or descending order
-#' - `$cast(target_type, safe = TRUE, options = cast_options(safe))`: Alter the
-#' data in the array to change its type.
-#' - `$null_count()`: The number of null entries in the array
-#' - `$chunks()`: return a list of `Array`s
-#' - `$num_chunks()`: integer number of chunks in the `ChunkedArray`
-#' - `$type()`: logical type of data
-#' - `$View(type)`: Construct a zero-copy view of this `ChunkedArray` with the
-#' given type.
-#' - `$Validate()`: Perform any validation checks to determine obvious inconsistencies
-#' within the array's internal data. This can be an expensive check, potentially `O(length)`
-#'
-#' @rdname ChunkedArray
-#' @name ChunkedArray
-#' @seealso [Array]
-#' @export
-ChunkedArray <- R6Class("ChunkedArray", inherit = ArrowDatum,
- public = list(
- length = function() ChunkedArray__length(self),
- chunk = function(i) Array$create(ChunkedArray__chunk(self, i)),
- as_vector = function() ChunkedArray__as_vector(self),
- Slice = function(offset, length = NULL){
- if (is.null(length)) {
- ChunkedArray__Slice1(self, offset)
- } else {
- ChunkedArray__Slice2(self, offset, length)
- }
- },
- Take = function(i) {
- if (is.numeric(i)) {
- i <- as.integer(i)
- }
- if (is.integer(i)) {
- i <- Array$create(i)
- }
- call_function("take", self, i)
- },
- Filter = function(i, keep_na = TRUE) {
- if (is.logical(i)) {
- i <- Array$create(i)
- }
- call_function("filter", self, i, options = list(keep_na = keep_na))
- },
- SortIndices = function(descending = FALSE) {
- assert_that(is.logical(descending))
- assert_that(length(descending) == 1L)
- assert_that(!is.na(descending))
- # TODO: after ARROW-12042 is closed, review whether this and the
- # Array$SortIndices definition can be consolidated
- call_function(
- "sort_indices",
- self,
- options = list(names = "", orders = as.integer(descending))
- )
- },
- View = function(type) {
- ChunkedArray__View(self, as_type(type))
- },
- Validate = function() {
- ChunkedArray__Validate(self)
- },
- ToString = function() {
- ChunkedArray__ToString(self)
- },
- Equals = function(other, ...) {
- inherits(other, "ChunkedArray") && ChunkedArray__Equals(self, other)
- }
- ),
- active = list(
- null_count = function() ChunkedArray__null_count(self),
- num_chunks = function() ChunkedArray__num_chunks(self),
- chunks = function() map(ChunkedArray__chunks(self), Array$create),
- type = function() ChunkedArray__type(self)
- )
-)
-
-ChunkedArray$create <- function(..., type = NULL) {
- if (!is.null(type)) {
- type <- as_type(type)
- }
- ChunkedArray__from_list(list2(...), type)
-}
-
-#' @param \dots Vectors to coerce
-#' @param type currently ignored
-#' @rdname ChunkedArray
-#' @export
-chunked_array <- ChunkedArray$create
diff --git a/r/R/compression.R b/r/R/compression.R
deleted file mode 100644
index ebd4c54..0000000
--- a/r/R/compression.R
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include enums.R
-#' @include arrow-package.R
-#' @include io.R
-
-#' @title Compression Codec class
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @description Codecs allow you to create [compressed input and output
-#' streams][compression].
-#' @section Factory:
-#' The `Codec$create()` factory method takes the following arguments:
-#' * `type`: string name of the compression method. Possible values are
-#' "uncompressed", "snappy", "gzip", "brotli", "zstd", "lz4", "lzo", or
-#' "bz2". `type` may be upper- or lower-cased. Not all methods may be
-#' available; support depends on build-time flags for the C++ library.
-#' See [codec_is_available()]. Most builds support at least "snappy" and
-#' "gzip". All support "uncompressed".
-#' * `compression_level`: compression level, the default value (`NA`) uses the
-#' default compression level for the selected compression `type`.
-#' @rdname Codec
-#' @name Codec
-#' @export
-Codec <- R6Class("Codec", inherit = ArrowObject,
- active = list(
- name = function() util___Codec__name(self),
- level = function() abort("Codec$level() not yet implemented")
- )
-)
-Codec$create <- function(type = "gzip", compression_level = NA) {
- if (is.string(type)) {
- type <- util___Codec__Create(
- compression_from_name(type), compression_level
- )
- }
- assert_is(type, "Codec")
- type
-}
-
-#' Check whether a compression codec is available
-#'
-#' Support for compression libraries depends on the build-time settings of
-#' the Arrow C++ library. This function lets you know which are available for
-#' use.
-#' @param type A string, one of "uncompressed", "snappy", "gzip", "brotli",
-#' "zstd", "lz4", "lzo", or "bz2", case insensitive.
-#' @return Logical: is `type` available?
-#' @export
-codec_is_available <- function(type) {
- util___Codec__IsAvailable(compression_from_name(type))
-}
-
-compression_from_name <- function(name) {
- map_int(name, ~CompressionType[[match.arg(toupper(.x), names(CompressionType))]])
-}
-
-#' @title Compressed stream classes
-#' @rdname compression
-#' @name compression
-#' @aliases CompressedInputStream CompressedOutputStream
-#' @docType class
-#' @usage NULL
-#' @format NULL
-#' @description `CompressedInputStream` and `CompressedOutputStream`
-#' allow you to apply a compression [Codec] to an
-#' input or output stream.
-#'
-#' @section Factory:
-#'
-#' The `CompressedInputStream$create()` and `CompressedOutputStream$create()`
-#' factory methods instantiate the object and take the following arguments:
-#'
-#' - `stream` An [InputStream] or [OutputStream], respectively
-#' - `codec` A `Codec`, either a [Codec][Codec] instance or a string
-#' - `compression_level` compression level for when the `codec` argument is given as a string
-#'
-#' @section Methods:
-#'
-#' Methods are inherited from [InputStream] and [OutputStream], respectively
-#' @export
-#' @include arrow-package.R
-CompressedOutputStream <- R6Class("CompressedOutputStream", inherit = OutputStream)
-CompressedOutputStream$create <- function(stream, codec = "gzip", compression_level = NA){
- codec <- Codec$create(codec, compression_level = compression_level)
- if (is.string(stream)) {
- stream <- FileOutputStream$create(stream)
- }
- assert_is(stream, "OutputStream")
- io___CompressedOutputStream__Make(codec, stream)
-}
-
-#' @rdname compression
-#' @usage NULL
-#' @format NULL
-#' @export
-CompressedInputStream <- R6Class("CompressedInputStream", inherit = InputStream)
-CompressedInputStream$create <- function(stream, codec = "gzip", compression_level = NA){
- codec <- Codec$create(codec, compression_level = compression_level)
- if (is.string(stream)) {
- stream <- ReadableFile$create(stream)
- }
- assert_is(stream, "InputStream")
- io___CompressedInputStream__Make(codec, stream)
-}
diff --git a/r/R/compute.R b/r/R/compute.R
deleted file mode 100644
index 1b79d76..0000000
--- a/r/R/compute.R
+++ /dev/null
@@ -1,257 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Call an Arrow compute function
-#'
-#' This function provides a lower-level API for calling Arrow functions by their
-#' string function name. You won't use it directly for most applications.
-#' Many Arrow compute functions are mapped to R methods,
-#' and in a `dplyr` evaluation context, [all Arrow functions][list_compute_functions()]
-#' are callable with an `arrow_` prefix.
-#' @param function_name string Arrow compute function name
-#' @param ... Function arguments, which may include `Array`, `ChunkedArray`, `Scalar`,
-#' `RecordBatch`, or `Table`.
-#' @param args list arguments as an alternative to specifying in `...`
-#' @param options named list of C++ function options.
-#' @return An `Array`, `ChunkedArray`, `Scalar`, `RecordBatch`, or `Table`, whatever the compute function results in.
-#' @seealso [Arrow C++ documentation](https://arrow.apache.org/docs/cpp/compute.html) for the functions and their respective options.
-#' @examples
-#' \donttest{
-#' a <- Array$create(c(1L, 2L, 3L, NA, 5L))
-#' s <- Scalar$create(4L)
-#' call_function("fill_null", a, s)
-#'
-#' a <- Array$create(rnorm(10000))
-#' call_function("quantile", a, options = list(q = seq(0, 1, 0.25)))
-#' }
-#' @export
-#' @include array.R
-#' @include chunked-array.R
-#' @include scalar.R
-call_function <- function(function_name, ..., args = list(...), options = empty_named_list()) {
- assert_that(is.string(function_name))
- assert_that(is.list(options), !is.null(names(options)))
-
- datum_classes <- c("Array", "ChunkedArray", "RecordBatch", "Table", "Scalar")
- valid_args <- map_lgl(args, ~inherits(., datum_classes))
- if (!all(valid_args)) {
- # Lame, just pick one to report
- first_bad <- min(which(!valid_args))
- stop("Argument ", first_bad, " is of class ", head(class(args[[first_bad]]), 1), " but it must be one of ", oxford_paste(datum_classes, "or"), call. = FALSE)
- }
-
- compute__CallFunction(function_name, args, options)
-}
-
-#' List available Arrow C++ compute functions
-#'
-#' This function lists the names of all available Arrow C++ library compute functions.
-#' These can be called by passing to [call_function()], or they can be
-#' called by name with an `arrow_` prefix inside a `dplyr` verb.
-#'
-#' The resulting list describes the capabilities of your `arrow` build.
-#' Some functions, such as string and regular expression functions,
-#' require optional build-time C++ dependencies. If your `arrow` package
-#' was not compiled with those features enabled, those functions will
-#' not appear in this list.
-#'
-#' Some functions take options that need to be passed when calling them
-#' (in a list called `options`). These options require custom handling
-#' in C++; many functions already have that handling set up but not all do.
-#' If you encounter one that needs special handling for options, please
-#' report an issue.
-#'
-#' Note that this list does *not* enumerate all of the R bindings for these functions.
-#' The package includes Arrow methods for many base R functions that can
-#' be called directly on Arrow objects, as well as some tidyverse-flavored versions
-#' available inside `dplyr` verbs.
-#'
-#' @param pattern Optional regular expression to filter the function list
-#' @param ... Additional parameters passed to `grep()`
-#' @return A character vector of available Arrow C++ function names
-#' @export
-list_compute_functions <- function(pattern = NULL, ...) {
- funcs <- compute__GetFunctionNames()
- if (!is.null(pattern)) {
- funcs <- grep(pattern, funcs, value = TRUE, ...)
- }
- funcs
-}
-
-#' @export
-sum.ArrowDatum <- function(..., na.rm = FALSE) scalar_aggregate("sum", ..., na.rm = na.rm)
-
-#' @export
-mean.ArrowDatum <- function(..., na.rm = FALSE) scalar_aggregate("mean", ..., na.rm = na.rm)
-
-#' @export
-min.ArrowDatum <- function(..., na.rm = FALSE) {
- scalar_aggregate("min_max", ..., na.rm = na.rm)$GetFieldByName("min")
-}
-
-#' @export
-max.ArrowDatum <- function(..., na.rm = FALSE) {
- scalar_aggregate("min_max", ..., na.rm = na.rm)$GetFieldByName("max")
-}
-
-scalar_aggregate <- function(FUN, ..., na.rm = FALSE) {
- a <- collect_arrays_from_dots(list(...))
- if (!na.rm && a$null_count > 0 && (FUN %in% c("mean", "sum"))) {
- # Arrow sum/mean function always drops NAs so handle that here
- # https://issues.apache.org/jira/browse/ARROW-9054
- return(Scalar$create(NA_real_))
- }
-
- call_function(FUN, a, options = list(na.rm = na.rm))
-}
-
-collect_arrays_from_dots <- function(dots) {
- # Given a list that may contain both Arrays and ChunkedArrays,
- # return a single ChunkedArray containing all of those chunks
- # (may return a regular Array if there is only one element in dots)
- assert_that(all(map_lgl(dots, is.Array)))
- if (length(dots) == 1) {
- return(dots[[1]])
- }
-
- arrays <- unlist(lapply(dots, function(x) {
- if (inherits(x, "ChunkedArray")) {
- x$chunks
- } else {
- x
- }
- }))
- ChunkedArray$create(!!!arrays)
-}
-
-#' @export
-quantile.ArrowDatum <- function(x,
- probs = seq(0, 1, 0.25),
- na.rm = FALSE,
- type = 7,
- interpolation = c("linear", "lower", "higher", "nearest", "midpoint"),
- ...) {
- if (inherits(x, "Scalar")) x <- Array$create(x)
- assert_is(probs, c("numeric", "integer"))
- assert_that(length(probs) > 0)
- assert_that(all(probs >= 0 & probs <= 1))
- if (!na.rm && x$null_count > 0) {
- stop("Missing values not allowed if 'na.rm' is FALSE", call. = FALSE)
- }
- if (type != 7) {
- stop(
- "Argument `type` not supported in Arrow. To control the quantile ",
- "interpolation algorithm, set argument `interpolation` to one of: ",
- "\"linear\" (the default), \"lower\", \"higher\", \"nearest\", or ",
- "\"midpoint\".",
- call. = FALSE
- )
- }
- interpolation <- QuantileInterpolation[[toupper(match.arg(interpolation))]]
- out <- call_function("quantile", x, options = list(q = probs, interpolation = interpolation))
- if (length(out) == 0) {
- # When there are no non-missing values in the data, the Arrow quantile
- # function returns an empty Array, but for consistency with the R quantile
- # function, we want an Array of NA_real_ with the same length as probs
- out <- Array$create(rep(NA_real_, length(probs)))
- }
- out
-}
-
-#' @export
-median.ArrowDatum <- function(x, na.rm = FALSE, ...) {
- if (!na.rm && x$null_count > 0) {
- Scalar$create(NA_real_)
- } else {
- Scalar$create(quantile(x, probs = 0.5, na.rm = TRUE, ...))
- }
-}
-
-#' @export
-unique.ArrowDatum <- function(x, incomparables = FALSE, ...) {
- call_function("unique", x)
-}
-
-#' `match` and `%in%` for Arrow objects
-#'
-#' `base::match()` is not a generic, so we can't just define Arrow methods for
-#' it. This function exposes the analogous functions in the Arrow C++ library.
-#'
-#' @param x `Array` or `ChunkedArray`
-#' @param table `Array`, `ChunkedArray`, or R vector lookup table.
-#' @param ... additional arguments, ignored
-#' @return `match_arrow()` returns an `int32`-type `Array` of the same length
-#' as `x` with the (0-based) indexes into `table`. `is_in()` returns a
-#' `boolean`-type `Array` of the same length as `x` with values indicating
-#' per element of `x` it it is present in `table`.
-#' @export
-match_arrow <- function(x, table, ...) UseMethod("match_arrow")
-
-#' @export
-match_arrow.default <- function(x, table, ...) match(x, table, ...)
-
-#' @export
-match_arrow.ArrowDatum <- function(x, table, ...) {
- if (!inherits(table, c("Array", "ChunkedArray"))) {
- table <- Array$create(table)
- }
- call_function("index_in_meta_binary", x, table)
-}
-
-#' @rdname match_arrow
-#' @export
-is_in <- function(x, table, ...) UseMethod("is_in")
-
-#' @export
-is_in.default <- function(x, table, ...) x %in% table
-
-#' @export
-is_in.ArrowDatum <- function(x, table, ...) {
- if (!inherits(table, c("Array", "DictionaryArray", "ChunkedArray"))) {
- table <- Array$create(table)
- }
- call_function("is_in_meta_binary", x, table)
-}
-
-#' `table` for Arrow objects
-#'
-#' This function tabulates the values in the array and returns a table of counts.
-#' @param x `Array` or `ChunkedArray`
-#' @return A `StructArray` containing "values" (same type as `x`) and "counts"
-#' `Int64`.
-#' @export
-value_counts <- function(x) {
- call_function("value_counts", x)
-}
-
-#' Cast options
-#'
-#' @param safe logical: enforce safe conversion? Default `TRUE`
-#' @param ... additional cast options, such as `allow_int_overflow`,
-#' `allow_time_truncate`, and `allow_float_truncate`, which are set to `!safe`
-#' by default
-#' @return A list
-#' @export
-#' @keywords internal
-cast_options <- function(safe = TRUE, ...) {
- opts <- list(
- allow_int_overflow = !safe,
- allow_time_truncate = !safe,
- allow_float_truncate = !safe
- )
- modifyList(opts, list(...))
-}
diff --git a/r/R/config.R b/r/R/config.R
deleted file mode 100644
index 301d0fa..0000000
--- a/r/R/config.R
+++ /dev/null
@@ -1,30 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Manage the global CPU thread pool in libarrow
-#'
-#' @export
-cpu_count <- function() {
- GetCpuThreadPoolCapacity()
-}
-
-#' @rdname cpu_count
-#' @param num_threads integer: New number of threads for thread pool
-#' @export
-set_cpu_count <- function(num_threads) {
- SetCpuThreadPoolCapacity(as.integer(num_threads))
-}
diff --git a/r/R/csv.R b/r/R/csv.R
deleted file mode 100644
index 160c46e..0000000
--- a/r/R/csv.R
+++ /dev/null
@@ -1,587 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Read a CSV or other delimited file with Arrow
-#'
-#' These functions uses the Arrow C++ CSV reader to read into a `data.frame`.
-#' Arrow C++ options have been mapped to argument names that follow those of
-#' `readr::read_delim()`, and `col_select` was inspired by `vroom::vroom()`.
-#'
-#' `read_csv_arrow()` and `read_tsv_arrow()` are wrappers around
-#' `read_delim_arrow()` that specify a delimiter.
-#'
-#' Note that not all `readr` options are currently implemented here. Please file
-#' an issue if you encounter one that `arrow` should support.
-#'
-#' If you need to control Arrow-specific reader parameters that don't have an
-#' equivalent in `readr::read_csv()`, you can either provide them in the
-#' `parse_options`, `convert_options`, or `read_options` arguments, or you can
-#' use [CsvTableReader] directly for lower-level access.
-#'
-#' @section Specifying column types and names:
-#'
-#' By default, the CSV reader will infer the column names and data types from the file, but there
-#' are a few ways you can specify them directly.
-#'
-#' One way is to provide an Arrow [Schema] in the `schema` argument,
-#' which is an ordered map of column name to type.
-#' When provided, it satisfies both the `col_names` and `col_types` arguments.
-#' This is good if you know all of this information up front.
-#'
-#' You can also pass a `Schema` to the `col_types` argument. If you do this,
-#' column names will still be inferred from the file unless you also specify
-#' `col_names`. In either case, the column names in the `Schema` must match the
-#' data's column names, whether they are explicitly provided or inferred. That
-#' said, this `Schema` does not have to reference all columns: those omitted
-#' will have their types inferred.
-#'
-#' Alternatively, you can declare column types by providing the compact string representation
-#' that `readr` uses to the `col_types` argument. This means you provide a
-#' single string, one character per column, where the characters map to Arrow
-#' types analogously to the `readr` type mapping:
-#'
-#' * "c": `utf8()`
-#' * "i": `int32()`
-#' * "n": `float64()`
-#' * "d": `float64()`
-#' * "l": `bool()`
-#' * "f": `dictionary()`
-#' * "D": `date32()`
-#' * "T": `time32()`
-#' * "t": `timestamp()`
-#' * "_": `null()`
-#' * "-": `null()`
-#' * "?": infer the type from the data
-#'
-#' If you use the compact string representation for `col_types`, you must also
-#' specify `col_names`.
-#'
-#' Regardless of how types are specified, all columns with a `null()` type will
-#' be dropped.
-#'
-#' Note that if you are specifying column names, whether by `schema` or
-#' `col_names`, and the CSV file has a header row that would otherwise be used
-#' to idenfity column names, you'll need to add `skip = 1` to skip that row.
-#'
-#' @param file A character file name or URI, `raw` vector, an Arrow input stream,
-#' or a `FileSystem` with path (`SubTreeFileSystem`).
-#' If a file name, a memory-mapped Arrow [InputStream] will be opened and
-#' closed when finished; compression will be detected from the file extension
-#' and handled automatically. If an input stream is provided, it will be left
-#' open.
-#' @param delim Single character used to separate fields within a record.
-#' @param quote Single character used to quote strings.
-#' @param escape_double Does the file escape quotes by doubling them?
-#' i.e. If this option is `TRUE`, the value `""""` represents
-#' a single quote, `\"`.
-#' @param escape_backslash Does the file use backslashes to escape special
-#' characters? This is more general than `escape_double` as backslashes
-#' can be used to escape the delimiter character, the quote character, or
-#' to add special characters like `\\n`.
-#' @param schema [Schema] that describes the table. If provided, it will be
-#' used to satisfy both `col_names` and `col_types`.
-#' @param col_names If `TRUE`, the first row of the input will be used as the
-#' column names and will not be included in the data frame. If `FALSE`, column
-#' names will be generated by Arrow, starting with "f0", "f1", ..., "fN".
-#' Alternatively, you can specify a character vector of column names.
-#' @param col_types A compact string representation of the column types, or
-#' `NULL` (the default) to infer types from the data.
-#' @param col_select A character vector of column names to keep, as in the
-#' "select" argument to `data.table::fread()`, or a
-#' [tidy selection specification][tidyselect::vars_select()]
-#' of columns, as used in `dplyr::select()`.
-#' @param na A character vector of strings to interpret as missing values.
-#' @param quoted_na Should missing values inside quotes be treated as missing
-#' values (the default) or strings. (Note that this is different from the
-#' the Arrow C++ default for the corresponding convert option,
-#' `strings_can_be_null`.)
-#' @param skip_empty_rows Should blank rows be ignored altogether? If
-#' `TRUE`, blank rows will not be represented at all. If `FALSE`, they will be
-#' filled with missings.
-#' @param skip Number of lines to skip before reading data.
-#' @param timestamp_parsers User-defined timestamp parsers. If more than one
-#' parser is specified, the CSV conversion logic will try parsing values
-#' starting from the beginning of this vector. Possible values are:
-#' - `NULL`: the default, which uses the ISO-8601 parser
-#' - a character vector of [strptime][base::strptime()] parse strings
-#' - a list of [TimestampParser] objects
-#' @param parse_options see [file reader options][CsvReadOptions].
-#' If given, this overrides any
-#' parsing options provided in other arguments (e.g. `delim`, `quote`, etc.).
-#' @param convert_options see [file reader options][CsvReadOptions]
-#' @param read_options see [file reader options][CsvReadOptions]
-#' @param as_data_frame Should the function return a `data.frame` (default) or
-#' an Arrow [Table]?
-#'
-#' @return A `data.frame`, or a Table if `as_data_frame = FALSE`.
-#' @export
-#' @examples
-#' \donttest{
-#' tf <- tempfile()
-#' on.exit(unlink(tf))
-#' write.csv(mtcars, file = tf)
-#' df <- read_csv_arrow(tf)
-#' dim(df)
-#' # Can select columns
-#' df <- read_csv_arrow(tf, col_select = starts_with("d"))
-#' }
-read_delim_arrow <- function(file,
- delim = ",",
- quote = '"',
- escape_double = TRUE,
- escape_backslash = FALSE,
- schema = NULL,
- col_names = TRUE,
- col_types = NULL,
- col_select = NULL,
- na = c("", "NA"),
- quoted_na = TRUE,
- skip_empty_rows = TRUE,
- skip = 0L,
- parse_options = NULL,
- convert_options = NULL,
- read_options = NULL,
- as_data_frame = TRUE,
- timestamp_parsers = NULL) {
- if (inherits(schema, "Schema")) {
- col_names <- names(schema)
- col_types <- schema
- }
- if (is.null(parse_options)) {
- parse_options <- readr_to_csv_parse_options(
- delim,
- quote,
- escape_double,
- escape_backslash,
- skip_empty_rows
- )
- }
- if (is.null(read_options)) {
- read_options <- readr_to_csv_read_options(skip, col_names)
- }
- if (is.null(convert_options)) {
- convert_options <- readr_to_csv_convert_options(
- na,
- quoted_na,
- col_types = col_types,
- col_names = read_options$column_names,
- timestamp_parsers = timestamp_parsers
- )
- }
-
- if (!inherits(file, "InputStream")) {
- file <- make_readable_file(file)
- on.exit(file$close())
- }
- reader <- CsvTableReader$create(
- file,
- read_options = read_options,
- parse_options = parse_options,
- convert_options = convert_options
- )
-
- tab <- reader$Read()
-
- # TODO: move this into convert_options using include_columns
- col_select <- enquo(col_select)
- if (!quo_is_null(col_select)) {
- tab <- tab[vars_select(names(tab), !!col_select)]
- }
-
- if (isTRUE(as_data_frame)) {
- tab <- as.data.frame(tab)
- }
-
- tab
-}
-
-#' @rdname read_delim_arrow
-#' @export
-read_csv_arrow <- function(file,
- quote = '"',
- escape_double = TRUE,
- escape_backslash = FALSE,
- schema = NULL,
- col_names = TRUE,
- col_types = NULL,
- col_select = NULL,
- na = c("", "NA"),
- quoted_na = TRUE,
- skip_empty_rows = TRUE,
- skip = 0L,
- parse_options = NULL,
- convert_options = NULL,
- read_options = NULL,
- as_data_frame = TRUE,
- timestamp_parsers = NULL) {
-
- mc <- match.call()
- mc$delim <- ","
- mc[[1]] <- get("read_delim_arrow", envir = asNamespace("arrow"))
- eval.parent(mc)
-}
-
-#' @rdname read_delim_arrow
-#' @export
-read_tsv_arrow <- function(file,
- quote = '"',
- escape_double = TRUE,
- escape_backslash = FALSE,
- schema = NULL,
- col_names = TRUE,
- col_types = NULL,
- col_select = NULL,
- na = c("", "NA"),
- quoted_na = TRUE,
- skip_empty_rows = TRUE,
- skip = 0L,
- parse_options = NULL,
- convert_options = NULL,
- read_options = NULL,
- as_data_frame = TRUE,
- timestamp_parsers = NULL) {
-
- mc <- match.call()
- mc$delim <- "\t"
- mc[[1]] <- get("read_delim_arrow", envir = asNamespace("arrow"))
- eval.parent(mc)
-}
-
-#' @title Arrow CSV and JSON table reader classes
-#' @rdname CsvTableReader
-#' @name CsvTableReader
-#' @docType class
-#' @usage NULL
-#' @format NULL
-#' @description `CsvTableReader` and `JsonTableReader` wrap the Arrow C++ CSV
-#' and JSON table readers. See their usage in [read_csv_arrow()] and
-#' [read_json_arrow()], respectively.
-#'
-#' @section Factory:
-#'
-#' The `CsvTableReader$create()` and `JsonTableReader$create()` factory methods
-#' take the following arguments:
-#'
-#' - `file` An Arrow [InputStream]
-#' - `convert_options` (CSV only), `parse_options`, `read_options`: see
-#' [CsvReadOptions]
-#' - `...` additional parameters.
-#'
-#' @section Methods:
-#'
-#' - `$Read()`: returns an Arrow Table.
-#'
-#' @include arrow-package.R
-#' @export
-CsvTableReader <- R6Class("CsvTableReader", inherit = ArrowObject,
- public = list(
- Read = function() csv___TableReader__Read(self)
- )
-)
-CsvTableReader$create <- function(file,
- read_options = CsvReadOptions$create(),
- parse_options = CsvParseOptions$create(),
- convert_options = CsvConvertOptions$create(),
- ...) {
- assert_is(file, "InputStream")
- csv___TableReader__Make(file, read_options, parse_options, convert_options)
-}
-
-#' @title File reader options
-#' @rdname CsvReadOptions
-#' @name CsvReadOptions
-#' @docType class
-#' @usage NULL
-#' @format NULL
-#' @description `CsvReadOptions`, `CsvParseOptions`, `CsvConvertOptions`,
-#' `JsonReadOptions`, `JsonParseOptions`, and `TimestampParser` are containers for various
-#' file reading options. See their usage in [read_csv_arrow()] and
-#' [read_json_arrow()], respectively.
-#'
-#' @section Factory:
-#'
-#' The `CsvReadOptions$create()` and `JsonReadOptions$create()` factory methods
-#' take the following arguments:
-#'
-#' - `use_threads` Whether to use the global CPU thread pool
-#' - `block_size` Block size we request from the IO layer; also determines
-#' the size of chunks when use_threads is `TRUE`. NB: if `FALSE`, JSON input
-#' must end with an empty line.
-#'
-#' `CsvReadOptions$create()` further accepts these additional arguments:
-#'
-#' - `skip_rows` Number of lines to skip before reading data (default 0)
-#' - `column_names` Character vector to supply column names. If length-0
-#' (the default), the first non-skipped row will be parsed to generate column
-#' names, unless `autogenerate_column_names` is `TRUE`.
-#' - `autogenerate_column_names` Logical: generate column names instead of
-#' using the first non-skipped row (the default)? If `TRUE`, column names will
-#' be "f0", "f1", ..., "fN".
-#'
-#' `CsvParseOptions$create()` takes the following arguments:
-#'
-#' - `delimiter` Field delimiting character (default `","`)
-#' - `quoting` Logical: are strings quoted? (default `TRUE`)
-#' - `quote_char` Quoting character, if `quoting` is `TRUE`
-#' - `double_quote` Logical: are quotes inside values double-quoted? (default `TRUE`)
-#' - `escaping` Logical: whether escaping is used (default `FALSE`)
-#' - `escape_char` Escaping character, if `escaping` is `TRUE`
-#' - `newlines_in_values` Logical: are values allowed to contain CR (`0x0d`)
-#' and LF (`0x0a`) characters? (default `FALSE`)
-#' - `ignore_empty_lines` Logical: should empty lines be ignored (default) or
-#' generate a row of missing values (if `FALSE`)?
-#'
-#' `JsonParseOptions$create()` accepts only the `newlines_in_values` argument.
-#'
-#' `CsvConvertOptions$create()` takes the following arguments:
-#'
-#' - `check_utf8` Logical: check UTF8 validity of string columns? (default `TRUE`)
-#' - `null_values` character vector of recognized spellings for null values.
-#' Analogous to the `na.strings` argument to
-#' [`read.csv()`][utils::read.csv()] or `na` in `readr::read_csv()`.
-#' - `strings_can_be_null` Logical: can string / binary columns have
-#' null values? Similar to the `quoted_na` argument to `readr::read_csv()`.
-#' (default `FALSE`)
-#' - `true_values` character vector of recognized spellings for `TRUE` values
-#' - `false_values` character vector of recognized spellings for `FALSE` values
-#' - `col_types` A `Schema` or `NULL` to infer types
-#' - `auto_dict_encode` Logical: Whether to try to automatically
-#' dictionary-encode string / binary data (think `stringsAsFactors`). Default `FALSE`.
-#' This setting is ignored for non-inferred columns (those in `col_types`).
-#' - `auto_dict_max_cardinality` If `auto_dict_encode`, string/binary columns
-#' are dictionary-encoded up to this number of unique values (default 50),
-#' after which it switches to regular encoding.
-#' - `include_columns` If non-empty, indicates the names of columns from the
-#' CSV file that should be actually read and converted (in the vector's order).
-#' - `include_missing_columns` Logical: if `include_columns` is provided, should
-#' columns named in it but not found in the data be included as a column of
-#' type `null()`? The default (`FALSE`) means that the reader will instead
-#' raise an error.
-#' - `timestamp_parsers` User-defined timestamp parsers. If more than one
-#' parser is specified, the CSV conversion logic will try parsing values
-#' starting from the beginning of this vector. Possible values are
-#' (a) `NULL`, the default, which uses the ISO-8601 parser;
-#' (b) a character vector of [strptime][base::strptime()] parse strings; or
-#' (c) a list of [TimestampParser] objects.
-#'
-#' `TimestampParser$create()` takes an optional `format` string argument.
-#' See [`strptime()`][base::strptime()] for example syntax.
-#' The default is to use an ISO-8601 format parser.
-#' @section Active bindings:
-#'
-#' - `column_names`: from `CsvReadOptions`
-#'
-#' @export
-CsvReadOptions <- R6Class("CsvReadOptions",
- inherit = ArrowObject,
- active = list(
- column_names = function() csv___ReadOptions__column_names(self)
- )
-)
-CsvReadOptions$create <- function(use_threads = option_use_threads(),
- block_size = 1048576L,
- skip_rows = 0L,
- column_names = character(0),
- autogenerate_column_names = FALSE) {
- csv___ReadOptions__initialize(
- list(
- use_threads = use_threads,
- block_size = block_size,
- skip_rows = skip_rows,
- column_names = column_names,
- autogenerate_column_names = autogenerate_column_names
- )
- )
-}
-
-readr_to_csv_read_options <- function(skip, col_names, col_types) {
- if (isTRUE(col_names)) {
- # C++ default to parse is 0-length string array
- col_names <- character(0)
- }
- if (identical(col_names, FALSE)) {
- CsvReadOptions$create(skip_rows = skip, autogenerate_column_names = TRUE)
- } else {
- CsvReadOptions$create(skip_rows = skip, column_names = col_names)
- }
-}
-
-#' @rdname CsvReadOptions
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-CsvParseOptions <- R6Class("CsvParseOptions", inherit = ArrowObject)
-CsvParseOptions$create <- function(delimiter = ",",
- quoting = TRUE,
- quote_char = '"',
- double_quote = TRUE,
- escaping = FALSE,
- escape_char = '\\',
- newlines_in_values = FALSE,
- ignore_empty_lines = TRUE) {
-
- csv___ParseOptions__initialize(
- list(
- delimiter = delimiter,
- quoting = quoting,
- quote_char = quote_char,
- double_quote = double_quote,
- escaping = escaping,
- escape_char = escape_char,
- newlines_in_values = newlines_in_values,
- ignore_empty_lines = ignore_empty_lines
- )
- )
-}
-
-readr_to_csv_parse_options <- function(delim = ",",
- quote = '"',
- escape_double = TRUE,
- escape_backslash = FALSE,
- skip_empty_rows = TRUE) {
- # This function translates from the readr argument list to the arrow arg names
- # TODO: validate inputs
- CsvParseOptions$create(
- delimiter = delim,
- quoting = nzchar(quote),
- quote_char = quote,
- double_quote = escape_double,
- escaping = escape_backslash,
- escape_char = '\\',
- newlines_in_values = escape_backslash,
- ignore_empty_lines = skip_empty_rows
- )
-}
-
-#' @rdname CsvReadOptions
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-TimestampParser <- R6Class("TimestampParser", inherit = ArrowObject,
- public = list(
- kind = function() TimestampParser__kind(self),
- format = function() TimestampParser__format(self)
- )
-)
-TimestampParser$create <- function(format = NULL) {
- if (is.null(format)) {
- TimestampParser__MakeISO8601()
- } else {
- TimestampParser__MakeStrptime(format)
- }
-}
-
-#' @rdname CsvReadOptions
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-CsvConvertOptions <- R6Class("CsvConvertOptions", inherit = ArrowObject)
-CsvConvertOptions$create <- function(check_utf8 = TRUE,
- null_values = c("", "NA"),
- true_values = c("T", "true", "TRUE"),
- false_values= c("F", "false", "FALSE"),
- strings_can_be_null = FALSE,
- col_types = NULL,
- auto_dict_encode = FALSE,
- auto_dict_max_cardinality = 50L,
- include_columns = character(),
- include_missing_columns = FALSE,
- timestamp_parsers = NULL) {
-
- if (!is.null(col_types) && !inherits(col_types, "Schema")) {
- abort(c(
- "Unsupported `col_types` specification.",
- i = "`col_types` must be NULL, or a <Schema>."
- ))
- }
-
- csv___ConvertOptions__initialize(
- list(
- check_utf8 = check_utf8,
- null_values = null_values,
- strings_can_be_null = strings_can_be_null,
- col_types = col_types,
- true_values = true_values,
- false_values = false_values,
- auto_dict_encode = auto_dict_encode,
- auto_dict_max_cardinality = auto_dict_max_cardinality,
- include_columns = include_columns,
- include_missing_columns = include_missing_columns,
- timestamp_parsers = timestamp_parsers
- )
- )
-}
-
-readr_to_csv_convert_options <- function(na,
- quoted_na,
- col_types = NULL,
- col_names = NULL,
- timestamp_parsers = NULL) {
- include_columns <- character()
-
- if (is.character(col_types)) {
- if (length(col_types) != 1L) {
- abort("`col_types` is a character vector that is not of size 1")
- }
- n <- nchar(col_types)
- specs <- substring(col_types, seq_len(n), seq_len(n))
- if (!is_bare_character(col_names, n)) {
- abort("Compact specification for `col_types` requires `col_names`")
- }
-
- col_types <- set_names(nm = col_names, map2(specs, col_names, ~{
- switch(.x,
- "c" = utf8(),
- "i" = int32(),
- "n" = float64(),
- "d" = float64(),
- "l" = bool(),
- "f" = dictionary(),
- "D" = date32(),
- "T" = time32(),
- "t" = timestamp(),
- "_" = null(),
- "-" = null(),
- "?" = NULL,
- abort("Unsupported compact specification: '", .x,"' for column '", .y, "'")
- )
- }))
- # To "guess" types, omit them from col_types
- col_types <- keep(col_types, ~!is.null(.x))
- col_types <- schema(!!!col_types)
- }
-
- if (!is.null(col_types)) {
- assert_is(col_types, "Schema")
- # If any columns are null(), drop them
- # (by specifying the other columns in include_columns)
- nulls <- map_lgl(col_types$fields, ~.$type$Equals(null()))
- if (any(nulls)) {
- include_columns <- setdiff(col_names, names(col_types)[nulls])
- }
- }
- CsvConvertOptions$create(
- null_values = na,
- strings_can_be_null = quoted_na,
- col_types = col_types,
- timestamp_parsers = timestamp_parsers,
- include_columns = include_columns
- )
-}
diff --git a/r/R/dataset-factory.R b/r/R/dataset-factory.R
deleted file mode 100644
index 0e029cb..0000000
--- a/r/R/dataset-factory.R
+++ /dev/null
@@ -1,169 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include dataset.R
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Dataset
-#' @export
-DatasetFactory <- R6Class("DatasetFactory", inherit = ArrowObject,
- public = list(
- Finish = function(schema = NULL, unify_schemas = FALSE) {
- if (is.null(schema)) {
- dataset___DatasetFactory__Finish1(self, unify_schemas)
- } else {
- assert_is(schema, "Schema")
- dataset___DatasetFactory__Finish2(self, schema)
- }
- },
- Inspect = function(unify_schemas = FALSE) {
- dataset___DatasetFactory__Inspect(self, unify_schemas)
- }
- )
-)
-DatasetFactory$create <- function(x,
- filesystem = NULL,
- format = c("parquet", "arrow", "ipc", "feather", "csv", "tsv", "text"),
- partitioning = NULL,
- ...) {
- if (is_list_of(x, "DatasetFactory")) {
- return(dataset___UnionDatasetFactory__Make(x))
- }
-
- if (is.character(format)) {
- format <- FileFormat$create(match.arg(format), ...)
- } else {
- assert_is(format, "FileFormat")
- }
-
- path_and_fs <- get_paths_and_filesystem(x, filesystem)
- info <- path_and_fs$fs$GetFileInfo(path_and_fs$path)
-
- if (length(info) > 1 || info[[1]]$type == FileType$File) {
- # x looks like a vector of one or more file paths (not a directory path)
- return(FileSystemDatasetFactory$create(path_and_fs$fs, NULL, path_and_fs$path, format))
- }
-
- if (!is.null(partitioning)) {
- if (inherits(partitioning, "Schema")) {
- partitioning <- DirectoryPartitioning$create(partitioning)
- } else if (is.character(partitioning)) {
- # These are the column/field names, and we should autodetect their types
- partitioning <- DirectoryPartitioningFactory$create(partitioning)
- }
- }
-
- selector <- FileSelector$create(path_and_fs$path, allow_not_found = FALSE, recursive = TRUE)
-
- FileSystemDatasetFactory$create(path_and_fs$fs, selector, NULL, format, partitioning)
-}
-
-#' Create a DatasetFactory
-#'
-#' A [Dataset] can constructed using one or more [DatasetFactory]s.
-#' This function helps you construct a `DatasetFactory` that you can pass to
-#' [open_dataset()].
-#'
-#' If you would only have a single `DatasetFactory` (for example, you have a
-#' single directory containing Parquet files), you can call `open_dataset()`
-#' directly. Use `dataset_factory()` when you
-#' want to combine different directories, file systems, or file formats.
-#'
-#' @param x A string path to a directory containing data files, a vector of one
-#' one or more string paths to data files, or a list of `DatasetFactory` objects
-#' whose datasets should be combined. If this argument is specified it will be
-#' used to construct a `UnionDatasetFactory` and other arguments will be
-#' ignored.
-#' @param filesystem A [FileSystem] object; if omitted, the `FileSystem` will
-#' be detected from `x`
-#' @param format A [FileFormat] object, or a string identifier of the format of
-#' the files in `x`. Currently supported values:
-#' * "parquet"
-#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
-#' only version 2 files are supported
-#' * "csv"/"text", aliases for the same thing (because comma is the default
-#' delimiter for text files
-#' * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
-#'
-#' Default is "parquet", unless a `delimiter` is also specified, in which case
-#' it is assumed to be "text".
-#' @param partitioning One of
-#' * A `Schema`, in which case the file paths relative to `sources` will be
-#' parsed, and path segments will be matched with the schema fields. For
-#' example, `schema(year = int16(), month = int8())` would create partitions
-#' for file paths like "2019/01/file.parquet", "2019/02/file.parquet", etc.
-#' * A character vector that defines the field names corresponding to those
-#' path segments (that is, you're providing the names that would correspond
-#' to a `Schema` but the types will be autodetected)
-#' * A `HivePartitioning` or `HivePartitioningFactory`, as returned
-#' by [hive_partition()] which parses explicit or autodetected fields from
-#' Hive-style path segments
-#' * `NULL` for no partitioning
-#' @param ... Additional format-specific options, passed to
-#' `FileFormat$create()`. For CSV options, note that you can specify them either
-#' with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
-#' `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.).
-#' Not all `readr` options are currently supported; please file an issue if you
-#' encounter one that `arrow` should support.
-#' @return A `DatasetFactory` object. Pass this to [open_dataset()],
-#' in a list potentially with other `DatasetFactory` objects, to create
-#' a `Dataset`.
-#' @export
-dataset_factory <- DatasetFactory$create
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Dataset
-#' @export
-FileSystemDatasetFactory <- R6Class("FileSystemDatasetFactory",
- inherit = DatasetFactory
-)
-FileSystemDatasetFactory$create <- function(filesystem,
- selector = NULL,
- paths = NULL,
- format,
- partitioning = NULL) {
- assert_is(filesystem, "FileSystem")
- is.null(selector) || assert_is(selector, "FileSelector")
- is.null(paths) || assert_is(paths, "character")
- assert_that(
- xor(is.null(selector), is.null(paths)),
- msg = "Either selector or paths must be specified"
- )
- assert_is(format, "FileFormat")
- if (!is.null(paths)) {
- assert_that(is.null(partitioning), msg = "Partitioning not supported with paths")
- }
-
- if (!is.null(paths)) {
- ptr <- dataset___FileSystemDatasetFactory__Make0(filesystem, paths, format)
- } else if (is.null(partitioning)) {
- ptr <- dataset___FileSystemDatasetFactory__Make1(filesystem, selector, format)
- } else if (inherits(partitioning, "PartitioningFactory")) {
- ptr <- dataset___FileSystemDatasetFactory__Make3(filesystem, selector, format, partitioning)
- } else if (inherits(partitioning, "Partitioning")) {
- ptr <- dataset___FileSystemDatasetFactory__Make2(filesystem, selector, format, partitioning)
- } else {
- stop(
- "Expected 'partitioning' to be NULL, PartitioningFactory or Partitioning",
- call. = FALSE
- )
- }
-
- ptr
-}
diff --git a/r/R/dataset-format.R b/r/R/dataset-format.R
deleted file mode 100644
index 854672b..0000000
--- a/r/R/dataset-format.R
+++ /dev/null
@@ -1,320 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Dataset file formats
-#'
-#' @description
-#' A `FileFormat` holds information about how to read and parse the files
-#' included in a `Dataset`. There are subclasses corresponding to the supported
-#' file formats (`ParquetFileFormat` and `IpcFileFormat`).
-#'
-#' @section Factory:
-#' `FileFormat$create()` takes the following arguments:
-#' * `format`: A string identifier of the file format. Currently supported values:
-#' * "parquet"
-#' * "ipc"/"arrow"/"feather", all aliases for each other; for Feather, note that
-#' only version 2 files are supported
-#' * "csv"/"text", aliases for the same thing (because comma is the default
-#' delimiter for text files
-#' * "tsv", equivalent to passing `format = "text", delimiter = "\t"`
-#' * `...`: Additional format-specific options
-#'
-#' `format = "parquet"``:
-#' * `dict_columns`: Names of columns which should be read as dictionaries.
-#' * Any Parquet options from [FragmentScanOptions].
-#'
-#' `format = "text"`: see [CsvParseOptions]. Note that you can specify them either
-#' with the Arrow C++ library naming ("delimiter", "quoting", etc.) or the
-#' `readr`-style naming used in [read_csv_arrow()] ("delim", "quote", etc.).
-#' Not all `readr` options are currently supported; please file an issue if
-#' you encounter one that `arrow` should support. Also, the following options are
-#' supported. From [CsvReadOptions]:
-#' * `skip_rows`
-#' * `column_names`
-#' * `autogenerate_column_names`
-#' From [CsvFragmentScanOptions] (these values can be overridden at scan time):
-#' * `convert_options`: a [CsvConvertOptions]
-#' * `block_size`
-#'
-#' It returns the appropriate subclass of `FileFormat` (e.g. `ParquetFileFormat`)
-#' @rdname FileFormat
-#' @name FileFormat
-#' @export
-FileFormat <- R6Class("FileFormat", inherit = ArrowObject,
- active = list(
- # @description
- # Return the `FileFormat`'s type
- type = function() dataset___FileFormat__type_name(self)
- )
-)
-FileFormat$create <- function(format, ...) {
- opt_names <- names(list(...))
- if (format %in% c("csv", "text") || any(opt_names %in% c("delim", "delimiter"))) {
- CsvFileFormat$create(...)
- } else if (format == c("tsv")) {
- CsvFileFormat$create(delimiter = "\t", ...)
- } else if (format == "parquet") {
- ParquetFileFormat$create(...)
- } else if (format %in% c("ipc", "arrow", "feather")) { # These are aliases for the same thing
- dataset___IpcFileFormat__Make()
- } else {
- stop("Unsupported file format: ", format, call. = FALSE)
- }
-}
-
-#' @export
-as.character.FileFormat <- function(x, ...) {
- out <- x$type
- # Slight hack: special case IPC -> feather, otherwise is just the type_name
- ifelse(out == "ipc", "feather", out)
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
-#' @export
-ParquetFileFormat <- R6Class("ParquetFileFormat", inherit = FileFormat)
-ParquetFileFormat$create <- function(...,
- dict_columns = character(0)) {
- options <- ParquetFragmentScanOptions$create(...)
- dataset___ParquetFileFormat__Make(options, dict_columns)
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
-#' @export
-IpcFileFormat <- R6Class("IpcFileFormat", inherit = FileFormat)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileFormat
-#' @export
-CsvFileFormat <- R6Class("CsvFileFormat", inherit = FileFormat)
-CsvFileFormat$create <- function(..., opts = csv_file_format_parse_options(...),
- convert_options = csv_file_format_convert_options(...),
- read_options = csv_file_format_read_options(...)) {
- dataset___CsvFileFormat__Make(opts, convert_options, read_options)
-}
-
-# Support both readr-style option names and Arrow C++ option names
-csv_file_format_parse_options <- function(...) {
- opts <- list(...)
- # Filter out arguments meant for CsvConvertOptions/CsvReadOptions
- convert_opts <- names(formals(CsvConvertOptions$create))
- read_opts <- names(formals(CsvReadOptions$create))
- opts[convert_opts] <- NULL
- opts[read_opts] <- NULL
- opt_names <- names(opts)
- # Catch any readr-style options specified with full option names that are
- # supported by read_delim_arrow() (and its wrappers) but are not yet
- # supported here
- unsup_readr_opts <- setdiff(
- names(formals(read_delim_arrow)),
- names(formals(readr_to_csv_parse_options))
- )
- is_unsup_opt <- opt_names %in% unsup_readr_opts
- unsup_opts <- opt_names[is_unsup_opt]
- if (length(unsup_opts)) {
- stop(
- "The following ",
- ngettext(length(unsup_opts), "option is ", "options are "),
- "supported in \"read_delim_arrow\" functions ",
- "but not yet supported here: ",
- oxford_paste(unsup_opts),
- call. = FALSE
- )
- }
- # Catch any options with full or partial names that do not match any of the
- # recognized Arrow C++ option names or readr-style option names
- arrow_opts <- names(formals(CsvParseOptions$create))
- readr_opts <- names(formals(readr_to_csv_parse_options))
- is_arrow_opt <- !is.na(pmatch(opt_names, arrow_opts))
- is_readr_opt <- !is.na(pmatch(opt_names, readr_opts))
- unrec_opts <- opt_names[!is_arrow_opt & !is_readr_opt]
- if (length(unrec_opts)) {
- stop(
- "Unrecognized ",
- ngettext(length(unrec_opts), "option", "options"),
- ": ",
- oxford_paste(unrec_opts),
- call. = FALSE
- )
- }
- # Catch options with ambiguous partial names (such as "del") that make it
- # unclear whether the user is specifying Arrow C++ options ("delimiter") or
- # readr-style options ("delim")
- is_ambig_opt <- is.na(pmatch(opt_names, c(arrow_opts, readr_opts)))
- ambig_opts <- opt_names[is_ambig_opt]
- if (length(ambig_opts)) {
- stop("Ambiguous ",
- ngettext(length(ambig_opts), "option", "options"),
- ": ",
- oxford_paste(ambig_opts),
- ". Use full argument names",
- call. = FALSE)
- }
- if (any(is_readr_opt)) {
- # Catch cases when the user specifies a mix of Arrow C++ options and
- # readr-style options
- if (!all(is_readr_opt)) {
- stop("Use either Arrow parse options or readr parse options, not both",
- call. = FALSE)
- }
- do.call(readr_to_csv_parse_options, opts) # all options have readr-style names
- } else {
- do.call(CsvParseOptions$create, opts) # all options have Arrow C++ names
- }
-}
-
-csv_file_format_convert_options <- function(...) {
- opts <- list(...)
- # Filter out arguments meant for CsvParseOptions/CsvReadOptions
- arrow_opts <- names(formals(CsvParseOptions$create))
- readr_opts <- names(formals(readr_to_csv_parse_options))
- read_opts <- names(formals(CsvReadOptions$create))
- opts[arrow_opts] <- NULL
- opts[readr_opts] <- NULL
- opts[read_opts] <- NULL
- do.call(CsvConvertOptions$create, opts)
-}
-
-csv_file_format_read_options <- function(...) {
- opts <- list(...)
- # Filter out arguments meant for CsvParseOptions/CsvConvertOptions
- arrow_opts <- names(formals(CsvParseOptions$create))
- readr_opts <- names(formals(readr_to_csv_parse_options))
- convert_opts <- names(formals(CsvConvertOptions$create))
- opts[arrow_opts] <- NULL
- opts[readr_opts] <- NULL
- opts[convert_opts] <- NULL
- do.call(CsvReadOptions$create, opts)
-}
-
-#' Format-specific scan options
-#'
-#' @description
-#' A `FragmentScanOptions` holds options specific to a `FileFormat` and a scan
-#' operation.
-#'
-#' @section Factory:
-#' `FragmentScanOptions$create()` takes the following arguments:
-#' * `format`: A string identifier of the file format. Currently supported values:
-#' * "parquet"
-#' * "csv"/"text", aliases for the same format.
-#' * `...`: Additional format-specific options
-#'
-#' `format = "parquet"``:
-#' * `use_buffered_stream`: Read files through buffered input streams rather than
-#' loading entire row groups at once. This may be enabled
-#' to reduce memory overhead. Disabled by default.
-#' * `buffer_size`: Size of buffered stream, if enabled. Default is 8KB.
-#' * `pre_buffer`: Pre-buffer the raw Parquet data. This can improve performance
-#' on high-latency filesystems. Disabled by default.
-#
-#' `format = "text"`: see [CsvConvertOptions]. Note that options can only be
-#' specified with the Arrow C++ library naming. Also, "block_size" from
-#' [CsvReadOptions] may be given.
-#'
-#' It returns the appropriate subclass of `FragmentScanOptions`
-#' (e.g. `CsvFragmentScanOptions`).
-#' @rdname FragmentScanOptions
-#' @name FragmentScanOptions
-#' @export
-FragmentScanOptions <- R6Class("FragmentScanOptions", inherit = ArrowObject,
- active = list(
- # @description
- # Return the `FragmentScanOptions`'s type
- type = function() dataset___FragmentScanOptions__type_name(self)
- )
-)
-FragmentScanOptions$create <- function(format, ...) {
- opt_names <- names(list(...))
- if (format %in% c("csv", "text", "tsv")) {
- CsvFragmentScanOptions$create(...)
- } else if (format == "parquet") {
- ParquetFragmentScanOptions$create(...)
- } else {
- stop("Unsupported file format: ", format, call. = FALSE)
- }
-}
-
-#' @export
-as.character.FragmentScanOptions <- function(x, ...) {
- x$type
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FragmentScanOptions
-#' @export
-CsvFragmentScanOptions <- R6Class("CsvFragmentScanOptions", inherit = FragmentScanOptions)
-CsvFragmentScanOptions$create <- function(...,
- convert_opts = csv_file_format_convert_options(...),
- read_opts = csv_file_format_read_options(...)) {
- dataset___CsvFragmentScanOptions__Make(convert_opts, read_opts)
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FragmentScanOptions
-#' @export
-ParquetFragmentScanOptions <- R6Class("ParquetFragmentScanOptions", inherit = FragmentScanOptions)
-ParquetFragmentScanOptions$create <- function(use_buffered_stream = FALSE,
- buffer_size = 8196,
- pre_buffer = FALSE) {
- dataset___ParquetFragmentScanOptions__Make(use_buffered_stream, buffer_size, pre_buffer)
-}
-
-#' Format-specific write options
-#'
-#' @description
-#' A `FileWriteOptions` holds write options specific to a `FileFormat`.
-FileWriteOptions <- R6Class("FileWriteOptions", inherit = ArrowObject,
- public = list(
- update = function(...) {
- if (self$type == "parquet") {
- dataset___ParquetFileWriteOptions__update(self,
- ParquetWriterProperties$create(...),
- ParquetArrowWriterProperties$create(...))
- } else if (self$type == "ipc") {
- args <- list(...)
- if (is.null(args$codec)) {
- dataset___IpcFileWriteOptions__update1(self,
- get_ipc_use_legacy_format(args$use_legacy_format),
- get_ipc_metadata_version(args$metadata_version))
- } else {
- dataset___IpcFileWriteOptions__update2(self,
- get_ipc_use_legacy_format(args$use_legacy_format),
- args$codec,
- get_ipc_metadata_version(args$metadata_version))
- }
- }
- invisible(self)
- }
- ),
- active = list(
- type = function() dataset___FileWriteOptions__type_name(self)
- )
-)
-FileWriteOptions$create <- function(format, ...) {
- if (!inherits(format, "FileFormat")) {
- format <- FileFormat$create(format)
- }
- options <- dataset___FileFormat__DefaultWriteOptions(format)
- options$update(...)
-}
diff --git a/r/R/dataset-partition.R b/r/R/dataset-partition.R
deleted file mode 100644
index e40427a..0000000
--- a/r/R/dataset-partition.R
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Define Partitioning for a Dataset
-#'
-#' @description
-#' Pass a `Partitioning` object to a [FileSystemDatasetFactory]'s `$create()`
-#' method to indicate how the file's paths should be interpreted to define
-#' partitioning.
-#'
-#' `DirectoryPartitioning` describes how to interpret raw path segments, in
-#' order. For example, `schema(year = int16(), month = int8())` would define
-#' partitions for file paths like "2019/01/file.parquet",
-#' "2019/02/file.parquet", etc. In this scheme `NULL` values will be skipped. In
-#' the previous example: when writing a dataset if the month was `NA` (or
-#' `NULL`), the files would be placed in "2019/file.parquet". When reading, the
-#' rows in "2019/file.parquet" would return an `NA` for the month column. An
-#' error will be raised if an outer directory is `NULL` and an inner directory
-#' is not.
-#'
-#' `HivePartitioning` is for Hive-style partitioning, which embeds field
-#' names and values in path segments, such as
-#' "/year=2019/month=2/data.parquet". Because fields are named in the path
-#' segments, order does not matter. This partitioning scheme allows `NULL`
-#' values. They will be replaced by a configurable `null_fallback` which
-#' defaults to the string `"__HIVE_DEFAULT_PARTITION__"` when writing. When
-#' reading, the `null_fallback` string will be replaced with `NA`s as
-#' appropriate.
-#'
-#' `PartitioningFactory` subclasses instruct the `DatasetFactory` to detect
-#' partition features from the file paths.
-#' @section Factory:
-#' Both `DirectoryPartitioning$create()` and `HivePartitioning$create()`
-#' methods take a [Schema] as a single input argument. The helper
-#' function [`hive_partition(...)`][hive_partition] is shorthand for
-#' `HivePartitioning$create(schema(...))`.
-#'
-#' With `DirectoryPartitioningFactory$create()`, you can provide just the
-#' names of the path segments (in our example, `c("year", "month")`), and
-#' the `DatasetFactory` will infer the data types for those partition variables.
-#' `HivePartitioningFactory$create()` takes no arguments: both variable names
-#' and their types can be inferred from the file paths. `hive_partition()` with
-#' no arguments returns a `HivePartitioningFactory`.
-#' @name Partitioning
-#' @rdname Partitioning
-#' @export
-Partitioning <- R6Class("Partitioning", inherit = ArrowObject)
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-DirectoryPartitioning <- R6Class("DirectoryPartitioning", inherit = Partitioning)
-DirectoryPartitioning$create <- dataset___DirectoryPartitioning
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-HivePartitioning <- R6Class("HivePartitioning", inherit = Partitioning)
-HivePartitioning$create <- function(schm, null_fallback = NULL) {
- dataset___HivePartitioning(schm, null_fallback = null_fallback_or_default(null_fallback))
-}
-
-#' Construct Hive partitioning
-#'
-#' Hive partitioning embeds field names and values in path segments, such as
-#' "/year=2019/month=2/data.parquet".
-#'
-#' Because fields are named in the path segments, order of fields passed to
-#' `hive_partition()` does not matter.
-#' @param ... named list of [data types][data-type], passed to [schema()]
-#' @param null_fallback character to be used in place of missing values (`NA` or `NULL`)
-#' in partition columns. Default is `"__HIVE_DEFAULT_PARTITION__"`,
-#' which is what Hive uses.
-#' @return A [HivePartitioning][Partitioning], or a `HivePartitioningFactory` if
-#' calling `hive_partition()` with no arguments.
-#' @examples
-#' \dontrun{
-#' hive_partition(year = int16(), month = int8())
-#' }
-#' @export
-hive_partition <- function(..., null_fallback = NULL) {
- schm <- schema(...)
- if (length(schm) == 0) {
- HivePartitioningFactory$create(null_fallback)
- } else {
- HivePartitioning$create(schm, null_fallback)
- }
-}
-
-PartitioningFactory <- R6Class("PartitioningFactory", inherit = ArrowObject)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-DirectoryPartitioningFactory <- R6Class("DirectoryPartitioningFactory ", inherit = PartitioningFactory)
-DirectoryPartitioningFactory$create <- dataset___DirectoryPartitioning__MakeFactory
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Partitioning
-#' @export
-HivePartitioningFactory <- R6Class("HivePartitioningFactory", inherit = PartitioningFactory)
-HivePartitioningFactory$create <- function(null_fallback = NULL) {
- dataset___HivePartitioning__MakeFactory(null_fallback_or_default(null_fallback))
-}
-
-null_fallback_or_default <- function(null_fallback) {
- null_fallback %||% "__HIVE_DEFAULT_PARTITION__"
-}
diff --git a/r/R/dataset-scan.R b/r/R/dataset-scan.R
deleted file mode 100644
index 750401e..0000000
--- a/r/R/dataset-scan.R
+++ /dev/null
@@ -1,202 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Scan the contents of a dataset
-#'
-#' @description
-#' A `Scanner` iterates over a [Dataset]'s fragments and returns data
-#' according to given row filtering and column projection. A `ScannerBuilder`
-#' can help create one.
-#'
-#' @section Factory:
-#' `Scanner$create()` wraps the `ScannerBuilder` interface to make a `Scanner`.
-#' It takes the following arguments:
-#'
-#' * `dataset`: A `Dataset` or `arrow_dplyr_query` object, as returned by the
-#' `dplyr` methods on `Dataset`.
-#' * `projection`: A character vector of column names to select
-#' * `filter`: A `Expression` to filter the scanned rows by, or `TRUE` (default)
-#' to keep all rows.
-#' * `use_threads`: logical: should scanning use multithreading? Default `TRUE`
-#' * `...`: Additional arguments, currently ignored
-#' @section Methods:
-#' `ScannerBuilder` has the following methods:
-#'
-#' - `$Project(cols)`: Indicate that the scan should only return columns given
-#' by `cols`, a character vector of column names
-#' - `$Filter(expr)`: Filter rows by an [Expression].
-#' - `$UseThreads(threads)`: logical: should the scan use multithreading?
-#' The method's default input is `TRUE`, but you must call the method to enable
-#' multithreading because the scanner default is `FALSE`.
-#' - `$BatchSize(batch_size)`: integer: Maximum row count of scanned record
-#' batches, default is 32K. If scanned record batches are overflowing memory
-#' then this method can be called to reduce their size.
-#' - `$schema`: Active binding, returns the [Schema] of the Dataset
-#' - `$Finish()`: Returns a `Scanner`
-#'
-#' `Scanner` currently has a single method, `$ToTable()`, which evaluates the
-#' query and returns an Arrow [Table].
-#' @rdname Scanner
-#' @name Scanner
-#' @export
-Scanner <- R6Class("Scanner", inherit = ArrowObject,
- public = list(
- ToTable = function() dataset___Scanner__ToTable(self),
- ScanBatches = function() dataset___Scanner__ScanBatches(self)
- ),
- active = list(
- schema = function() dataset___Scanner__schema(self)
- )
-)
-Scanner$create <- function(dataset,
- projection = NULL,
- filter = TRUE,
- use_threads = option_use_threads(),
- batch_size = NULL,
- fragment_scan_options = NULL,
- ...) {
- if (inherits(dataset, "arrow_dplyr_query")) {
- if (inherits(dataset$.data, "ArrowTabular")) {
- # To handle mutate() on Table/RecordBatch, we need to collect(as_data_frame=FALSE) now
- dataset <- dplyr::collect(dataset, as_data_frame = FALSE)
- }
- return(Scanner$create(
- dataset$.data,
- c(dataset$selected_columns, dataset$temp_columns),
- dataset$filtered_rows,
- use_threads,
- batch_size,
- fragment_scan_options,
- ...
- ))
- }
- if (inherits(dataset, c("data.frame", "RecordBatch", "Table"))) {
- dataset <- InMemoryDataset$create(dataset)
- }
- assert_is(dataset, "Dataset")
-
- scanner_builder <- dataset$NewScan()
- if (use_threads) {
- scanner_builder$UseThreads()
- }
- if (!is.null(projection)) {
- scanner_builder$Project(projection)
- }
- if (!isTRUE(filter)) {
- scanner_builder$Filter(filter)
- }
- if (is_integerish(batch_size)) {
- scanner_builder$BatchSize(batch_size)
- }
- if (!is.null(fragment_scan_options)) {
- scanner_builder$FragmentScanOptions(fragment_scan_options)
- }
- scanner_builder$Finish()
-}
-
-#' @export
-names.Scanner <- function(x) names(x$schema)
-
-ScanTask <- R6Class("ScanTask", inherit = ArrowObject,
- public = list(
- Execute = function() dataset___ScanTask__get_batches(self)
- )
-)
-
-#' Apply a function to a stream of RecordBatches
-#'
-#' As an alternative to calling `collect()` on a `Dataset` query, you can
-#' use this function to access the stream of `RecordBatch`es in the `Dataset`.
-#' This lets you aggregate on each chunk and pull the intermediate results into
-#' a `data.frame` for further aggregation, even if you couldn't fit the whole
-#' `Dataset` result in memory.
-#'
-#' This is experimental and not recommended for production use.
-#'
-#' @param X A `Dataset` or `arrow_dplyr_query` object, as returned by the
-#' `dplyr` methods on `Dataset`.
-#' @param FUN A function or `purrr`-style lambda expression to apply to each
-#' batch
-#' @param ... Additional arguments passed to `FUN`
-#' @param .data.frame logical: collect the resulting chunks into a single
-#' `data.frame`? Default `TRUE`
-#' @export
-map_batches <- function(X, FUN, ..., .data.frame = TRUE) {
- if (.data.frame) {
- lapply <- map_dfr
- }
- scanner <- Scanner$create(ensure_group_vars(X))
- FUN <- as_mapper(FUN)
- # message("Making ScanTasks")
- lapply(scanner$ScanBatches(), function(batch) {
- # message("Processing Batch")
- # TODO: wrap batch in arrow_dplyr_query with X$selected_columns,
- # X$temp_columns, and X$group_by_vars
- # if X is arrow_dplyr_query, if some other arg (.dplyr?) == TRUE
- FUN(batch, ...)
- })
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname Scanner
-#' @export
-ScannerBuilder <- R6Class("ScannerBuilder", inherit = ArrowObject,
- public = list(
- Project = function(cols) {
- # cols is either a character vector or a named list of Expressions
- if (is.character(cols)) {
- dataset___ScannerBuilder__ProjectNames(self, cols)
- } else {
- # If we have expressions, but they all turn out to be field_refs,
- # we can still call the simple method
- field_names <- get_field_names(cols)
- if (all(nzchar(field_names))) {
- dataset___ScannerBuilder__ProjectNames(self, field_names)
- } else {
- # Else, we are projecting/mutating
- dataset___ScannerBuilder__ProjectExprs(self, cols, names(cols))
- }
- }
- self
- },
- Filter = function(expr) {
- assert_is(expr, "Expression")
- dataset___ScannerBuilder__Filter(self, expr)
- self
- },
- UseThreads = function(threads = option_use_threads()) {
- dataset___ScannerBuilder__UseThreads(self, threads)
- self
- },
- BatchSize = function(batch_size) {
- dataset___ScannerBuilder__BatchSize(self, batch_size)
- self
- },
- FragmentScanOptions = function(options) {
- dataset___ScannerBuilder__FragmentScanOptions(self, options)
- self
- },
- Finish = function() dataset___ScannerBuilder__Finish(self)
- ),
- active = list(
- schema = function() dataset___ScannerBuilder__schema(self)
- )
-)
-
-#' @export
-names.ScannerBuilder <- function(x) names(x$schema)
diff --git a/r/R/dataset-write.R b/r/R/dataset-write.R
deleted file mode 100644
index 8c9a1ef..0000000
--- a/r/R/dataset-write.R
+++ /dev/null
@@ -1,99 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Write a dataset
-#'
-#' This function allows you to write a dataset. By writing to more efficient
-#' binary storage formats, and by specifying relevant partitioning, you can
-#' make it much faster to read and query.
-#'
-#' @param dataset [Dataset], [RecordBatch], [Table], `arrow_dplyr_query`, or
-#' `data.frame`. If an `arrow_dplyr_query` or `grouped_df`,
-#' `schema` and `partitioning` will be taken from the result of any `select()`
-#' and `group_by()` operations done on the dataset. `filter()` queries will be
-#' applied to restrict written rows.
-#' Note that `select()`-ed columns may not be renamed.
-#' @param path string path, URI, or `SubTreeFileSystem` referencing a directory
-#' to write to (directory will be created if it does not exist)
-#' @param format a string identifier of the file format. Default is to use
-#' "parquet" (see [FileFormat])
-#' @param partitioning `Partitioning` or a character vector of columns to
-#' use as partition keys (to be written as path segments). Default is to
-#' use the current `group_by()` columns.
-#' @param basename_template string template for the names of files to be written.
-#' Must contain `"{i}"`, which will be replaced with an autoincremented
-#' integer to generate basenames of datafiles. For example, `"part-{i}.feather"`
-#' will yield `"part-0.feather", ...`.
-#' @param hive_style logical: write partition segments as Hive-style
-#' (`key1=value1/key2=value2/file.ext`) or as just bare values. Default is `TRUE`.
-#' @param ... additional format-specific arguments. For available Parquet
-#' options, see [write_parquet()]. The available Feather options are
-#' - `use_legacy_format` logical: write data formatted so that Arrow libraries
-#' versions 0.14 and lower can read it. Default is `FALSE`. You can also
-#' enable this by setting the environment variable `ARROW_PRE_0_15_IPC_FORMAT=1`.
-#' - `metadata_version`: A string like "V5" or the equivalent integer indicating
-#' the Arrow IPC MetadataVersion. Default (NULL) will use the latest version,
-#' unless the environment variable `ARROW_PRE_1_0_METADATA_VERSION=1`, in
-#' which case it will be V4.
-#' - `codec`: A [Codec] which will be used to compress body buffers of written
-#' files. Default (NULL) will not compress body buffers.
-#' - `null_fallback`: character to be used in place of missing values (`NA` or
-#' `NULL`) when using Hive-style partitioning. See [hive_partition()].
-#' @return The input `dataset`, invisibly
-#' @export
-write_dataset <- function(dataset,
- path,
- format = c("parquet", "feather", "arrow", "ipc"),
- partitioning = dplyr::group_vars(dataset),
- basename_template = paste0("part-{i}.", as.character(format)),
- hive_style = TRUE,
- ...) {
- format <- match.arg(format)
- if (inherits(dataset, "arrow_dplyr_query")) {
- if (inherits(dataset$.data, "ArrowTabular")) {
- # collect() to materialize any mutate/rename
- dataset <- dplyr::collect(dataset, as_data_frame = FALSE)
- }
- # We can select a subset of columns but we can't rename them
- if (!all(get_field_names(dataset) == names(dataset$selected_columns))) {
- stop("Renaming columns when writing a dataset is not yet supported", call. = FALSE)
- }
- # partitioning vars need to be in the `select` schema
- dataset <- ensure_group_vars(dataset)
- } else if (inherits(dataset, "grouped_df")) {
- force(partitioning)
- # Drop the grouping metadata before writing; we've already consumed it
- # now to construct `partitioning` and don't want it in the metadata$r
- dataset <- dplyr::ungroup(dataset)
- }
-
- scanner <- Scanner$create(dataset)
- if (!inherits(partitioning, "Partitioning")) {
- partition_schema <- scanner$schema[partitioning]
- if (isTRUE(hive_style)) {
- partitioning <- HivePartitioning$create(partition_schema, null_fallback = list(...)$null_fallback)
- } else {
- partitioning <- DirectoryPartitioning$create(partition_schema)
- }
- }
-
- path_and_fs <- get_path_and_filesystem(path)
- options <- FileWriteOptions$create(format, table = scanner, ...)
-
- dataset___Dataset__Write(options, path_and_fs$fs, path_and_fs$path,
- partitioning, basename_template, scanner)
-}
diff --git a/r/R/dataset.R b/r/R/dataset.R
deleted file mode 100644
index 2666339..0000000
--- a/r/R/dataset.R
+++ /dev/null
@@ -1,320 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Open a multi-file dataset
-#'
-#' Arrow Datasets allow you to query against data that has been split across
-#' multiple files. This sharding of data may indicate partitioning, which
-#' can accelerate queries that only touch some partitions (files). Call
-#' `open_dataset()` to point to a directory of data files and return a
-#' `Dataset`, then use `dplyr` methods to query it.
-#'
-#' @param sources One of:
-#' * a string path or URI to a directory containing data files
-#' * a string path or URI to a single file
-#' * a character vector of paths or URIs to individual data files
-#' * a list of `Dataset` objects as created by this function
-#' * a list of `DatasetFactory` objects as created by [dataset_factory()].
-#'
-#' When `sources` is a vector of file URIs, they must all use the same protocol
-#' and point to files located in the same file system and having the same
-#' format.
-#' @param schema [Schema] for the `Dataset`. If `NULL` (the default), the schema
-#' will be inferred from the data sources.
-#' @param partitioning When `sources` is a directory path/URI, one of:
-#' * a `Schema`, in which case the file paths relative to `sources` will be
-#' parsed, and path segments will be matched with the schema fields. For
-#' example, `schema(year = int16(), month = int8())` would create partitions
-#' for file paths like `"2019/01/file.parquet"`, `"2019/02/file.parquet"`,
-#' etc.
-#' * a character vector that defines the field names corresponding to those
-#' path segments (that is, you're providing the names that would correspond
-#' to a `Schema` but the types will be autodetected)
-#' * a `HivePartitioning` or `HivePartitioningFactory`, as returned
-#' by [hive_partition()] which parses explicit or autodetected fields from
-#' Hive-style path segments
-#' * `NULL` for no partitioning
-#'
-#' The default is to autodetect Hive-style partitions. When `sources` is not a
-#' directory path/URI, `partitioning` is ignored.
-#' @param unify_schemas logical: should all data fragments (files, `Dataset`s)
-#' be scanned in order to create a unified schema from them? If `FALSE`, only
-#' the first fragment will be inspected for its schema. Use this fast path
-#' when you know and trust that all fragments have an identical schema.
-#' The default is `FALSE` when creating a dataset from a directory path/URI or
-#' vector of file paths/URIs (because there may be many files and scanning may
-#' be slow) but `TRUE` when `sources` is a list of `Dataset`s (because there
-#' should be few `Dataset`s in the list and their `Schema`s are already in
-#' memory).
-#' @param ... additional arguments passed to `dataset_factory()` when `sources`
-#' is a directory path/URI or vector of file paths/URIs, otherwise ignored.
-#' These may include `format` to indicate the file format, or other
-#' format-specific options.
-#' @return A [Dataset] R6 object. Use `dplyr` methods on it to query the data,
-#' or call [`$NewScan()`][Scanner] to construct a query directly.
-#' @export
-#' @seealso `vignette("dataset", package = "arrow")`
-#' @include arrow-package.R
-open_dataset <- function(sources,
- schema = NULL,
- partitioning = hive_partition(),
- unify_schemas = NULL,
- ...) {
- if (is_list_of(sources, "Dataset")) {
- if (is.null(schema)) {
- if (is.null(unify_schemas) || isTRUE(unify_schemas)) {
- # Default is to unify schemas here
- schema <- unify_schemas(schemas = map(sources, ~.$schema))
- } else {
- # Take the first one.
- schema <- sources[[1]]$schema
- }
- }
- # Enforce that all datasets have the same schema
- assert_is(schema, "Schema")
- sources <- lapply(sources, function(x) {
- x$schema <- schema
- x
- })
- return(dataset___UnionDataset__create(sources, schema))
- }
- factory <- DatasetFactory$create(sources, partitioning = partitioning, ...)
- # Default is _not_ to inspect/unify schemas
- factory$Finish(schema, isTRUE(unify_schemas))
-}
-
-#' Multi-file datasets
-#'
-#' @description
-#' Arrow Datasets allow you to query against data that has been split across
-#' multiple files. This sharding of data may indicate partitioning, which
-#' can accelerate queries that only touch some partitions (files).
-#'
-#' A `Dataset` contains one or more `Fragments`, such as files, of potentially
-#' differing type and partitioning.
-#'
-#' For `Dataset$create()`, see [open_dataset()], which is an alias for it.
-#'
-#' `DatasetFactory` is used to provide finer control over the creation of `Dataset`s.
-#'
-#' @section Factory:
-#' `DatasetFactory` is used to create a `Dataset`, inspect the [Schema] of the
-#' fragments contained in it, and declare a partitioning.
-#' `FileSystemDatasetFactory` is a subclass of `DatasetFactory` for
-#' discovering files in the local file system, the only currently supported
-#' file system.
-#'
-#' For the `DatasetFactory$create()` factory method, see [dataset_factory()], an
-#' alias for it. A `DatasetFactory` has:
-#'
-#' - `$Inspect(unify_schemas)`: If `unify_schemas` is `TRUE`, all fragments
-#' will be scanned and a unified [Schema] will be created from them; if `FALSE`
-#' (default), only the first fragment will be inspected for its schema. Use this
-#' fast path when you know and trust that all fragments have an identical schema.
-#' - `$Finish(schema, unify_schemas)`: Returns a `Dataset`. If `schema` is provided,
-#' it will be used for the `Dataset`; if omitted, a `Schema` will be created from
-#' inspecting the fragments (files) in the dataset, following `unify_schemas`
-#' as described above.
-#'
-#' `FileSystemDatasetFactory$create()` is a lower-level factory method and
-#' takes the following arguments:
-#' * `filesystem`: A [FileSystem]
-#' * `selector`: Either a [FileSelector] or `NULL`
-#' * `paths`: Either a character vector of file paths or `NULL`
-#' * `format`: A [FileFormat]
-#' * `partitioning`: Either `Partitioning`, `PartitioningFactory`, or `NULL`
-#' @section Methods:
-#'
-#' A `Dataset` has the following methods:
-#' - `$NewScan()`: Returns a [ScannerBuilder] for building a query
-#' - `$schema`: Active binding that returns the [Schema] of the Dataset; you
-#' may also replace the dataset's schema by using `ds$schema <- new_schema`.
-#' This method currently supports only adding, removing, or reordering
-#' fields in the schema: you cannot alter or cast the field types.
-#'
-#' `FileSystemDataset` has the following methods:
-#' - `$files`: Active binding, returns the files of the `FileSystemDataset`
-#' - `$format`: Active binding, returns the [FileFormat] of the `FileSystemDataset`
-#'
-#' `UnionDataset` has the following methods:
-#' - `$children`: Active binding, returns all child `Dataset`s.
-#'
-#' @export
-#' @seealso [open_dataset()] for a simple interface to creating a `Dataset`
-Dataset <- R6Class("Dataset", inherit = ArrowObject,
- public = list(
- # @description
- # Start a new scan of the data
- # @return A [ScannerBuilder]
- NewScan = function() dataset___Dataset__NewScan(self),
- ToString = function() self$schema$ToString()
- ),
- active = list(
- schema = function(schema) {
- if (missing(schema)) {
- dataset___Dataset__schema(self)
- } else {
- assert_is(schema, "Schema")
- invisible(dataset___Dataset__ReplaceSchema(self, schema))
- }
- },
- metadata = function() self$schema$metadata,
- num_rows = function() {
- warning("Number of rows unknown; returning NA", call. = FALSE)
- NA_integer_
- },
- num_cols = function() length(self$schema),
- # @description
- # Return the Dataset's type.
- type = function() dataset___Dataset__type_name(self)
- )
-)
-Dataset$create <- open_dataset
-
-#' @name FileSystemDataset
-#' @rdname Dataset
-#' @export
-FileSystemDataset <- R6Class("FileSystemDataset", inherit = Dataset,
- public = list(
- .class_title = function() {
- nfiles <- length(self$files)
- file_type <- self$format$type
- pretty_file_type <- list(
- parquet = "Parquet",
- ipc = "Feather"
- )[[file_type]]
-
- paste(
- class(self)[[1]],
- "with",
- nfiles,
- pretty_file_type %||% file_type,
- ifelse(nfiles == 1, "file", "files")
- )
- }
- ),
- active = list(
- # @description
- # Return the files contained in this `FileSystemDataset`
- files = function() dataset___FileSystemDataset__files(self),
- # @description
- # Return the format of files in this `Dataset`
- format = function() {
- dataset___FileSystemDataset__format(self)
- },
- # @description
- # Return the filesystem of files in this `Dataset`
- filesystem = function() {
- dataset___FileSystemDataset__filesystem(self)
- },
- num_rows = function() {
- if (inherits(self$format, "ParquetFileFormat")) {
- # It's generally fast enough to skim the files directly
- sum(map_int(self$files, ~ParquetFileReader$create(.x)$num_rows))
- } else {
- # TODO: implement for other file formats
- warning("Number of rows unknown; returning NA", call. = FALSE)
- NA_integer_
- # Could do a scan, picking only the last column, which hopefully is virtual
- # But this is can be slow
- # Scanner$create(self, projection = tail(names(self), 1))$ToTable()$num_rows
- # See also https://issues.apache.org/jira/browse/ARROW-9697
- }
- }
- )
-)
-
-#' @name UnionDataset
-#' @rdname Dataset
-#' @export
-UnionDataset <- R6Class("UnionDataset", inherit = Dataset,
- active = list(
- # @description
- # Return the UnionDataset's child `Dataset`s
- children = function() {
- dataset___UnionDataset__children(self)
- }
- )
-)
-
-#' @name InMemoryDataset
-#' @rdname Dataset
-#' @export
-InMemoryDataset <- R6Class("InMemoryDataset", inherit = Dataset)
-InMemoryDataset$create <- function(x) {
- if (!inherits(x, "Table")) {
- x <- Table$create(x)
- }
- dataset___InMemoryDataset__create(x)
-}
-
-
-#' @export
-names.Dataset <- function(x) names(x$schema)
-
-#' @export
-dim.Dataset <- function(x) c(x$num_rows, x$num_cols)
-
-#' @export
-c.Dataset <- function(...) Dataset$create(list(...))
-
-#' @export
-head.Dataset <- function(x, n = 6L, ...) {
- assert_that(n > 0) # For now
- scanner <- Scanner$create(ensure_group_vars(x))
- dataset___Scanner__head(scanner, n)
-}
-
-#' @export
-tail.Dataset <- function(x, n = 6L, ...) {
- assert_that(n > 0) # For now
- result <- list()
- batch_num <- 0
- scanner <- Scanner$create(ensure_group_vars(x))
- for (batch in rev(dataset___Scanner__ScanBatches(scanner))) {
- batch_num <- batch_num + 1
- result[[batch_num]] <- tail(batch, n)
- n <- n - nrow(batch)
- if (n <= 0) break
- }
- Table$create(!!!rev(result))
-}
-
-#' @export
-`[.Dataset` <- function(x, i, j, ..., drop = FALSE) {
- if (nargs() == 2L) {
- # List-like column extraction (x[i])
- return(x[, i])
- }
- if (!missing(j)) {
- x <- select.Dataset(x, j)
- }
-
- if (!missing(i)) {
- x <- take_dataset_rows(x, i)
- }
- x
-}
-
-take_dataset_rows <- function(x, i) {
- if (!is.numeric(i) || any(i < 0)) {
- stop("Only slicing with positive indices is supported", call. = FALSE)
- }
- scanner <- Scanner$create(ensure_group_vars(x))
- i <- Array$create(i - 1)
- dataset___Scanner__TakeRows(scanner, i)
-}
diff --git a/r/R/deprecated.R b/r/R/deprecated.R
deleted file mode 100644
index e8848c4..0000000
--- a/r/R/deprecated.R
+++ /dev/null
@@ -1,40 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @rdname read_ipc_stream
-#' @export
-read_arrow <- function(file, ...) {
- .Deprecated(msg = "Use 'read_ipc_stream' or 'read_feather' instead.")
- if (inherits(file, "raw")) {
- read_ipc_stream(file, ...)
- } else {
- read_feather(file, ...)
- }
-}
-
-#' @rdname write_ipc_stream
-#' @export
-write_arrow <- function(x, sink, ...) {
- .Deprecated(msg = "Use 'write_ipc_stream' or 'write_feather' instead.")
- if (inherits(sink, "raw")) {
- # HACK for sparklyr
- # Note that this returns a new R raw vector, not the one passed as `sink`
- write_to_raw(x)
- } else {
- write_feather(x, sink, ...)
- }
-}
diff --git a/r/R/dictionary.R b/r/R/dictionary.R
deleted file mode 100644
index b701768..0000000
--- a/r/R/dictionary.R
+++ /dev/null
@@ -1,69 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include type.R
-
-#' @title class DictionaryType
-#'
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @section Methods:
-#'
-#' TODO
-#'
-#' @rdname DictionaryType
-#' @name DictionaryType
-DictionaryType <- R6Class("DictionaryType",
- inherit = FixedWidthType,
- public = list(
- ToString = function() {
- prettier_dictionary_type(DataType__ToString(self))
- }
- ),
- active = list(
- index_type = function() DictionaryType__index_type(self),
- value_type = function() DictionaryType__value_type(self),
- name = function() DictionaryType__name(self),
- ordered = function() DictionaryType__ordered(self)
- )
-)
-DictionaryType$create <- function(index_type = int32(),
- value_type = utf8(),
- ordered = FALSE) {
- assert_is(index_type, "DataType")
- assert_is(value_type, "DataType")
- DictionaryType__initialize(index_type, value_type, ordered)
-}
-
-#' Create a dictionary type
-#'
-#' @param index_type A DataType for the indices (default [int32()])
-#' @param value_type A DataType for the values (default [utf8()])
-#' @param ordered Is this an ordered dictionary (default `FALSE`)?
-#'
-#' @return A [DictionaryType]
-#' @seealso [Other Arrow data types][data-type]
-#' @export
-dictionary <- DictionaryType$create
-
-prettier_dictionary_type <- function(x) {
- # Prettier format the "ordered" attribute
- x <- sub(", ordered=0", "", x)
- sub("ordered=1", "ordered", x)
-}
diff --git a/r/R/dplyr.R b/r/R/dplyr.R
deleted file mode 100644
index 845cb3a..0000000
--- a/r/R/dplyr.R
+++ /dev/null
@@ -1,1101 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include expression.R
-#' @include record-batch.R
-#' @include table.R
-
-arrow_dplyr_query <- function(.data) {
- # An arrow_dplyr_query is a container for an Arrow data object (Table,
- # RecordBatch, or Dataset) and the state of the user's dplyr query--things
- # like selected columns, filters, and group vars.
-
- # For most dplyr methods,
- # method.Table == method.RecordBatch == method.Dataset == method.arrow_dplyr_query
- # This works because the functions all pass .data through arrow_dplyr_query()
- if (inherits(.data, "arrow_dplyr_query")) {
- return(.data)
- }
- structure(
- list(
- .data = .data$clone(),
- # selected_columns is a named list:
- # * contents are references/expressions pointing to the data
- # * names are the names they should be in the end (i.e. this
- # records any renaming)
- selected_columns = make_field_refs(names(.data), dataset = inherits(.data, "Dataset")),
- # filtered_rows will be an Expression
- filtered_rows = TRUE,
- # group_by_vars is a character vector of columns (as renamed)
- # in the data. They will be kept when data is pulled into R.
- group_by_vars = character(),
- # drop_empty_groups is a logical value indicating whether to drop
- # groups formed by factor levels that don't appear in the data. It
- # should be non-null only when the data is grouped.
- drop_empty_groups = NULL,
- # arrange_vars will be a list of expressions named by their associated
- # column names
- arrange_vars = list(),
- # arrange_desc will be a logical vector indicating the sort order for each
- # expression in arrange_vars (FALSE for ascending, TRUE for descending)
- arrange_desc = logical()
- ),
- class = "arrow_dplyr_query"
- )
-}
-
-#' @export
-print.arrow_dplyr_query <- function(x, ...) {
- schm <- x$.data$schema
- cols <- get_field_names(x)
- # If cols are expressions, they won't be in the schema and will be "" in cols
- fields <- map_chr(cols, function(name) {
- if (nzchar(name)) {
- schm$GetFieldByName(name)$ToString()
- } else {
- "expr"
- }
- })
- # Strip off the field names as they are in the dataset and add the renamed ones
- fields <- paste(names(cols), sub("^.*?: ", "", fields), sep = ": ", collapse = "\n")
- cat(class(x$.data)[1], " (query)\n", sep = "")
- cat(fields, "\n", sep = "")
- cat("\n")
- if (!isTRUE(x$filtered_rows)) {
- if (query_on_dataset(x)) {
- filter_string <- x$filtered_rows$ToString()
- } else {
- filter_string <- .format_array_expression(x$filtered_rows)
- }
- cat("* Filter: ", filter_string, "\n", sep = "")
- }
- if (length(x$group_by_vars)) {
- cat("* Grouped by ", paste(x$group_by_vars, collapse = ", "), "\n", sep = "")
- }
- if (length(x$arrange_vars)) {
- if (query_on_dataset(x)) {
- arrange_strings <- map_chr(x$arrange_vars, function(x) x$ToString())
- } else {
- arrange_strings <- map_chr(x$arrange_vars, .format_array_expression)
- }
- cat(
- "* Sorted by ",
- paste(
- paste0(
- arrange_strings,
- " [", ifelse(x$arrange_desc, "desc", "asc"), "]"
- ),
- collapse = ", "
- ),
- "\n",
- sep = ""
- )
- }
- cat("See $.data for the source Arrow object\n")
- invisible(x)
-}
-
-get_field_names <- function(selected_cols) {
- if (inherits(selected_cols, "arrow_dplyr_query")) {
- selected_cols <- selected_cols$selected_columns
- }
- map_chr(selected_cols, function(x) {
- if (inherits(x, "Expression")) {
- out <- x$field_name
- } else if (inherits(x, "array_expression")) {
- out <- x$args$field_name
- } else {
- out <- NULL
- }
- # If x isn't some kind of field reference, out is NULL,
- # but we always need to return a string
- out %||% ""
- })
-}
-
-make_field_refs <- function(field_names, dataset = TRUE) {
- if (dataset) {
- out <- lapply(field_names, Expression$field_ref)
- } else {
- out <- lapply(field_names, function(x) array_expression("array_ref", field_name = x))
- }
- set_names(out, field_names)
-}
-
-# These are the names reflecting all select/rename, not what is in Arrow
-#' @export
-names.arrow_dplyr_query <- function(x) names(x$selected_columns)
-
-#' @export
-dim.arrow_dplyr_query <- function(x) {
- cols <- length(names(x))
-
- if (isTRUE(x$filtered)) {
- rows <- x$.data$num_rows
- } else if (query_on_dataset(x)) {
- warning("Number of rows unknown; returning NA", call. = FALSE)
- # TODO: https://issues.apache.org/jira/browse/ARROW-9697
- rows <- NA_integer_
- } else {
- # Evaluate the filter expression to a BooleanArray and count
- rows <- as.integer(sum(eval_array_expression(x$filtered_rows, x$.data), na.rm = TRUE))
- }
- c(rows, cols)
-}
-
-#' @export
-as.data.frame.arrow_dplyr_query <- function(x, row.names = NULL, optional = FALSE, ...) {
- collect.arrow_dplyr_query(x, as_data_frame = TRUE, ...)
-}
-
-#' @export
-head.arrow_dplyr_query <- function(x, n = 6L, ...) {
- if (query_on_dataset(x)) {
- head.Dataset(x, n, ...)
- } else {
- out <- collect.arrow_dplyr_query(x, as_data_frame = FALSE)
- if (inherits(out, "arrow_dplyr_query")) {
- out$.data <- head(out$.data, n)
- } else {
- out <- head(out, n)
- }
- out
- }
-}
-
-#' @export
-tail.arrow_dplyr_query <- function(x, n = 6L, ...) {
- if (query_on_dataset(x)) {
- tail.Dataset(x, n, ...)
- } else {
- out <- collect.arrow_dplyr_query(x, as_data_frame = FALSE)
- if (inherits(out, "arrow_dplyr_query")) {
- out$.data <- tail(out$.data, n)
- } else {
- out <- tail(out, n)
- }
- out
- }
-}
-
-#' @export
-`[.arrow_dplyr_query` <- function(x, i, j, ..., drop = FALSE) {
- if (query_on_dataset(x)) {
- `[.Dataset`(x, i, j, ..., drop = FALSE)
- } else {
- stop(
- "[ method not implemented for queries. Call 'collect(x, as_data_frame = FALSE)' first",
- call. = FALSE
- )
- }
-}
-
-# The following S3 methods are registered on load if dplyr is present
-tbl_vars.arrow_dplyr_query <- function(x) names(x$selected_columns)
-
-select.arrow_dplyr_query <- function(.data, ...) {
- check_select_helpers(enexprs(...))
- column_select(arrow_dplyr_query(.data), !!!enquos(...))
-}
-select.Dataset <- select.ArrowTabular <- select.arrow_dplyr_query
-
-rename.arrow_dplyr_query <- function(.data, ...) {
- check_select_helpers(enexprs(...))
- column_select(arrow_dplyr_query(.data), !!!enquos(...), .FUN = vars_rename)
-}
-rename.Dataset <- rename.ArrowTabular <- rename.arrow_dplyr_query
-
-column_select <- function(.data, ..., .FUN = vars_select) {
- # .FUN is either tidyselect::vars_select or tidyselect::vars_rename
- # It operates on the names() of selected_columns, i.e. the column names
- # factoring in any renaming that may already have happened
- out <- .FUN(names(.data), !!!enquos(...))
- # Make sure that the resulting selected columns map back to the original data,
- # as in when there are multiple renaming steps
- .data$selected_columns <- set_names(.data$selected_columns[out], names(out))
-
- # If we've renamed columns, we need to project that renaming into other
- # query parameters we've collected
- renamed <- out[names(out) != out]
- if (length(renamed)) {
- # Massage group_by
- gbv <- .data$group_by_vars
- renamed_groups <- gbv %in% renamed
- gbv[renamed_groups] <- names(renamed)[match(gbv[renamed_groups], renamed)]
- .data$group_by_vars <- gbv
- # No need to massage filters because those contain references to Arrow objects
- }
- .data
-}
-
-relocate.arrow_dplyr_query <- function(.data, ..., .before = NULL, .after = NULL) {
- # The code in this function is adapted from the code in dplyr::relocate.data.frame
- # at https://github.com/tidyverse/dplyr/blob/master/R/relocate.R
- # TODO: revisit this after https://github.com/tidyverse/dplyr/issues/5829
- check_select_helpers(c(enexprs(...), enexpr(.before), enexpr(.after)))
-
- .data <- arrow_dplyr_query(.data)
-
- to_move <- eval_select(expr(c(...)), .data$selected_columns)
-
- .before <- enquo(.before)
- .after <- enquo(.after)
- has_before <- !quo_is_null(.before)
- has_after <- !quo_is_null(.after)
-
- if (has_before && has_after) {
- abort("Must supply only one of `.before` and `.after`.")
- } else if (has_before) {
- where <- min(unname(eval_select(.before, .data$selected_columns)))
- if (!where %in% to_move) {
- to_move <- c(to_move, where)
- }
- } else if (has_after) {
- where <- max(unname(eval_select(.after, .data$selected_columns)))
- if (!where %in% to_move) {
- to_move <- c(where, to_move)
- }
- } else {
- where <- 1L
- if (!where %in% to_move) {
- to_move <- c(to_move, where)
- }
- }
-
- lhs <- setdiff(seq2(1, where - 1), to_move)
- rhs <- setdiff(seq2(where + 1, length(.data$selected_columns)), to_move)
-
- pos <- vec_unique(c(lhs, to_move, rhs))
- new_names <- names(pos)
- .data$selected_columns <- .data$selected_columns[pos]
-
- if (!is.null(new_names)) {
- names(.data$selected_columns)[new_names != ""] <- new_names[new_names != ""]
- }
- .data
-}
-relocate.Dataset <- relocate.ArrowTabular <- relocate.arrow_dplyr_query
-
-check_select_helpers <- function(exprs) {
- # Throw an error if unsupported tidyselect selection helpers in `exprs`
- exprs <- lapply(exprs, function(x) if (is_quosure(x)) quo_get_expr(x) else x)
- unsup_select_helpers <- "where"
- funs_in_exprs <- unlist(lapply(exprs, all_funs))
- unsup_funs <- funs_in_exprs[funs_in_exprs %in% unsup_select_helpers]
- if (length(unsup_funs)) {
- stop(
- "Unsupported selection ",
- ngettext(length(unsup_funs), "helper: ", "helpers: "),
- oxford_paste(paste0(unsup_funs, "()"), quote = FALSE),
- call. = FALSE
- )
- }
-}
-
-filter.arrow_dplyr_query <- function(.data, ..., .preserve = FALSE) {
- # TODO something with the .preserve argument
- filts <- quos(...)
- if (length(filts) == 0) {
- # Nothing to do
- return(.data)
- }
-
- .data <- arrow_dplyr_query(.data)
- # tidy-eval the filter expressions inside an Arrow data_mask
- filters <- lapply(filts, arrow_eval, arrow_mask(.data))
- bad_filters <- map_lgl(filters, ~inherits(., "try-error"))
- if (any(bad_filters)) {
- bads <- oxford_paste(map_chr(filts, as_label)[bad_filters], quote = FALSE)
- if (query_on_dataset(.data)) {
- # Abort. We don't want to auto-collect if this is a Dataset because that
- # could blow up, too big.
- stop(
- "Filter expression not supported for Arrow Datasets: ", bads,
- "\nCall collect() first to pull data into R.",
- call. = FALSE
- )
- } else {
- # TODO: only show this in some debug mode?
- warning(
- "Filter expression not implemented in Arrow: ", bads, "; pulling data into R",
- immediate. = TRUE,
- call. = FALSE
- )
- # Set any valid filters first, then collect and then apply the invalid ones in R
- .data <- set_filters(.data, filters[!bad_filters])
- return(dplyr::filter(dplyr::collect(.data), !!!filts[bad_filters]))
- }
- }
-
- set_filters(.data, filters)
-}
-filter.Dataset <- filter.ArrowTabular <- filter.arrow_dplyr_query
-
-arrow_eval <- function (expr, mask) {
- # filter(), mutate(), etc. work by evaluating the quoted `exprs` to generate Expressions
- # with references to Arrays (if .data is Table/RecordBatch) or Fields (if
- # .data is a Dataset).
-
- # This yields an Expression as long as the `exprs` are implemented in Arrow.
- # Otherwise, it returns a try-error
- tryCatch(eval_tidy(expr, mask), error = function(e) {
- # Look for the cases where bad input was given, i.e. this would fail
- # in regular dplyr anyway, and let those raise those as errors;
- # else, for things not supported by Arrow return a "try-error",
- # which we'll handle differently
- msg <- conditionMessage(e)
- patterns <- dplyr_functions$i18ized_error_pattern
- if (is.null(patterns)) {
- patterns <- i18ize_error_messages()
- # Memoize it
- dplyr_functions$i18ized_error_pattern <- patterns
- }
- if (grepl(patterns, msg)) {
- stop(e)
- }
- invisible(structure(msg, class = "try-error", condition = e))
- })
-}
-
-i18ize_error_messages <- function() {
- # Figure out what the error messages will be with this LANGUAGE
- # so that we can look for them
- out <- list(
- obj = tryCatch(eval(parse(text = "X_____X")), error = function(e) conditionMessage(e)),
- fun = tryCatch(eval(parse(text = "X_____X()")), error = function(e) conditionMessage(e))
- )
- paste(map(out, ~sub("X_____X", ".*", .)), collapse = "|")
-}
-
-# Helper to assemble the functions that go in the NSE data mask
-# The only difference between the Dataset and the Table/RecordBatch versions
-# is that they use a different wrapping function (FUN) to hold the unevaluated
-# expression.
-build_function_list <- function(FUN) {
- wrapper <- function(operator) {
- force(operator)
- function(...) FUN(operator, ...)
- }
- all_arrow_funs <- list_compute_functions()
-
- c(
- # Include mappings from R function name spellings
- lapply(set_names(names(.array_function_map)), wrapper),
- # Plus some special handling where it's not 1:1
- cast = function(x, target_type, safe = TRUE, ...) {
- opts <- cast_options(safe, ...)
- opts$to_type <- as_type(target_type)
- FUN("cast", x, options = opts)
- },
- dictionary_encode = function(x, null_encoding_behavior = c("mask", "encode")) {
- null_encoding_behavior <-
- NullEncodingBehavior[[toupper(match.arg(null_encoding_behavior))]]
- FUN(
- "dictionary_encode",
- x,
- options = list(null_encoding_behavior = null_encoding_behavior)
- )
- },
- # as.factor() is mapped in expression.R
- as.character = function(x) {
- FUN("cast", x, options = cast_options(to_type = string()))
- },
- as.double = function(x) {
- FUN("cast", x, options = cast_options(to_type = float64()))
- },
- as.integer = function(x) {
- FUN(
- "cast",
- x,
- options = cast_options(
- to_type = int32(),
- allow_float_truncate = TRUE,
- allow_decimal_truncate = TRUE
- )
- )
- },
- as.integer64 = function(x) {
- FUN(
- "cast",
- x,
- options = cast_options(
- to_type = int64(),
- allow_float_truncate = TRUE,
- allow_decimal_truncate = TRUE
- )
- )
- },
- as.logical = function(x) {
- FUN("cast", x, options = cast_options(to_type = boolean()))
- },
- as.numeric = function(x) {
- FUN("cast", x, options = cast_options(to_type = float64()))
- },
- nchar = function(x, type = "chars", allowNA = FALSE, keepNA = NA) {
- if (allowNA) {
- stop("allowNA = TRUE not supported for Arrow", call. = FALSE)
- }
- if (is.na(keepNA)) {
- keepNA <- !identical(type, "width")
- }
- if (!keepNA) {
- # TODO: I think there is a fill_null kernel we could use, set null to 2
- stop("keepNA = TRUE not supported for Arrow", call. = FALSE)
- }
- if (identical(type, "bytes")) {
- FUN("binary_length", x)
- } else {
- FUN("utf8_length", x)
- }
- },
- str_trim = function(string, side = c("both", "left", "right")) {
- side <- match.arg(side)
- switch(
- side,
- left = FUN("utf8_ltrim_whitespace", string),
- right = FUN("utf8_rtrim_whitespace", string),
- both = FUN("utf8_trim_whitespace", string)
- )
- },
- grepl = arrow_r_string_match_function(FUN),
- str_detect = arrow_stringr_string_match_function(FUN),
- sub = arrow_r_string_replace_function(FUN, 1L),
- gsub = arrow_r_string_replace_function(FUN, -1L),
- str_replace = arrow_stringr_string_replace_function(FUN, 1L),
- str_replace_all = arrow_stringr_string_replace_function(FUN, -1L),
- between = function(x, left, right) {
- x >= left & x <= right
- },
- # Now also include all available Arrow Compute functions,
- # namespaced as arrow_fun
- set_names(
- lapply(all_arrow_funs, wrapper),
- paste0("arrow_", all_arrow_funs)
- )
- )
-}
-
-arrow_r_string_match_function <- function(FUN) {
- function(pattern, x, ignore.case = FALSE, fixed = FALSE) {
- FUN(
- ifelse(fixed && !ignore.case, "match_substring", "match_substring_regex"),
- x,
- options = list(pattern = format_string_pattern(pattern, ignore.case, fixed))
- )
- }
-}
-
-arrow_stringr_string_match_function <- function(FUN) {
- function(string, pattern, negate = FALSE) {
- opts <- get_stringr_pattern_options(enexpr(pattern))
- out <- arrow_r_string_match_function(FUN)(
- pattern = opts$pattern,
- x = string,
- ignore.case = opts$ignore_case,
- fixed = opts$fixed
- )
- if (negate) out <- FUN("invert", out)
- out
- }
-}
-
-arrow_r_string_replace_function <- function(FUN, max_replacements) {
- function(pattern, replacement, x, ignore.case = FALSE, fixed = FALSE) {
- FUN(
- ifelse(fixed && !ignore.case, "replace_substring", "replace_substring_regex"),
- x,
- options = list(
- pattern = format_string_pattern(pattern, ignore.case, fixed),
- replacement = format_string_replacement(replacement, ignore.case, fixed),
- max_replacements = max_replacements
- )
- )
- }
-}
-
-arrow_stringr_string_replace_function <- function(FUN, max_replacements) {
- function(string, pattern, replacement) {
- opts <- get_stringr_pattern_options(enexpr(pattern))
- arrow_r_string_replace_function(FUN, max_replacements)(
- pattern = opts$pattern,
- replacement = replacement,
- x = string,
- ignore.case = opts$ignore_case,
- fixed = opts$fixed
- )
- }
-}
-
-# format `pattern` as needed for case insensitivity and literal matching by RE2
-format_string_pattern <- function(pattern, ignore.case, fixed) {
- # Arrow lacks native support for case-insensitive literal string matching and
- # replacement, so we use the regular expression engine (RE2) to do this.
- # https://github.com/google/re2/wiki/Syntax
- if (ignore.case) {
- if (fixed) {
- # Everything between "\Q" and "\E" is treated as literal text.
- # If the search text contains any literal "\E" strings, make them
- # lowercase so they won't signal the end of the literal text:
- pattern <- gsub("\\E", "\\e", pattern, fixed = TRUE)
- pattern <- paste0("\\Q", pattern, "\\E")
- }
- # Prepend "(?i)" for case-insensitive matching
- pattern <- paste0("(?i)", pattern)
- }
- pattern
-}
-
-# format `replacement` as needed for literal replacement by RE2
-format_string_replacement <- function(replacement, ignore.case, fixed) {
- # Arrow lacks native support for case-insensitive literal string
- # replacement, so we use the regular expression engine (RE2) to do this.
- # https://github.com/google/re2/wiki/Syntax
- if (ignore.case && fixed) {
- # Escape single backslashes in the regex replacement text so they are
- # interpreted as literal backslashes:
- replacement <- gsub("\\", "\\\\", replacement, fixed = TRUE)
- }
- replacement
-}
-
-# this function assigns definitions for the stringr pattern modifier functions
-# (fixed, regex, etc.) in itself, and uses them to evaluate the quoted
-# expression `pattern`
-get_stringr_pattern_options <- function(pattern) {
- fixed <- function(pattern, ignore_case = FALSE, ...) {
- check_dots(...)
- list(pattern = pattern, fixed = TRUE, ignore_case = ignore_case)
- }
- regex <- function(pattern, ignore_case = FALSE, ...) {
- check_dots(...)
- list(pattern = pattern, fixed = FALSE, ignore_case = ignore_case)
- }
- coll <- boundary <- function(...) {
- stop(
- "Pattern modifier `",
- match.call()[[1]],
- "()` is not supported in Arrow",
- call. = FALSE
- )
- }
- check_dots <- function(...) {
- dots <- list(...)
- if (length(dots)) {
- warning(
- "Ignoring pattern modifier ",
- ngettext(length(dots), "argument ", "arguments "),
- "not supported in Arrow: ",
- oxford_paste(names(dots)),
- call. = FALSE
- )
- }
- }
- ensure_opts <- function(opts) {
- if (is.character(opts)) {
- opts <- list(pattern = opts, fixed = TRUE, ignore_case = FALSE)
- }
- opts
- }
- ensure_opts(eval(pattern))
-}
-
-# We'll populate these at package load time.
-dplyr_functions <- NULL
-init_env <- function () {
- dplyr_functions <<- new.env(hash = TRUE)
-}
-init_env()
-
-# Create a data mask for evaluating a dplyr expression
-arrow_mask <- function(.data) {
- if (query_on_dataset(.data)) {
- f_env <- new_environment(dplyr_functions$dataset)
- } else {
- f_env <- new_environment(dplyr_functions$array)
- }
-
- # Add functions that need to error hard and clear.
- # Some R functions will still try to evaluate on an Expression
- # and return NA with a warning
- fail <- function(...) stop("Not implemented")
- for (f in c("mean")) {
- f_env[[f]] <- fail
- }
-
- # Add the column references and make the mask
- out <- new_data_mask(
- new_environment(.data$selected_columns, parent = f_env),
- f_env
- )
- # Then insert the data pronoun
- # TODO: figure out what rlang::as_data_pronoun does/why we should use it
- # (because if we do we get `Error: Can't modify the data pronoun` in mutate())
- out$.data <- .data$selected_columns
- out
-}
-
-set_filters <- function(.data, expressions) {
- if (length(expressions)) {
- # expressions is a list of Expressions. AND them together and set them on .data
- new_filter <- Reduce("&", expressions)
- if (isTRUE(.data$filtered_rows)) {
- # TRUE is default (i.e. no filter yet), so we don't need to & with it
- .data$filtered_rows <- new_filter
- } else {
- .data$filtered_rows <- .data$filtered_rows & new_filter
- }
- }
- .data
-}
-
-collect.arrow_dplyr_query <- function(x, as_data_frame = TRUE, ...) {
- x <- ensure_group_vars(x)
- x <- ensure_arrange_vars(x) # this sets x$temp_columns
- # Pull only the selected rows and cols into R
- if (query_on_dataset(x)) {
- # See dataset.R for Dataset and Scanner(Builder) classes
- tab <- Scanner$create(x)$ToTable()
- } else {
- # This is a Table or RecordBatch
-
- # Filter and select the data referenced in selected columns
- if (isTRUE(x$filtered_rows)) {
- filter <- TRUE
- } else {
- filter <- eval_array_expression(x$filtered_rows, x$.data)
- }
- # TODO: shortcut if identical(names(x$.data), find_array_refs(c(x$selected_columns, x$temp_columns)))?
- tab <- x$.data[
- filter,
- find_array_refs(c(x$selected_columns, x$temp_columns)),
- keep_na = FALSE
- ]
- # Now evaluate those expressions on the filtered table
- cols <- lapply(c(x$selected_columns, x$temp_columns), eval_array_expression, data = tab)
- if (length(cols) == 0) {
- tab <- tab[, integer(0)]
- } else {
- if (inherits(x$.data, "Table")) {
- tab <- Table$create(!!!cols)
- } else {
- tab <- RecordBatch$create(!!!cols)
- }
- }
- }
- # Arrange rows
- if (length(x$arrange_vars) > 0) {
- tab <- tab[
- tab$SortIndices(names(x$arrange_vars), x$arrange_desc),
- names(x$selected_columns), # this omits x$temp_columns from the result
- drop = FALSE
- ]
- }
- if (as_data_frame) {
- df <- as.data.frame(tab)
- tab$invalidate()
- restore_dplyr_features(df, x)
- } else {
- restore_dplyr_features(tab, x)
- }
-}
-collect.ArrowTabular <- function(x, as_data_frame = TRUE, ...) {
- if (as_data_frame) {
- as.data.frame(x, ...)
- } else {
- x
- }
-}
-collect.Dataset <- function(x, ...) dplyr::collect(arrow_dplyr_query(x), ...)
-
-compute.arrow_dplyr_query <- function(x, ...) dplyr::collect(x, as_data_frame = FALSE)
-compute.ArrowTabular <- function(x, ...) x
-compute.Dataset <- compute.arrow_dplyr_query
-
-ensure_group_vars <- function(x) {
- if (inherits(x, "arrow_dplyr_query")) {
- # Before pulling data from Arrow, make sure all group vars are in the projection
- gv <- set_names(setdiff(dplyr::group_vars(x), names(x)))
- if (length(gv)) {
- # Add them back
- x$selected_columns <- c(
- x$selected_columns,
- make_field_refs(gv, dataset = query_on_dataset(.data))
- )
- }
- }
- x
-}
-
-ensure_arrange_vars <- function(x) {
- # The arrange() operation is not performed until later, because:
- # - It must be performed after mutate(), to enable sorting by new columns.
- # - It should be performed after filter() and select(), for efficiency.
- # However, we need users to be able to arrange() by columns and expressions
- # that are *not* returned in the query result. To enable this, we must
- # *temporarily* include these columns and expressions in the projection. We
- # use x$temp_columns to store these. Later, after the arrange() operation has
- # been performed, these are omitted from the result. This differs from the
- # columns in x$group_by_vars which *are* returned in the result.
- x$temp_columns <- x$arrange_vars[!names(x$arrange_vars) %in% names(x$selected_columns)]
- x
-}
-
-restore_dplyr_features <- function(df, query) {
- # An arrow_dplyr_query holds some attributes that Arrow doesn't know about
- # After calling collect(), make sure these features are carried over
-
- grouped <- length(query$group_by_vars) > 0
- renamed <- ncol(df) && !identical(names(df), names(query))
- if (renamed) {
- # In case variables were renamed, apply those names
- names(df) <- names(query)
- }
- if (grouped) {
- # Preserve groupings, if present
- if (is.data.frame(df)) {
- df <- dplyr::grouped_df(
- df,
- dplyr::group_vars(query),
- drop = dplyr::group_by_drop_default(query)
- )
- } else {
- # This is a Table, via compute() or collect(as_data_frame = FALSE)
- df <- arrow_dplyr_query(df)
- df$group_by_vars <- query$group_by_vars
- df$drop_empty_groups <- query$drop_empty_groups
- }
- }
- df
-}
-
-pull.arrow_dplyr_query <- function(.data, var = -1) {
- .data <- arrow_dplyr_query(.data)
- var <- vars_pull(names(.data), !!enquo(var))
- .data$selected_columns <- set_names(.data$selected_columns[var], var)
- dplyr::collect(.data)[[1]]
-}
-pull.Dataset <- pull.ArrowTabular <- pull.arrow_dplyr_query
-
-summarise.arrow_dplyr_query <- function(.data, ...) {
- call <- match.call()
- .data <- arrow_dplyr_query(.data)
- if (query_on_dataset(.data)) {
- not_implemented_for_dataset("summarize()")
- }
- exprs <- quos(...)
- # Only retain the columns we need to do our aggregations
- vars_to_keep <- unique(c(
- unlist(lapply(exprs, all.vars)), # vars referenced in summarise
- dplyr::group_vars(.data) # vars needed for grouping
- ))
- .data <- dplyr::select(.data, vars_to_keep)
- if (isTRUE(getOption("arrow.summarize", FALSE))) {
- # Try stuff, if successful return()
- out <- try(do_arrow_group_by(.data, ...), silent = TRUE)
- if (inherits(out, "try-error")) {
- return(abandon_ship(call, .data, format(out)))
- } else {
- return(out)
- }
- } else {
- # If unsuccessful or if option not set, do the work in R
- dplyr::summarise(dplyr::collect(.data), ...)
- }
-}
-summarise.Dataset <- summarise.ArrowTabular <- summarise.arrow_dplyr_query
-
-do_arrow_group_by <- function(.data, ...) {
- exprs <- quos(...)
- mask <- arrow_mask(.data)
- # Add aggregation wrappers to arrow_mask somehow
- # (this is not ideal, would overwrite same-named objects)
- mask$sum <- function(x, na.rm = FALSE) {
- list(
- fun = "sum",
- data = x,
- options = list(na.rm = na.rm)
- )
- }
- results <- list()
- for (i in seq_along(exprs)) {
- # Iterate over the indices and not the names because names may be repeated
- # (which overwrites the previous name)
- new_var <- names(exprs)[i]
- results[[new_var]] <- arrow_eval(exprs[[i]], mask)
- if (inherits(results[[new_var]], "try-error")) {
- msg <- paste('Expression', as_label(exprs[[i]]), 'not supported in Arrow')
- stop(msg, call. = FALSE)
- }
- # Put it in the data mask too?
- #mask[[new_var]] <- mask$.data[[new_var]] <- results[[new_var]]
- }
- # Now, from that, split out the array (expressions) and options
- opts <- lapply(results, function(x) x[c("fun", "options")])
- inputs <- lapply(results, function(x) eval_array_expression(x$data, .data$.data))
- grouping_vars <- lapply(.data$group_by_vars, function(x) eval_array_expression(.data$selected_columns[[x]], .data$.data))
- compute__GroupBy(inputs, grouping_vars, opts)
-}
-
-group_by.arrow_dplyr_query <- function(.data,
- ...,
- .add = FALSE,
- add = .add,
- .drop = dplyr::group_by_drop_default(.data)) {
- .data <- arrow_dplyr_query(.data)
- # ... can contain expressions (i.e. can add (or rename?) columns)
- # Check for those (they show up as named expressions)
- new_groups <- enquos(...)
- new_groups <- new_groups[nzchar(names(new_groups))]
- if (length(new_groups)) {
- # Add them to the data
- .data <- dplyr::mutate(.data, !!!new_groups)
- }
- if (".add" %in% names(formals(dplyr::group_by))) {
- # dplyr >= 1.0
- gv <- dplyr::group_by_prepare(.data, ..., .add = .add)$group_names
- } else {
- gv <- dplyr::group_by_prepare(.data, ..., add = add)$group_names
- }
- .data$group_by_vars <- gv
- .data$drop_empty_groups <- ifelse(length(gv), .drop, dplyr::group_by_drop_default(.data))
- .data
-}
-group_by.Dataset <- group_by.ArrowTabular <- group_by.arrow_dplyr_query
-
-groups.arrow_dplyr_query <- function(x) syms(dplyr::group_vars(x))
-groups.Dataset <- groups.ArrowTabular <- function(x) NULL
-
-group_vars.arrow_dplyr_query <- function(x) x$group_by_vars
-group_vars.Dataset <- group_vars.ArrowTabular <- function(x) NULL
-
-# the logical literal in the two functions below controls the default value of
-# the .drop argument to group_by()
-group_by_drop_default.arrow_dplyr_query <-
- function(.tbl) .tbl$drop_empty_groups %||% TRUE
-group_by_drop_default.Dataset <- group_by_drop_default.ArrowTabular <-
- function(.tbl) TRUE
-
-ungroup.arrow_dplyr_query <- function(x, ...) {
- x$group_by_vars <- character()
- x$drop_empty_groups <- NULL
- x
-}
-ungroup.Dataset <- ungroup.ArrowTabular <- force
-
-mutate.arrow_dplyr_query <- function(.data,
- ...,
- .keep = c("all", "used", "unused", "none"),
- .before = NULL,
- .after = NULL) {
- call <- match.call()
- exprs <- quos(...)
-
- .keep <- match.arg(.keep)
- .before <- enquo(.before)
- .after <- enquo(.after)
-
- if (.keep %in% c("all", "unused") && length(exprs) == 0) {
- # Nothing to do
- return(.data)
- }
-
- .data <- arrow_dplyr_query(.data)
-
- # Restrict the cases we support for now
- if (length(dplyr::group_vars(.data)) > 0) {
- # mutate() on a grouped dataset does calculations within groups
- # This doesn't matter on scalar ops (arithmetic etc.) but it does
- # for things with aggregations (e.g. subtracting the mean)
- return(abandon_ship(call, .data, 'mutate() on grouped data not supported in Arrow'))
- }
-
- # Check for unnamed expressions and fix if any
- unnamed <- !nzchar(names(exprs))
- # Deparse and take the first element in case they're long expressions
- names(exprs)[unnamed] <- map_chr(exprs[unnamed], as_label)
-
- is_dataset <- query_on_dataset(.data)
- mask <- arrow_mask(.data)
- results <- list()
- for (i in seq_along(exprs)) {
- # Iterate over the indices and not the names because names may be repeated
- # (which overwrites the previous name)
- new_var <- names(exprs)[i]
- results[[new_var]] <- arrow_eval(exprs[[i]], mask)
- if (inherits(results[[new_var]], "try-error")) {
- msg <- paste('Expression', as_label(exprs[[i]]), 'not supported in Arrow')
- return(abandon_ship(call, .data, msg))
- } else if (is_dataset &&
- !inherits(results[[new_var]], "Expression") &&
- !is.null(results[[new_var]])) {
- # We need some wrapping to handle literal values
- if (length(results[[new_var]]) != 1) {
- msg <- paste0('In ', new_var, " = ", as_label(exprs[[i]]), ", only values of size one are recycled")
- return(abandon_ship(call, .data, msg))
- }
- results[[new_var]] <- Expression$scalar(results[[new_var]])
- }
- # Put it in the data mask too
- mask[[new_var]] <- mask$.data[[new_var]] <- results[[new_var]]
- }
-
- old_vars <- names(.data$selected_columns)
- # Note that this is names(exprs) not names(results):
- # if results$new_var is NULL, that means we are supposed to remove it
- new_vars <- names(exprs)
-
- # Assign the new columns into the .data$selected_columns
- for (new_var in new_vars) {
- .data$selected_columns[[new_var]] <- results[[new_var]]
- }
-
- # Deduplicate new_vars and remove NULL columns from new_vars
- new_vars <- intersect(new_vars, names(.data$selected_columns))
-
- # Respect .before and .after
- if (!quo_is_null(.before) || !quo_is_null(.after)) {
- new <- setdiff(new_vars, old_vars)
- .data <- dplyr::relocate(.data, !!new, .before = !!.before, .after = !!.after)
- }
-
- # Respect .keep
- if (.keep == "none") {
- .data$selected_columns <- .data$selected_columns[new_vars]
- } else if (.keep != "all") {
- # "used" or "unused"
- used_vars <- unlist(lapply(exprs, all.vars), use.names = FALSE)
- if (.keep == "used") {
- .data$selected_columns[setdiff(old_vars, used_vars)] <- NULL
- } else {
- # "unused"
- .data$selected_columns[intersect(old_vars, used_vars)] <- NULL
- }
- }
- # Even if "none", we still keep group vars
- ensure_group_vars(.data)
-}
-mutate.Dataset <- mutate.ArrowTabular <- mutate.arrow_dplyr_query
-
-transmute.arrow_dplyr_query <- function(.data, ...) dplyr::mutate(.data, ..., .keep = "none")
-transmute.Dataset <- transmute.ArrowTabular <- transmute.arrow_dplyr_query
-
-# Helper to handle unsupported dplyr features
-# * For Table/RecordBatch, we collect() and then call the dplyr method in R
-# * For Dataset, we just error
-abandon_ship <- function(call, .data, msg = NULL) {
- dplyr_fun_name <- sub("^(.*?)\\..*", "\\1", as.character(call[[1]]))
- if (query_on_dataset(.data)) {
- if (is.null(msg)) {
- # Default message: function not implemented
- not_implemented_for_dataset(paste0(dplyr_fun_name, "()"))
- } else {
- stop(msg, "\nCall collect() first to pull data into R.", call. = FALSE)
- }
- }
-
- # else, collect and call dplyr method
- if (!is.null(msg)) {
- warning(msg, "; pulling data into R", immediate. = TRUE, call. = FALSE)
- }
- call$.data <- dplyr::collect(.data)
- call[[1]] <- get(dplyr_fun_name, envir = asNamespace("dplyr"))
- eval.parent(call, 2)
-}
-
-arrange.arrow_dplyr_query <- function(.data, ..., .by_group = FALSE) {
- call <- match.call()
- exprs <- quos(...)
- if (.by_group) {
- # when the data is is grouped and .by_group is TRUE, order the result by
- # the grouping columns first
- exprs <- c(quos(!!!dplyr::groups(.data)), exprs)
- }
- if (length(exprs) == 0) {
- # Nothing to do
- return(.data)
- }
- .data <- arrow_dplyr_query(.data)
- # find and remove any dplyr::desc() and tidy-eval
- # the arrange expressions inside an Arrow data_mask
- sorts <- vector("list", length(exprs))
- descs <- logical(0)
- mask <- arrow_mask(.data)
- for (i in seq_along(exprs)) {
- x <- find_and_remove_desc(exprs[[i]])
- exprs[[i]] <- x[["quos"]]
- sorts[[i]] <- arrow_eval(exprs[[i]], mask)
- if (inherits(sorts[[i]], "try-error")) {
- msg <- paste('Expression', as_label(exprs[[i]]), 'not supported in Arrow')
- return(abandon_ship(call, .data, msg))
- }
- names(sorts)[i] <- as_label(exprs[[i]])
- descs[i] <- x[["desc"]]
- }
- .data$arrange_vars <- c(sorts, .data$arrange_vars)
- .data$arrange_desc <- c(descs, .data$arrange_desc)
- .data
-}
-arrange.Dataset <- arrange.ArrowTabular <- arrange.arrow_dplyr_query
-
-# Helper to handle desc() in arrange()
-# * Takes a quosure as input
-# * Returns a list with two elements:
-# 1. The quosure with any wrapping parentheses and desc() removed
-# 2. A logical value indicating whether desc() was found
-# * Performs some other validation
-find_and_remove_desc <- function(quosure) {
- expr <- quo_get_expr(quosure)
- descending <- FALSE
- if (length(all.vars(expr)) < 1L) {
- stop(
- "Expression in arrange() does not contain any field names: ",
- deparse(expr),
- call. = FALSE
- )
- }
- # Use a while loop to remove any number of nested pairs of enclosing
- # parentheses and any number of nested desc() calls. In the case of multiple
- # nested desc() calls, each one toggles the sort order.
- while (identical(typeof(expr), "language") && is.call(expr)) {
- if (identical(expr[[1]], quote(`(`))) {
- # remove enclosing parentheses
- expr <- expr[[2]]
- } else if (identical(expr[[1]], quote(desc))) {
- # remove desc() and toggle descending
- expr <- expr[[2]]
- descending <- !descending
- } else {
- break
- }
- }
- return(
- list(
- quos = quo_set_expr(quosure, expr),
- desc = descending
- )
- )
-}
-
-query_on_dataset <- function(x) inherits(x$.data, "Dataset")
-
-not_implemented_for_dataset <- function(method) {
- stop(
- method, " is not currently implemented for Arrow Datasets. ",
- "Call collect() first to pull data into R.",
- call. = FALSE
- )
-}
diff --git a/r/R/enums.R b/r/R/enums.R
deleted file mode 100644
index ae44ccf..0000000
--- a/r/R/enums.R
+++ /dev/null
@@ -1,142 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @export
-`print.arrow-enum` <- function(x, ...){
- NextMethod()
-}
-
-enum <- function(class, ..., .list = list(...)){
- structure(
- .list,
- class = c(class, "arrow-enum")
- )
-}
-
-#' Arrow enums
-#' @name enums
-#' @export
-#' @keywords internal
-TimeUnit <- enum("TimeUnit::type",
- SECOND = 0L, MILLI = 1L, MICRO = 2L, NANO = 3L
-)
-
-#' @rdname enums
-#' @export
-DateUnit <- enum("DateUnit", DAY = 0L, MILLI = 1L)
-
-#' @rdname enums
-#' @export
-Type <- enum("Type::type",
- "NA" = 0L,
- BOOL = 1L,
- UINT8 = 2L,
- INT8 = 3L,
- UINT16 = 4L,
- INT16 = 5L,
- UINT32 = 6L,
- INT32 = 7L,
- UINT64 = 8L,
- INT64 = 9L,
- HALF_FLOAT = 10L,
- FLOAT = 11L,
- DOUBLE = 12L,
- STRING = 13L,
- BINARY = 14L,
- FIXED_SIZE_BINARY = 15L,
- DATE32 = 16L,
- DATE64 = 17L,
- TIMESTAMP = 18L,
- TIME32 = 19L,
- TIME64 = 20L,
- INTERVAL_MONTHS = 21L,
- INTERVAL_DAY_TIME = 22L,
- DECIMAL = 23L,
- DECIMAL256 = 24L,
- LIST = 25L,
- STRUCT = 26L,
- SPARSE_UNION = 27L,
- DENSE_UNION = 28L,
- DICTIONARY = 29L,
- MAP = 30L,
- EXTENSION = 31L,
- FIXED_SIZE_LIST = 32L,
- DURATION = 33L,
- LARGE_STRING = 34L,
- LARGE_BINARY = 35L,
- LARGE_LIST = 36L
-)
-
-#' @rdname enums
-#' @export
-StatusCode <- enum("StatusCode",
- OK = 0L, OutOfMemory = 1L, KeyError = 2L, TypeError = 3L,
- Invalid = 4L, IOError = 5L, CapacityError = 6L, IndexError = 7L,
- UnknownError = 9L, NotImplemented = 10L, SerializationError = 11L,
- PythonError = 12L, RError = 13L,
- PlasmaObjectExists = 20L, PlasmaObjectNotFound = 21L,
- PlasmaStoreFull = 22L, PlasmaObjectAlreadySealed = 23L
-)
-
-#' @rdname enums
-#' @export
-FileMode <- enum("FileMode",
- READ = 0L, WRITE = 1L, READWRITE = 2L
-)
-
-#' @rdname enums
-#' @export
-MessageType <- enum("MessageType",
- NONE = 0L, SCHEMA = 1L, DICTIONARY_BATCH = 2L, RECORD_BATCH = 3L, TENSOR = 4L
-)
-
-#' @rdname enums
-#' @export
-CompressionType <- enum("Compression::type",
- UNCOMPRESSED = 0L, SNAPPY = 1L, GZIP = 2L, BROTLI = 3L, ZSTD = 4L, LZ4 = 5L,
- LZ4_FRAME = 6L, LZO = 7L, BZ2 = 8L
-)
-
-#' @export
-#' @rdname enums
-FileType <- enum("FileType",
- NotFound = 0L, Unknown = 1L, File = 2L, Directory = 3L
-)
-
-#' @export
-#' @rdname enums
-ParquetVersionType <- enum("ParquetVersionType",
- PARQUET_1_0 = 0L, PARQUET_2_0 = 1L
-)
-
-#' @export
-#' @rdname enums
-MetadataVersion <- enum("MetadataVersion",
- V1 = 0L, V2 = 1L, V3 = 2L, V4 = 3L, V5 = 4L
-)
-
-#' @export
-#' @rdname enums
-QuantileInterpolation <- enum("QuantileInterpolation",
- LINEAR = 0L, LOWER = 1L, HIGHER = 2L, NEAREST = 3L, MIDPOINT = 4L
-)
-
-#' @export
-#' @rdname enums
-NullEncodingBehavior <- enum("NullEncodingBehavior",
- ENCODE = 0L, MASK = 1L
-)
diff --git a/r/R/expression.R b/r/R/expression.R
deleted file mode 100644
index b3fdd52..0000000
--- a/r/R/expression.R
+++ /dev/null
@@ -1,346 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrowExports.R
-
-array_expression <- function(FUN,
- ...,
- args = list(...),
- options = empty_named_list()) {
- structure(
- list(
- fun = FUN,
- args = args,
- options = options
- ),
- class = "array_expression"
- )
-}
-
-#' @export
-Ops.ArrowDatum <- function(e1, e2) {
- if (.Generic == "!") {
- eval_array_expression(build_array_expression(.Generic, e1))
- } else if (.Generic %in% names(.array_function_map)) {
- eval_array_expression(build_array_expression(.Generic, e1, e2))
- } else {
- stop(paste0("Unsupported operation on `", class(e1)[1L], "` : "), .Generic, call. = FALSE)
- }
-}
-
-#' @export
-Ops.array_expression <- function(e1, e2) {
- if (.Generic == "!") {
- build_array_expression(.Generic, e1)
- } else {
- build_array_expression(.Generic, e1, e2)
- }
-}
-
-build_array_expression <- function(FUN,
- ...,
- args = list(...),
- options = empty_named_list()) {
- if (FUN == "-" && length(args) == 1L) {
- # Unary -, i.e. just make it negative, and somehow this works
- if (inherits(args[[1]], c("ArrowObject", "array_expression"))) {
- # Make it be 0 - arg
- # TODO(ARROW-11950): do this in C++ compute
- args <- list(0L, args[[1]])
- } else {
- # Somehow this works
- return(-args[[1]])
- }
- }
- args <- lapply(args, .wrap_arrow, FUN)
-
- # In Arrow, "divide" is one function, which does integer division on
- # integer inputs and floating-point division on floats
- if (FUN == "/") {
- # TODO: omg so many ways it's wrong to assume these types
- args <- lapply(args, cast_array_expression, float64())
- } else if (FUN == "%/%") {
- # In R, integer division works like floor(float division)
- out <- build_array_expression("/", args = args, options = options)
- return(cast_array_expression(out, int32(), allow_float_truncate = TRUE))
- } else if (FUN == "%%") {
- # {e1 - e2 * ( e1 %/% e2 )}
- # ^^^ form doesn't work because Ops.Array evaluates eagerly,
- # but we can build that up
- quotient <- build_array_expression("%/%", args = args)
- base <- build_array_expression("*", quotient, args[[2]])
- # this cast is to ensure that the result of this and e1 are the same
- # (autocasting only applies to scalars)
- base <- cast_array_expression(base, args[[1]]$type)
- return(build_array_expression("-", args[[1]], base))
- }
-
- array_expression(.array_function_map[[FUN]] %||% FUN, args = args, options = options)
-}
-
-cast_array_expression <- function(x, to_type, safe = TRUE, ...) {
- opts <- list(
- to_type = to_type,
- allow_int_overflow = !safe,
- allow_time_truncate = !safe,
- allow_float_truncate = !safe
- )
- array_expression("cast", x, options = modifyList(opts, list(...)))
-}
-
-.wrap_arrow <- function(arg, fun) {
- if (!inherits(arg, c("ArrowObject", "array_expression"))) {
- # TODO: Array$create if lengths are equal?
- # TODO: these kernels should autocast like the dataset ones do (e.g. int vs. float)
- if (fun == "%in%") {
- arg <- Array$create(arg)
- } else {
- arg <- Scalar$create(arg)
- }
- }
- arg
-}
-
-.unary_function_map <- list(
- "!" = "invert",
- "as.factor" = "dictionary_encode",
- "is.na" = "is_null",
- "is.nan" = "is_nan",
- # nchar is defined in dplyr.R because it is more complex
- # "nchar" = "utf8_length",
- "tolower" = "utf8_lower",
- "toupper" = "utf8_upper",
- # stringr spellings of those
- "str_length" = "utf8_length",
- "str_to_lower" = "utf8_lower",
- "str_to_upper" = "utf8_upper"
- # str_trim is defined in dplyr.R
-)
-
-.binary_function_map <- list(
- "==" = "equal",
- "!=" = "not_equal",
- ">" = "greater",
- ">=" = "greater_equal",
- "<" = "less",
- "<=" = "less_equal",
- "&" = "and_kleene",
- "|" = "or_kleene",
- "+" = "add_checked",
- "-" = "subtract_checked",
- "*" = "multiply_checked",
- "/" = "divide_checked",
- "%/%" = "divide_checked",
- # we don't actually use divide_checked with `%%`, rather it is rewritten to
- # use %/% above.
- "%%" = "divide_checked",
- "^" = "power_checked",
- "%in%" = "is_in_meta_binary"
-)
-
-.array_function_map <- c(.unary_function_map, .binary_function_map)
-
-eval_array_expression <- function(x, data = NULL) {
- if (!is.null(data)) {
- x <- bind_array_refs(x, data)
- }
- if (!inherits(x, "array_expression")) {
- # Nothing to evaluate
- return(x)
- }
- x$args <- lapply(x$args, function (a) {
- if (inherits(a, "array_expression")) {
- eval_array_expression(a)
- } else {
- a
- }
- })
- if (x$fun == "is_in_meta_binary" && inherits(x$args[[2]], "Scalar")) {
- x$args[[2]] <- Array$create(x$args[[2]])
- }
- call_function(x$fun, args = x$args, options = x$options %||% empty_named_list())
-}
-
-find_array_refs <- function(x) {
- if (identical(x$fun, "array_ref")) {
- out <- x$args$field_name
- } else {
- out <- lapply(x$args, find_array_refs)
- }
- unlist(out)
-}
-
-# Take an array_expression and replace array_refs with arrays/chunkedarrays from data
-bind_array_refs <- function(x, data) {
- if (inherits(x, "array_expression")) {
- if (identical(x$fun, "array_ref")) {
- x <- data[[x$args$field_name]]
- } else {
- x$args <- lapply(x$args, bind_array_refs, data)
- }
- }
- x
-}
-
-#' @export
-is.na.array_expression <- function(x) array_expression("is.na", x)
-
-#' @export
-as.vector.array_expression <- function(x, ...) {
- as.vector(eval_array_expression(x))
-}
-
-#' @export
-print.array_expression <- function(x, ...) {
- cat(.format_array_expression(x), "\n", sep = "")
- invisible(x)
-}
-
-.format_array_expression <- function(x) {
- printed_args <- map_chr(x$args, function(arg) {
- if (inherits(arg, "Scalar")) {
- deparse(as.vector(arg))
- } else if (inherits(arg, "ArrowObject")) {
- paste0("<", class(arg)[1], ">")
- } else if (inherits(arg, "array_expression")) {
- .format_array_expression(arg)
- } else {
- # Should not happen
- deparse(arg)
- }
- })
- if (identical(x$fun, "array_ref")) {
- x$args$field_name
- } else {
- # Prune this for readability
- function_name <- sub("_kleene", "", x$fun)
- paste0(function_name, "(", paste(printed_args, collapse = ", "), ")")
- }
-}
-
-###########
-
-#' Arrow expressions
-#'
-#' @description
-#' `Expression`s are used to define filter logic for passing to a [Dataset]
-#' [Scanner].
-#'
-#' `Expression$scalar(x)` constructs an `Expression` which always evaluates to
-#' the provided scalar (length-1) R value.
-#'
-#' `Expression$field_ref(name)` is used to construct an `Expression` which
-#' evaluates to the named column in the `Dataset` against which it is evaluated.
-#'
-#' `Expression$create(function_name, ..., options)` builds a function-call
-#' `Expression` containing one or more `Expression`s.
-#' @name Expression
-#' @rdname Expression
-#' @export
-Expression <- R6Class("Expression", inherit = ArrowObject,
- public = list(
- ToString = function() dataset___expr__ToString(self),
- cast = function(to_type, safe = TRUE, ...) {
- opts <- list(
- to_type = to_type,
- allow_int_overflow = !safe,
- allow_time_truncate = !safe,
- allow_float_truncate = !safe
- )
- Expression$create("cast", self, options = modifyList(opts, list(...)))
- }
- ),
- active = list(
- field_name = function() dataset___expr__get_field_ref_name(self)
- )
-)
-Expression$create <- function(function_name,
- ...,
- args = list(...),
- options = empty_named_list()) {
- assert_that(is.string(function_name))
- dataset___expr__call(function_name, args, options)
-}
-Expression$field_ref <- function(name) {
- assert_that(is.string(name))
- dataset___expr__field_ref(name)
-}
-Expression$scalar <- function(x) {
- dataset___expr__scalar(Scalar$create(x))
-}
-
-build_dataset_expression <- function(FUN,
- ...,
- args = list(...),
- options = empty_named_list()) {
- if (FUN == "-" && length(args) == 1L) {
- # Unary -, i.e. make it negative
- if (inherits(args[[1]], c("ArrowObject", "Expression"))) {
- # TODO(ARROW-11950): do this in C++ compute
- args <- list(0L, args[[1]])
- } else {
- # Somehow this just works
- return(-args[[1]])
- }
- }
- if (FUN == "%in%") {
- # Special-case %in%, which is different from the Array function name
- expr <- Expression$create("is_in", args[[1]],
- options = list(
- # If args[[2]] is already an Arrow object (like a scalar),
- # this wouldn't work
- value_set = Array$create(args[[2]]),
- skip_nulls = TRUE
- )
- )
- } else {
- args <- lapply(args, function(x) {
- if (!inherits(x, "Expression")) {
- x <- Expression$scalar(x)
- }
- x
- })
-
- # In Arrow, "divide" is one function, which does integer division on
- # integer inputs and floating-point division on floats
- if (FUN == "/") {
- # TODO: omg so many ways it's wrong to assume these types
- args <- lapply(args, function(x) x$cast(float64()))
- } else if (FUN == "%/%") {
- # In R, integer division works like floor(float division)
- out <- build_dataset_expression("/", args = args)
- return(out$cast(int32(), allow_float_truncate = TRUE))
- } else if (FUN == "%%") {
- return(args[[1]] - args[[2]] * ( args[[1]] %/% args[[2]] ))
- }
-
- expr <- Expression$create(.array_function_map[[FUN]] %||% FUN, args = args, options = options)
- }
- expr
-}
-
-#' @export
-Ops.Expression <- function(e1, e2) {
- if (.Generic == "!") {
- build_dataset_expression(.Generic, e1)
- } else {
- build_dataset_expression(.Generic, e1, e2)
- }
-}
-
-#' @export
-is.na.Expression <- function(x) Expression$create("is_null", x)
diff --git a/r/R/feather.R b/r/R/feather.R
deleted file mode 100644
index a978110..0000000
--- a/r/R/feather.R
+++ /dev/null
@@ -1,221 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Write data in the Feather format
-#'
-#' Feather provides binary columnar serialization for data frames.
-#' It is designed to make reading and writing data frames efficient,
-#' and to make sharing data across data analysis languages easy.
-#' This function writes both the original, limited specification of the format
-#' and the version 2 specification, which is the Apache Arrow IPC file format.
-#'
-#' @param x `data.frame`, [RecordBatch], or [Table]
-#' @param sink A string file path, URI, or [OutputStream], or path in a file
-#' system (`SubTreeFileSystem`)
-#' @param version integer Feather file version. Version 2 is the current.
-#' Version 1 is the more limited legacy format.
-#' @param chunk_size For V2 files, the number of rows that each chunk of data
-#' should have in the file. Use a smaller `chunk_size` when you need faster
-#' random row access. Default is 64K. This option is not supported for V1.
-#' @param compression Name of compression codec to use, if any. Default is
-#' "lz4" if LZ4 is available in your build of the Arrow C++ library, otherwise
-#' "uncompressed". "zstd" is the other available codec and generally has better
-#' compression ratios in exchange for slower read and write performance
-#' See [codec_is_available()]. This option is not supported for V1.
-#' @param compression_level If `compression` is "zstd", you may
-#' specify an integer compression level. If omitted, the compression codec's
-#' default compression level is used.
-#'
-#' @return The input `x`, invisibly. Note that if `sink` is an [OutputStream],
-#' the stream will be left open.
-#' @export
-#' @seealso [RecordBatchWriter] for lower-level access to writing Arrow IPC data.
-#' @seealso [Schema] for information about schemas and metadata handling.
-#' @examples
-#' \donttest{
-#' tf <- tempfile()
-#' on.exit(unlink(tf))
-#' write_feather(mtcars, tf)
-#' }
-#' @include arrow-package.R
-write_feather <- function(x,
- sink,
- version = 2,
- chunk_size = 65536L,
- compression = c("default", "lz4", "uncompressed", "zstd"),
- compression_level = NULL) {
- # Handle and validate options before touching data
- version <- as.integer(version)
- assert_that(version %in% 1:2)
- compression <- match.arg(compression)
- chunk_size <- as.integer(chunk_size)
- assert_that(chunk_size > 0)
- if (compression == "default") {
- if (version == 2 && codec_is_available("lz4")) {
- compression <- "lz4"
- } else {
- compression <- "uncompressed"
- }
- }
- if (is.null(compression_level)) {
- # Use -1 as sentinal for "default"
- compression_level <- -1L
- }
- compression_level <- as.integer(compression_level)
- # Now make sure that options make sense together
- if (version == 1) {
- if (chunk_size != 65536L) {
- stop("Feather version 1 does not support the 'chunk_size' option", call. = FALSE)
- }
- if (compression != "uncompressed") {
- stop("Feather version 1 does not support the 'compression' option", call. = FALSE)
- }
- if (compression_level != -1L) {
- stop("Feather version 1 does not support the 'compression_level' option", call. = FALSE)
- }
- }
- if (compression != "zstd" && compression_level != -1L) {
- stop("Can only specify a 'compression_level' when 'compression' is 'zstd'", call. = FALSE)
- }
- # Finally, add 1 to version because 2 means V1 and 3 means V2 :shrug:
- version <- version + 1L
-
- # "lz4" is the convenience
- if (compression == "lz4") {
- compression <- "lz4_frame"
- }
-
- compression <- compression_from_name(compression)
-
- x_out <- x
- if (is.data.frame(x) || inherits(x, "RecordBatch")) {
- x <- Table$create(x)
- }
- assert_is(x, "Table")
-
- if (!inherits(sink, "OutputStream")) {
- sink <- make_output_stream(sink)
- on.exit(sink$close())
- }
- ipc___WriteFeather__Table(sink, x, version, chunk_size, compression, compression_level)
- invisible(x_out)
-}
-
-#' Read a Feather file
-#'
-#' Feather provides binary columnar serialization for data frames.
-#' It is designed to make reading and writing data frames efficient,
-#' and to make sharing data across data analysis languages easy.
-#' This function reads both the original, limited specification of the format
-#' and the version 2 specification, which is the Apache Arrow IPC file format.
-#'
-#' @inheritParams read_ipc_stream
-#' @inheritParams read_delim_arrow
-#' @param ... additional parameters, passed to [make_readable_file()].
-#'
-#' @return A `data.frame` if `as_data_frame` is `TRUE` (the default), or an
-#' Arrow [Table] otherwise
-#'
-#' @export
-#' @seealso [FeatherReader] and [RecordBatchReader] for lower-level access to reading Arrow IPC data.
-#' @examples
-#' \donttest{
-#' tf <- tempfile()
-#' on.exit(unlink(tf))
-#' write_feather(mtcars, tf)
-#' df <- read_feather(tf)
-#' dim(df)
-#' # Can select columns
-#' df <- read_feather(tf, col_select = starts_with("d"))
-#' }
-read_feather <- function(file, col_select = NULL, as_data_frame = TRUE, ...) {
- if (!inherits(file, "RandomAccessFile")) {
- file <- make_readable_file(file, ...)
- on.exit(file$close())
- }
- reader <- FeatherReader$create(file)
-
- col_select <- enquo(col_select)
- columns <- if (!quo_is_null(col_select)) {
- vars_select(names(reader), !!col_select)
- }
-
- out <- tryCatch(
- reader$Read(columns),
- error = read_compressed_error
- )
-
- if (isTRUE(as_data_frame)) {
- out <- as.data.frame(out)
- }
- out
-}
-
-#' @title FeatherReader class
-#' @rdname FeatherReader
-#' @name FeatherReader
-#' @docType class
-#' @usage NULL
-#' @format NULL
-#' @description This class enables you to interact with Feather files. Create
-#' one to connect to a file or other InputStream, and call `Read()` on it to
-#' make an `arrow::Table`. See its usage in [`read_feather()`].
-#'
-#' @section Factory:
-#'
-#' The `FeatherReader$create()` factory method instantiates the object and
-#' takes the following argument:
-#'
-#' - `file` an Arrow file connection object inheriting from `RandomAccessFile`.
-#'
-#' @section Methods:
-#'
-#' - `$Read(columns)`: Returns a `Table` of the selected columns, a vector of
-#' integer indices
-#' - `$column_names`: Active binding, returns the column names in the Feather file
-#' - `$schema`: Active binding, returns the schema of the Feather file
-#' - `$version`: Active binding, returns `1` or `2`, according to the Feather
-#' file version
-#'
-#' @export
-#' @include arrow-package.R
-FeatherReader <- R6Class("FeatherReader", inherit = ArrowObject,
- public = list(
- Read = function(columns) {
- ipc___feather___Reader__Read(self, columns)
- },
- print = function(...) {
- cat("FeatherReader:\n")
- print(self$schema)
- invisible(self)
- }
- ),
- active = list(
- # versions are officially 2 for V1 and 3 for V2 :shrug:
- version = function() ipc___feather___Reader__version(self) - 1L,
- column_names = function() names(self$schema),
- schema = function() ipc___feather___Reader__schema(self)
- )
-)
-
-#' @export
-names.FeatherReader <- function(x) x$column_names
-
-FeatherReader$create <- function(file) {
- assert_is(file, "RandomAccessFile")
- ipc___feather___Reader__Open(file)
-}
diff --git a/r/R/field.R b/r/R/field.R
deleted file mode 100644
index 33549d3..0000000
--- a/r/R/field.R
+++ /dev/null
@@ -1,82 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-#' @title Field class
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @description `field()` lets you create an `arrow::Field` that maps a
-#' [DataType][data-type] to a column name. Fields are contained in
-#' [Schemas][Schema].
-#' @section Methods:
-#'
-#' - `f$ToString()`: convert to a string
-#' - `f$Equals(other)`: test for equality. More naturally called as `f == other`
-#'
-#' @rdname Field
-#' @name Field
-#' @export
-Field <- R6Class("Field", inherit = ArrowObject,
- public = list(
- ToString = function() {
- prettier_dictionary_type(Field__ToString(self))
- },
- Equals = function(other, ...) {
- inherits(other, "Field") && Field__Equals(self, other)
- }
- ),
-
- active = list(
- name = function() {
- Field__name(self)
- },
- nullable = function() {
- Field__nullable(self)
- },
- type = function() {
- Field__type(self)
- }
- )
-)
-Field$create <- function(name, type, metadata) {
- assert_that(inherits(name, "character"), length(name) == 1L)
- type <- as_type(type, name)
- assert_that(missing(metadata), msg = "metadata= is currently ignored")
- Field__initialize(enc2utf8(name), type, TRUE)
-}
-
-#' @param name field name
-#' @param type logical type, instance of [DataType]
-#' @param metadata currently ignored
-#'
-#' @examples
-#' \donttest{
-#' field("x", int32())
-#' }
-#' @rdname Field
-#' @export
-field <- Field$create
-
-.fields <- function(.list) {
- if (length(.list)) {
- assert_that(!is.null(nms <- names(.list)))
- map2(nms, .list, field)
- } else {
- list()
- }
-}
diff --git a/r/R/filesystem.R b/r/R/filesystem.R
deleted file mode 100644
index 3a624fd..0000000
--- a/r/R/filesystem.R
+++ /dev/null
@@ -1,510 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-#' @title FileSystem entry info
-#' @usage NULL
-#' @format NULL
-#'
-#' @section Methods:
-#'
-#' - `base_name()` : The file base name (component after the last directory
-#' separator).
-#' - `extension()` : The file extension
-#'
-#' @section Active bindings:
-#'
-#' - `$type`: The file type
-#' - `$path`: The full file path in the filesystem
-#' - `$size`: The size in bytes, if available. Only regular files are
-#' guaranteed to have a size.
-#' - `$mtime`: The time of last modification, if available.
-#'
-#' @rdname FileInfo
-#' @export
-FileInfo <- R6Class("FileInfo",
- inherit = ArrowObject,
- public = list(
- base_name = function() fs___FileInfo__base_name(self),
- extension = function() fs___FileInfo__extension(self)
- ),
- active = list(
- type = function(type) {
- if (missing(type)) {
- fs___FileInfo__type(self)
- } else {
- fs___FileInfo__set_type(self, type)
- }
- },
- path = function(path) {
- if (missing(path)) {
- fs___FileInfo__path(self)
- } else {
- invisible(fs___FileInfo__set_path(self))
- }
- },
-
- size = function(size) {
- if (missing(size)) {
- fs___FileInfo__size(self)
- } else {
- invisible(fs___FileInfo__set_size(self, size))
- }
- },
-
- mtime = function(time) {
- if (missing(time)) {
- fs___FileInfo__mtime(self)
- } else {
- if (!inherits(time, "POSIXct") && length(time) == 1L) {
- abort("invalid time")
- }
- invisible(fs___FileInfo__set_mtime(self, time))
- }
- }
- )
-)
-
-#' @title file selector
-#' @format NULL
-#'
-#' @section Factory:
-#'
-#' The `$create()` factory method instantiates a `FileSelector` given the 3 fields
-#' described below.
-#'
-#' @section Fields:
-#'
-#' - `base_dir`: The directory in which to select files. If the path exists but
-#' doesn't point to a directory, this should be an error.
-#' - `allow_not_found`: The behavior if `base_dir` doesn't exist in the
-#' filesystem. If `FALSE`, an error is returned. If `TRUE`, an empty
-#' selection is returned
-#' - `recursive`: Whether to recurse into subdirectories.
-#'
-#' @rdname FileSelector
-#' @export
-FileSelector <- R6Class("FileSelector",
- inherit = ArrowObject,
- active = list(
- base_dir = function() fs___FileSelector__base_dir(self),
- allow_not_found = function() fs___FileSelector__allow_not_found(self),
- recursive = function() fs___FileSelector__recursive(self)
- )
-)
-
-FileSelector$create <- function(base_dir, allow_not_found = FALSE, recursive = FALSE) {
- fs___FileSelector__create(clean_path_rel(base_dir), allow_not_found, recursive)
-}
-
-#' @title FileSystem classes
-#' @description `FileSystem` is an abstract file system API,
-#' `LocalFileSystem` is an implementation accessing files
-#' on the local machine. `SubTreeFileSystem` is an implementation that delegates
-#' to another implementation after prepending a fixed base path
-#'
-#' @section Factory:
-#'
-#' `LocalFileSystem$create()` returns the object and takes no arguments.
-#'
-#' `SubTreeFileSystem$create()` takes the following arguments:
-#'
-#' - `base_path`, a string path
-#' - `base_fs`, a `FileSystem` object
-#'
-#' `S3FileSystem$create()` optionally takes arguments:
-#'
-#' - `anonymous`: logical, default `FALSE`. If true, will not attempt to look up
-#' credentials using standard AWS configuration methods.
-#' - `access_key`, `secret_key`: authentication credentials. If one is provided,
-#' the other must be as well. If both are provided, they will override any
-#' AWS configuration set at the environment level.
-#' - `session_token`: optional string for authentication along with
-#' `access_key` and `secret_key`
-#' - `role_arn`: string AWS ARN of an AccessRole. If provided instead of `access_key` and
-#' `secret_key`, temporary credentials will be fetched by assuming this role.
-#' - `session_name`: optional string identifier for the assumed role session.
-#' - `external_id`: optional unique string identifier that might be required
-#' when you assume a role in another account.
-#' - `load_frequency`: integer, frequency (in seconds) with which temporary
-#' credentials from an assumed role session will be refreshed. Default is
-#' 900 (i.e. 15 minutes)
-#' - `region`: AWS region to connect to. If omitted, the AWS library will
-#' provide a sensible default based on client configuration, falling back
-#' to "us-east-1" if no other alternatives are found.
-#' - `endpoint_override`: If non-empty, override region with a connect string
-#' such as "localhost:9000". This is useful for connecting to file systems
-#' that emulate S3.
-#' - `scheme`: S3 connection transport (default "https")
-#' - `background_writes`: logical, whether `OutputStream` writes will be issued
-#' in the background, without blocking (default `TRUE`)
-#'
-#' @section Methods:
-#'
-#' - `$GetFileInfo(x)`: `x` may be a [FileSelector][FileSelector] or a character
-#' vector of paths. Returns a list of [FileInfo][FileInfo]
-#' - `$CreateDir(path, recursive = TRUE)`: Create a directory and subdirectories.
-#' - `$DeleteDir(path)`: Delete a directory and its contents, recursively.
-#' - `$DeleteDirContents(path)`: Delete a directory's contents, recursively.
-#' Like `$DeleteDir()`,
-#' but doesn't delete the directory itself. Passing an empty path (`""`) will
-#' wipe the entire filesystem tree.
-#' - `$DeleteFile(path)` : Delete a file.
-#' - `$DeleteFiles(paths)` : Delete many files. The default implementation
-#' issues individual delete operations in sequence.
-#' - `$Move(src, dest)`: Move / rename a file or directory. If the destination
-#' exists:
-#' if it is a non-empty directory, an error is returned
-#' otherwise, if it has the same type as the source, it is replaced
-#' otherwise, behavior is unspecified (implementation-dependent).
-#' - `$CopyFile(src, dest)`: Copy a file. If the destination exists and is a
-#' directory, an error is returned. Otherwise, it is replaced.
-#' - `$OpenInputStream(path)`: Open an [input stream][InputStream] for
-#' sequential reading.
-#' - `$OpenInputFile(path)`: Open an [input file][RandomAccessFile] for random
-#' access reading.
-#' - `$OpenOutputStream(path)`: Open an [output stream][OutputStream] for
-#' sequential writing.
-#' - `$OpenAppendStream(path)`: Open an [output stream][OutputStream] for
-#' appending.
-#'
-#' @section Active bindings:
-#'
-#' - `$type_name`: string filesystem type name, such as "local", "s3", etc.
-#' - `$region`: string AWS region, for `S3FileSystem` and `SubTreeFileSystem`
-#' containing a `S3FileSystem`
-#' - `$base_fs`: for `SubTreeFileSystem`, the `FileSystem` it contains
-#' - `$base_path`: for `SubTreeFileSystem`, the path in `$base_fs` which is considered
-#' root in this `SubTreeFileSystem`.
-#'
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @rdname FileSystem
-#' @name FileSystem
-#' @export
-FileSystem <- R6Class("FileSystem", inherit = ArrowObject,
- public = list(
- GetFileInfo = function(x) {
- if (inherits(x, "FileSelector")) {
- fs___FileSystem__GetTargetInfos_FileSelector(self, x)
- } else if (is.character(x)){
- fs___FileSystem__GetTargetInfos_Paths(self, clean_path_rel(x))
- } else {
- abort("incompatible type for FileSystem$GetFileInfo()")
- }
- },
-
- CreateDir = function(path, recursive = TRUE) {
- fs___FileSystem__CreateDir(self, clean_path_rel(path), isTRUE(recursive))
- },
-
- DeleteDir = function(path) {
- fs___FileSystem__DeleteDir(self, clean_path_rel(path))
- },
-
- DeleteDirContents = function(path) {
- fs___FileSystem__DeleteDirContents(self, clean_path_rel(path))
- },
-
- DeleteFile = function(path) {
- fs___FileSystem__DeleteFile(self, clean_path_rel(path))
- },
-
- DeleteFiles = function(paths) {
- fs___FileSystem__DeleteFiles(self, clean_path_rel(paths))
- },
-
- Move = function(src, dest) {
- fs___FileSystem__Move(self, clean_path_rel(src), clean_path_rel(dest))
- },
-
- CopyFile = function(src, dest) {
- fs___FileSystem__CopyFile(self, clean_path_rel(src), clean_path_rel(dest))
- },
-
- OpenInputStream = function(path) {
- fs___FileSystem__OpenInputStream(self, clean_path_rel(path))
- },
- OpenInputFile = function(path) {
- fs___FileSystem__OpenInputFile(self, clean_path_rel(path))
- },
- OpenOutputStream = function(path) {
- fs___FileSystem__OpenOutputStream(self, clean_path_rel(path))
- },
- OpenAppendStream = function(path) {
- fs___FileSystem__OpenAppendStream(self, clean_path_rel(path))
- },
-
- # Friendlier R user interface
- path = function(x) SubTreeFileSystem$create(x, self),
- cd = function(x) SubTreeFileSystem$create(x, self),
- ls = function(path = "", ...) {
- selector <- FileSelector$create(path, ...) # ... for recursive = TRUE
- infos <- self$GetFileInfo(selector)
- map_chr(infos, ~.$path)
- # TODO: add full.names argument like base::dir() (default right now is TRUE)
- # TODO: see fs package for glob/regexp filtering
- # TODO: verbose method that shows other attributes as df
- # TODO: print methods for FileInfo, SubTreeFileSystem, S3FileSystem
- }
- ),
- active = list(
- type_name = function() fs___FileSystem__type_name(self)
- )
-)
-FileSystem$from_uri <- function(uri) {
- assert_that(is.string(uri))
- fs___FileSystemFromUri(uri)
-}
-
-get_paths_and_filesystem <- function(x, filesystem = NULL) {
- # Wrapper around FileSystem$from_uri that handles local paths
- # and an optional explicit filesystem
- if (inherits(x, "SubTreeFileSystem")) {
- return(list(fs = x$base_fs, path = x$base_path))
- }
- assert_that(is.character(x))
- are_urls <- are_urls(x)
- if (any(are_urls)) {
- if (!all(are_urls)) {
- stop("Vectors of mixed paths and URIs are not supported", call. = FALSE)
- }
- if (!is.null(filesystem)) {
- # Stop? Can't have URL (which yields a fs) and another fs
- }
- x <- lapply(x, FileSystem$from_uri)
- if (length(unique(map(x, ~class(.$fs)))) > 1) {
- stop(
- "Vectors of URIs for different file systems are not supported",
- call. = FALSE
- )
- }
- fs <- x[[1]]$fs
- path <- map_chr(x, ~.$path) # singular name "path" used for compatibility
- } else {
- fs <- filesystem %||% LocalFileSystem$create()
- if (inherits(fs, "LocalFileSystem")) {
- path <- clean_path_abs(x)
- } else {
- path <- clean_path_rel(x)
- }
- }
- list(
- fs = fs,
- path = path
- )
-}
-
-# variant of the above function that asserts that x is either a scalar string
-# or a SubTreeFileSystem
-get_path_and_filesystem <- function(x, filesystem = NULL) {
- assert_that(is.string(x) || inherits(x, "SubTreeFileSystem"))
- get_paths_and_filesystem(x, filesystem)
-}
-
-is_url <- function(x) is.string(x) && grepl("://", x)
-are_urls <- function(x) if (!is.character(x)) FALSE else grepl("://", x)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileSystem
-#' @export
-LocalFileSystem <- R6Class("LocalFileSystem", inherit = FileSystem)
-LocalFileSystem$create <- function() {
- fs___LocalFileSystem__create()
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileSystem
-#' @importFrom utils modifyList
-#' @export
-S3FileSystem <- R6Class("S3FileSystem", inherit = FileSystem,
- active = list(
- region = function() fs___S3FileSystem__region(self)
- )
-)
-S3FileSystem$create <- function(anonymous = FALSE, ...) {
- args <- list2(...)
- if (anonymous) {
- invalid_args <- intersect(c("access_key", "secret_key", "session_token", "role_arn", "session_name", "external_id", "load_frequency"), names(args))
- if (length(invalid_args)) {
- stop("Cannot specify ", oxford_paste(invalid_args), " when anonymous = TRUE", call. = FALSE)
- }
- } else {
- keys_present <- length(intersect(c("access_key", "secret_key"), names(args)))
- if (keys_present == 1) {
- stop("Key authentication requires both access_key and secret_key", call. = FALSE)
- }
- if ("session_token" %in% names(args) && keys_present != 2) {
- stop(
- "In order to initialize a session with temporary credentials, ",
- "both secret_key and access_key must be provided ",
- "in addition to session_token.",
- call. = FALSE
- )
- }
- arn <- "role_arn" %in% names(args)
- if (keys_present == 2 && arn) {
- stop("Cannot provide both key authentication and role_arn", call. = FALSE)
- }
- arn_extras <- intersect(c("session_name", "external_id", "load_frequency"), names(args))
- if (length(arn_extras) > 0 && !arn) {
- stop("Cannot specify ", oxford_paste(arn_extras), " without providing a role_arn string", call. = FALSE)
- }
- }
- args <- c(modifyList(default_s3_options, args), anonymous = anonymous)
- exec(fs___S3FileSystem__create, !!!args)
-}
-
-default_s3_options <- list(
- access_key = "",
- secret_key = "",
- session_token = "",
- role_arn = "",
- session_name = "",
- external_id = "",
- load_frequency = 900L,
- region = "",
- endpoint_override = "",
- scheme = "",
- background_writes = TRUE
-)
-
-#' Connect to an AWS S3 bucket
-#'
-#' `s3_bucket()` is a convenience function to create an `S3FileSystem` object
-#' that automatically detects the bucket's AWS region and holding onto the its
-#' relative path.
-#'
-#' @param bucket string S3 bucket name or path
-#' @param ... Additional connection options, passed to `S3FileSystem$create()`
-#' @return A `SubTreeFileSystem` containing an `S3FileSystem` and the bucket's
-#' relative path. Note that this function's success does not guarantee that you
-#' are authorized to access the bucket's contents.
-#' @examples
-#' if (arrow_with_s3()) {
-#' bucket <- s3_bucket("ursa-labs-taxi-data")
-#' }
-#' @export
-s3_bucket <- function(bucket, ...) {
- assert_that(is.string(bucket))
- args <- list2(...)
-
- # Use FileSystemFromUri to detect the bucket's region
- if (!is_url(bucket)) {
- bucket <- paste0("s3://", bucket)
- }
- fs_and_path <- FileSystem$from_uri(bucket)
- fs <- fs_and_path$fs
- # If there are no additional S3Options, we can use that filesystem
- # Otherwise, take the region that was detected and make a new fs with the args
- if (length(args)) {
- args$region <- fs$region
- fs <- exec(S3FileSystem$create, !!!args)
- }
- # Return a subtree pointing at that bucket path
- SubTreeFileSystem$create(fs_and_path$path, fs)
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname FileSystem
-#' @export
-SubTreeFileSystem <- R6Class("SubTreeFileSystem", inherit = FileSystem,
- public = list(
- print = function(...) {
- if (inherits(self$base_fs, "LocalFileSystem")) {
- cat("SubTreeFileSystem: ", "file://", self$base_path, "\n", sep = "")
- } else if (inherits(self$base_fs, "S3FileSystem")) {
- cat("SubTreeFileSystem: ", "s3://", self$base_path, "\n", sep = "")
- } else {
- cat("SubTreeFileSystem", "\n", sep = "")
- }
- invisible(self)
- }
- ),
- active = list(
- base_fs = function() {
- fs___SubTreeFileSystem__base_fs(self)
- },
- base_path = function() fs___SubTreeFileSystem__base_path(self)
- )
-)
-SubTreeFileSystem$create <- function(base_path, base_fs = NULL) {
- fs_and_path <- get_path_and_filesystem(base_path, base_fs)
- fs___SubTreeFileSystem__create(fs_and_path$path, fs_and_path$fs)
-}
-
-#' @export
-`$.SubTreeFileSystem` <- function(x, name, ...) {
- # This is to allow delegating methods/properties to the base_fs
- assert_that(is.string(name))
- if (name %in% ls(envir = x)) {
- get(name, x)
- } else if (name %in% ls(envir = x$base_fs)) {
- get(name, x$base_fs)
- } else {
- NULL
- }
-}
-
-#' Copy files between FileSystems
-#'
-#' @param from A string path to a local directory or file, a URI, or a
-#' `SubTreeFileSystem`. Files will be copied recursively from this path.
-#' @param to A string path to a local directory or file, a URI, or a
-#' `SubTreeFileSystem`. Directories will be created as necessary
-#' @param chunk_size The maximum size of block to read before flushing
-#' to the destination file. A larger chunk_size will use more memory while
-#' copying but may help accommodate high latency FileSystems.
-#' @return Nothing: called for side effects in the file system
-#' @export
-#' @examples
-#' \dontrun{
-#' # Copy an S3 bucket's files to a local directory:
-#' copy_files("s3://your-bucket-name", "local-directory")
-#' # Using a FileSystem object
-#' copy_files(s3_bucket("your-bucket-name"), "local-directory")
-#' # Or go the other way, from local to S3
-#' copy_files("local-directory", s3_bucket("your-bucket-name"))
-#' }
-copy_files <- function(from, to, chunk_size = 1024L * 1024L) {
- from <- get_path_and_filesystem(from)
- to <- get_path_and_filesystem(to)
- invisible(fs___CopyFiles(
- from$fs,
- FileSelector$create(from$path, recursive = TRUE),
- to$fs,
- to$path,
- chunk_size,
- option_use_threads()
- ))
-}
-
-clean_path_abs <- function(path) {
- # Make sure we have a valid, absolute, forward-slashed path for passing to Arrow
- normalizePath(path, winslash = "/", mustWork = FALSE)
-}
-
-clean_path_rel <- function(path) {
- # Make sure all path separators are "/", not "\" as on Windows
- path_sep <- ifelse(tolower(Sys.info()[["sysname"]]) == "windows", "\\\\", "/")
- gsub(path_sep, "/", path)
-}
diff --git a/r/R/flight.R b/r/R/flight.R
deleted file mode 100644
index 486c59a..0000000
--- a/r/R/flight.R
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Load a Python Flight server
-#'
-#' @param name string Python module name
-#' @param path file system path where the Python module is found. Default is
-#' to look in the `inst/` directory for included modules.
-#' @export
-load_flight_server <- function(name, path = system.file(package = "arrow")) {
- reticulate::import_from_path(name, path)
-}
-
-#' Connect to a Flight server
-#'
-#' @param host string hostname to connect to
-#' @param port integer port to connect on
-#' @param scheme URL scheme, default is "grpc+tcp"
-#' @return A `pyarrow.flight.FlightClient`.
-#' @export
-flight_connect <- function(host = "localhost", port, scheme = "grpc+tcp") {
- pa <- reticulate::import("pyarrow")
- location <- paste0(scheme, "://", host, ":", port)
- pa$flight$FlightClient(location)
-}
-
-#' Send data to a Flight server
-#'
-#' @param client `pyarrow.flight.FlightClient`, as returned by [flight_connect()]
-#' @param data `data.frame`, [RecordBatch], or [Table] to upload
-#' @param path string identifier to store the data under
-#' @param overwrite logical: if `path` exists on `client` already, should we
-#' replace it with the contents of `data`? Default is `TRUE`; if `FALSE` and
-#' `path` exists, the function will error.
-#' @return `client`, invisibly.
-#' @export
-flight_put <- function(client, data, path, overwrite = TRUE) {
- if (!overwrite && flight_path_exists(client, path)) {
- stop(path, " exists.", call. = FALSE)
- }
- if (is.data.frame(data)) {
- data <- Table$create(data)
- }
- py_data <- reticulate::r_to_py(data)
- writer <- client$do_put(descriptor_for_path(path), py_data$schema)[[1]]
- if (inherits(data, "RecordBatch")) {
- writer$write_batch(py_data)
- } else {
- writer$write_table(py_data)
- }
- writer$close()
- invisible(client)
-}
-
-#' Get data from a Flight server
-#'
-#' @param client `pyarrow.flight.FlightClient`, as returned by [flight_connect()]
-#' @param path string identifier under which data is stored
-#' @return A [Table]
-#' @export
-flight_get <- function(client, path) {
- reader <- flight_reader(client, path)
- reader$read_all()
-}
-
-# TODO: could use this as a RecordBatch iterator, call $read_chunk() on this
-flight_reader <- function(client, path) {
- info <- client$get_flight_info(descriptor_for_path(path))
- # Hack: assume a single ticket, on the same server as client is already connected
- ticket <- info$endpoints[[1]]$ticket
- client$do_get(ticket)
-}
-
-descriptor_for_path <- function(path) {
- pa <- reticulate::import("pyarrow")
- pa$flight$FlightDescriptor$for_path(path)
-}
-
-#' See available resources on a Flight server
-#'
-#' @inheritParams flight_get
-#' @return `list_flights()` returns a character vector of paths.
-#' `flight_path_exists()` returns a logical value, the equivalent of `path %in% list_flights()`
-#' @export
-list_flights <- function(client) {
- generator <- client$list_flights()
- out <- reticulate::iterate(generator, function(x) as.character(x$descriptor$path[[1]]))
- out
-}
-
-#' @rdname list_flights
-#' @export
-flight_path_exists <- function(client, path) {
- it_exists <- tryCatch({
- client$get_flight_info(descriptor_for_path(path))
- TRUE
- },
- error = function(e) {
- msg <- conditionMessage(e)
- if (!any(grepl("ArrowKeyError", msg))) {
- # Raise an error if this fails for any reason other than not found
- stop(e)
- }
- FALSE
- }
- )
-}
diff --git a/r/R/install-arrow.R b/r/R/install-arrow.R
deleted file mode 100644
index 63db8ed..0000000
--- a/r/R/install-arrow.R
+++ /dev/null
@@ -1,139 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Install or upgrade the Arrow library
-#'
-#' Use this function to install the latest release of `arrow`, to switch to or
-#' from a nightly development version, or on Linux to try reinstalling with
-#' all necessary C++ dependencies.
-#'
-#' Note that, unlike packages like `tensorflow`, `blogdown`, and others that
-#' require external dependencies, you do not need to run `install_arrow()`
-#' after a successful `arrow` installation.
-#'
-#' @param nightly logical: Should we install a development version of the
-#' package, or should we install from CRAN (the default).
-#' @param binary On Linux, value to set for the environment variable
-#' `LIBARROW_BINARY`, which governs how C++ binaries are used, if at all.
-#' The default value, `TRUE`, tells the installation script to detect the
-#' Linux distribution and version and find an appropriate C++ library. `FALSE`
-#' would tell the script not to retrieve a binary and instead build Arrow C++
-#' from source. Other valid values are strings corresponding to a Linux
-#' distribution-version, to override the value that would be detected.
-#' See `vignette("install", package = "arrow")` for further details.
-#' @param use_system logical: Should we use `pkg-config` to look for Arrow
-#' system packages? Default is `FALSE`. If `TRUE`, source installation may be
-#' faster, but there is a risk of version mismatch. This sets the
-#' `ARROW_USE_PKG_CONFIG` environment variable.
-#' @param minimal logical: If building from source, should we build without
-#' optional dependencies (compression libraries, for example)? Default is
-#' `FALSE`. This sets the `LIBARROW_MINIMAL` environment variable.
-#' @param verbose logical: Print more debugging output when installing? Default
-#' is `FALSE`. This sets the `ARROW_R_DEV` environment variable.
-#' @param repos character vector of base URLs of the repositories to install
-#' from (passed to `install.packages()`)
-#' @param ... Additional arguments passed to `install.packages()`
-#' @export
-#' @importFrom utils install.packages
-#' @seealso [arrow_available()] to see if the package was configured with
-#' necessary C++ dependencies. `vignette("install", package = "arrow")` for
-#' more ways to tune installation on Linux.
-install_arrow <- function(nightly = FALSE,
- binary = Sys.getenv("LIBARROW_BINARY", TRUE),
- use_system = Sys.getenv("ARROW_USE_PKG_CONFIG", FALSE),
- minimal = Sys.getenv("LIBARROW_MINIMAL", FALSE),
- verbose = Sys.getenv("ARROW_R_DEV", FALSE),
- repos = getOption("repos"),
- ...) {
- sysname <- tolower(Sys.info()[["sysname"]])
- conda <- isTRUE(grepl("conda", R.Version()$platform))
-
- if (conda) {
- if (nightly) {
- system("conda install -y -c arrow-nightlies -c conda-forge --strict-channel-priority r-arrow")
- } else {
- system("conda install -y -c conda-forge --strict-channel-priority r-arrow")
- }
- } else {
- Sys.setenv(
- LIBARROW_DOWNLOAD = "true",
- LIBARROW_BINARY = binary,
- LIBARROW_MINIMAL = minimal,
- ARROW_R_DEV = verbose,
- ARROW_USE_PKG_CONFIG = use_system
- )
- # On the M1, we can't use the usual autobrew, which pulls Intel dependencies
- apple_m1 <- grepl("arm-apple|aarch64.*darwin", R.Version()$platform)
- # On Rosetta, we have to build without JEMALLOC, so we also can't autobrew
- rosetta <- identical(sysname, "darwin") && identical(system("sysctl -n sysctl.proc_translated", intern = TRUE), "1")
- if (rosetta) {
- Sys.setenv(ARROW_JEMALLOC = "OFF")
- }
- if (apple_m1 || rosetta) {
- Sys.setenv(FORCE_BUNDLED_BUILD = "true")
- }
-
- opts <- list()
- if (apple_m1 || rosetta) {
- # Skip binaries (esp. for rosetta)
- opts$pkgType <- "source"
- } else if (isTRUE(binary)) {
- # Unless otherwise directed, don't consider newer source packages when
- # options(pkgType) == "both" (default on win/mac)
- opts$install.packages.check.source <- "no"
- opts$install.packages.compile.from.source <- "never"
- }
- if (length(opts)) {
- old <- options(opts)
- on.exit(options(old))
- }
- install.packages("arrow", repos = arrow_repos(repos, nightly), ...)
- }
- if ("arrow" %in% loadedNamespaces()) {
- # If you've just sourced this file, "arrow" won't be (re)loaded
- reload_arrow()
- }
-}
-
-arrow_repos <- function(repos = getOption("repos"), nightly = FALSE) {
- if (length(repos) == 0 || identical(repos, c(CRAN = "@CRAN@"))) {
- # Set the default/CDN
- repos <- "https://cloud.r-project.org/"
- }
- dev_repo <- getOption("arrow.dev_repo", "https://arrow-r-nightly.s3.amazonaws.com")
- # Remove it if it's there (so nightly=FALSE won't accidentally pull from it)
- repos <- setdiff(repos, dev_repo)
- if (nightly) {
- # Add it first
- repos <- c(dev_repo, repos)
- }
- repos
-}
-
-reload_arrow <- function() {
- if (requireNamespace("pkgload", quietly = TRUE)) {
- is_attached <- "package:arrow" %in% search()
- pkgload::unload("arrow")
- if (is_attached) {
- require("arrow", character.only = TRUE, quietly = TRUE)
- } else {
- requireNamespace("arrow", quietly = TRUE)
- }
- } else {
- message("Please restart R to use the 'arrow' package.")
- }
-}
diff --git a/r/R/io.R b/r/R/io.R
deleted file mode 100644
index 5f015ce..0000000
--- a/r/R/io.R
+++ /dev/null
@@ -1,290 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-#' @include enums.R
-#' @include buffer.R
-
-# OutputStream ------------------------------------------------------------
-
-Writable <- R6Class("Writable", inherit = ArrowObject,
- public = list(
- write = function(x) io___Writable__write(self, buffer(x))
- )
-)
-
-#' @title OutputStream classes
-#' @description `FileOutputStream` is for writing to a file;
-#' `BufferOutputStream` writes to a buffer;
-#' You can create one and pass it to any of the table writers, for example.
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @section Factory:
-#'
-#' The `$create()` factory methods instantiate the `OutputStream` object and
-#' take the following arguments, depending on the subclass:
-#'
-#' - `path` For `FileOutputStream`, a character file name
-#' - `initial_capacity` For `BufferOutputStream`, the size in bytes of the
-#' buffer.
-#'
-#' @section Methods:
-#'
-#' - `$tell()`: return the position in the stream
-#' - `$close()`: close the stream
-#' - `$write(x)`: send `x` to the stream
-#' - `$capacity()`: for `BufferOutputStream`
-#' - `$finish()`: for `BufferOutputStream`
-#' - `$GetExtentBytesWritten()`: for `MockOutputStream`, report how many bytes
-#' were sent.
-#'
-#' @rdname OutputStream
-#' @name OutputStream
-OutputStream <- R6Class("OutputStream", inherit = Writable,
- public = list(
- close = function() io___OutputStream__Close(self),
- tell = function() io___OutputStream__Tell(self)
- )
-)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname OutputStream
-#' @export
-FileOutputStream <- R6Class("FileOutputStream", inherit = OutputStream)
-FileOutputStream$create <- function(path) {
- io___FileOutputStream__Open(clean_path_abs(path))
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname OutputStream
-#' @export
-BufferOutputStream <- R6Class("BufferOutputStream", inherit = OutputStream,
- public = list(
- capacity = function() io___BufferOutputStream__capacity(self),
- finish = function() io___BufferOutputStream__Finish(self),
- write = function(bytes) io___BufferOutputStream__Write(self, bytes),
- tell = function() io___BufferOutputStream__Tell(self)
- )
-)
-BufferOutputStream$create <- function(initial_capacity = 0L) {
- io___BufferOutputStream__Create(initial_capacity)
-}
-
-# InputStream -------------------------------------------------------------
-
-
-Readable <- R6Class("Readable", inherit = ArrowObject,
- public = list(
- Read = function(nbytes) io___Readable__Read(self, nbytes)
- )
-)
-
-#' @title InputStream classes
-#' @description `RandomAccessFile` inherits from `InputStream` and is a base
-#' class for: `ReadableFile` for reading from a file; `MemoryMappedFile` for
-#' the same but with memory mapping; and `BufferReader` for reading from a
-#' buffer. Use these with the various table readers.
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @section Factory:
-#'
-#' The `$create()` factory methods instantiate the `InputStream` object and
-#' take the following arguments, depending on the subclass:
-#'
-#' - `path` For `ReadableFile`, a character file name
-#' - `x` For `BufferReader`, a [Buffer] or an object that can be
-#' made into a buffer via `buffer()`.
-#'
-#' To instantiate a `MemoryMappedFile`, call [mmap_open()].
-#'
-#' @section Methods:
-#'
-#' - `$GetSize()`:
-#' - `$supports_zero_copy()`: Logical
-#' - `$seek(position)`: go to that position in the stream
-#' - `$tell()`: return the position in the stream
-#' - `$close()`: close the stream
-#' - `$Read(nbytes)`: read data from the stream, either a specified `nbytes` or
-#' all, if `nbytes` is not provided
-#' - `$ReadAt(position, nbytes)`: similar to `$seek(position)$Read(nbytes)`
-#' - `$Resize(size)`: for a `MemoryMappedFile` that is writeable
-#'
-#' @rdname InputStream
-#' @name InputStream
-InputStream <- R6Class("InputStream", inherit = Readable,
- public = list(
- close = function() io___InputStream__Close(self)
- )
-)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname InputStream
-#' @export
-RandomAccessFile <- R6Class("RandomAccessFile", inherit = InputStream,
- public = list(
- GetSize = function() io___RandomAccessFile__GetSize(self),
- supports_zero_copy = function() io___RandomAccessFile__supports_zero_copy(self),
- seek = function(position) io___RandomAccessFile__Seek(self, position),
- tell = function() io___RandomAccessFile__Tell(self),
-
- Read = function(nbytes = NULL) {
- if (is.null(nbytes)) {
- io___RandomAccessFile__Read0(self)
- } else {
- io___Readable__Read(self, nbytes)
- }
- },
-
- ReadAt = function(position, nbytes = NULL) {
- if (is.null(nbytes)) {
- nbytes <- self$GetSize() - position
- }
- io___RandomAccessFile__ReadAt(self, position, nbytes)
- }
- )
-)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname InputStream
-#' @export
-MemoryMappedFile <- R6Class("MemoryMappedFile", inherit = RandomAccessFile,
- public = list(
- Resize = function(size) io___MemoryMappedFile__Resize(self, size)
- )
-)
-
-#' @usage NULL
-#' @format NULL
-#' @rdname InputStream
-#' @export
-ReadableFile <- R6Class("ReadableFile", inherit = RandomAccessFile)
-ReadableFile$create <- function(path) {
- io___ReadableFile__Open(clean_path_abs(path))
-}
-
-#' @usage NULL
-#' @format NULL
-#' @rdname InputStream
-#' @export
-BufferReader <- R6Class("BufferReader", inherit = RandomAccessFile)
-BufferReader$create <- function(x) {
- x <- buffer(x)
- io___BufferReader__initialize(x)
-}
-
-#' Create a new read/write memory mapped file of a given size
-#'
-#' @param path file path
-#' @param size size in bytes
-#'
-#' @return a [arrow::io::MemoryMappedFile][MemoryMappedFile]
-#'
-#' @export
-mmap_create <- function(path, size) {
- path <- clean_path_abs(path)
- io___MemoryMappedFile__Create(path, size)
-}
-
-#' Open a memory mapped file
-#'
-#' @param path file path
-#' @param mode file mode (read/write/readwrite)
-#'
-#' @export
-mmap_open <- function(path, mode = c("read", "write", "readwrite")) {
- mode <- match(match.arg(mode), c("read", "write", "readwrite")) - 1L
- path <- clean_path_abs(path)
- io___MemoryMappedFile__Open(path, mode)
-}
-
-#' Handle a range of possible input sources
-#' @param file A character file name, `raw` vector, or an Arrow input stream
-#' @param mmap Logical: whether to memory-map the file (default `TRUE`)
-#' @param compression If the file is compressed, created a [CompressedInputStream]
-#' with this compression codec, either a [Codec] or the string name of one.
-#' If `NULL` (default) and `file` is a string file name, the function will try
-#' to infer compression from the file extension.
-#' @param filesystem If not `NULL`, `file` will be opened via the
-#' `filesystem$OpenInputFile()` filesystem method, rather than the `io` module's
-#' `MemoryMappedFile` or `ReadableFile` constructors.
-#' @return An `InputStream` or a subclass of one.
-#' @keywords internal
-make_readable_file <- function(file, mmap = TRUE, compression = NULL, filesystem = NULL) {
- if (inherits(file, "SubTreeFileSystem")) {
- filesystem <- file$base_fs
- file <- file$base_path
- }
- if (is.string(file)) {
- if (is_url(file)) {
- fs_and_path <- FileSystem$from_uri(file)
- filesystem <- fs_and_path$fs
- file <- fs_and_path$path
- }
- if (is.null(compression)) {
- # Infer compression from the file path
- compression <- detect_compression(file)
- }
- if (!is.null(filesystem)) {
- file <- filesystem$OpenInputFile(file)
- } else if (isTRUE(mmap)) {
- file <- mmap_open(file)
- } else {
- file <- ReadableFile$create(file)
- }
- if (!identical(compression, "uncompressed")) {
- file <- CompressedInputStream$create(file, compression)
- }
- } else if (inherits(file, c("raw", "Buffer"))) {
- file <- BufferReader$create(file)
- }
- assert_is(file, "InputStream")
- file
-}
-
-make_output_stream <- function(x, filesystem = NULL) {
- if (inherits(x, "SubTreeFileSystem")) {
- filesystem <- x$base_fs
- x <- x$base_path
- } else if (is_url(x)) {
- fs_and_path <- FileSystem$from_uri(x)
- filesystem = fs_and_path$fs
- x <- fs_and_path$path
- }
- assert_that(is.string(x))
- if (is.null(filesystem)) {
- FileOutputStream$create(x)
- } else {
- filesystem$OpenOutputStream(x)
- }
-}
-
-detect_compression <- function(path) {
- assert_that(is.string(path))
- switch(tools::file_ext(path),
- bz2 = "bz2",
- gz = "gzip",
- lz4 = "lz4",
- zst = "zstd",
- "uncompressed"
- )
-}
diff --git a/r/R/ipc_stream.R b/r/R/ipc_stream.R
deleted file mode 100644
index 4f506f3..0000000
--- a/r/R/ipc_stream.R
+++ /dev/null
@@ -1,113 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Write Arrow IPC stream format
-#'
-#' Apache Arrow defines two formats for [serializing data for interprocess
-#' communication (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc):
-#' a "stream" format and a "file" format, known as Feather. `write_ipc_stream()`
-#' and [write_feather()] write those formats, respectively.
-#'
-#' `write_arrow()`, a wrapper around `write_ipc_stream()` and `write_feather()`
-#' with some nonstandard behavior, is deprecated. You should explicitly choose
-#' the function that will write the desired IPC format (stream or file) since
-#' either can be written to a file or `OutputStream`.
-#'
-#' @inheritParams write_feather
-#' @param ... extra parameters passed to `write_feather()`.
-#'
-#' @return `x`, invisibly.
-#' @seealso [write_feather()] for writing IPC files. [write_to_raw()] to
-#' serialize data to a buffer.
-#' [RecordBatchWriter] for a lower-level interface.
-#' @export
-write_ipc_stream <- function(x, sink, ...) {
- x_out <- x # So we can return the data we got
- if (is.data.frame(x)) {
- x <- Table$create(x)
- }
- if (!inherits(sink, "OutputStream")) {
- sink <- make_output_stream(sink)
- on.exit(sink$close())
- }
-
- writer <- RecordBatchStreamWriter$create(sink, x$schema)
- writer$write(x)
- writer$close()
- invisible(x_out)
-}
-
-#' Write Arrow data to a raw vector
-#'
-#' [write_ipc_stream()] and [write_feather()] write data to a sink and return
-#' the data (`data.frame`, `RecordBatch`, or `Table`) they were given.
-#' This function wraps those so that you can serialize data to a buffer and
-#' access that buffer as a `raw` vector in R.
-#' @inheritParams write_feather
-#' @param format one of `c("stream", "file")`, indicating the IPC format to use
-#' @return A `raw` vector containing the bytes of the IPC serialized data.
-#' @export
-write_to_raw <- function(x, format = c("stream", "file")) {
- sink <- BufferOutputStream$create()
- if (match.arg(format) == "stream") {
- write_ipc_stream(x, sink)
- } else {
- write_feather(x, sink)
- }
- as.raw(buffer(sink))
-}
-
-#' Read Arrow IPC stream format
-#'
-#' Apache Arrow defines two formats for [serializing data for interprocess
-#' communication (IPC)](https://arrow.apache.org/docs/format/Columnar.html#serialization-and-interprocess-communication-ipc):
-#' a "stream" format and a "file" format, known as Feather. `read_ipc_stream()`
-#' and [read_feather()] read those formats, respectively.
-#'
-#' `read_arrow()`, a wrapper around `read_ipc_stream()` and `read_feather()`,
-#' is deprecated. You should explicitly choose
-#' the function that will read the desired IPC format (stream or file) since
-#' a file or `InputStream` may contain either.
-#'
-#' @param file A character file name or URI, `raw` vector, an Arrow input stream,
-#' or a `FileSystem` with path (`SubTreeFileSystem`).
-#' If a file name or URI, an Arrow [InputStream] will be opened and
-#' closed when finished. If an input stream is provided, it will be left
-#' open.
-#' @param as_data_frame Should the function return a `data.frame` (default) or
-#' an Arrow [Table]?
-#' @param ... extra parameters passed to `read_feather()`.
-#'
-#' @return A `data.frame` if `as_data_frame` is `TRUE` (the default), or an
-#' Arrow [Table] otherwise
-#' @seealso [read_feather()] for writing IPC files. [RecordBatchReader] for a
-#' lower-level interface.
-#' @export
-read_ipc_stream <- function(file, as_data_frame = TRUE, ...) {
- if (!inherits(file, "InputStream")) {
- file <- make_readable_file(file)
- on.exit(file$close())
- }
-
- # TODO: this could take col_select, like the other readers
- # https://issues.apache.org/jira/browse/ARROW-6830
- out <- RecordBatchStreamReader$create(file)$read_table()
- if (as_data_frame) {
- out <- as.data.frame(out)
- }
- out
-}
diff --git a/r/R/json.R b/r/R/json.R
deleted file mode 100644
index 89595a5..0000000
--- a/r/R/json.R
+++ /dev/null
@@ -1,104 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' Read a JSON file
-#'
-#' Using [JsonTableReader]
-#'
-#' @inheritParams read_delim_arrow
-#' @param schema [Schema] that describes the table.
-#' @param ... Additional options passed to `JsonTableReader$create()`
-#'
-#' @return A `data.frame`, or a Table if `as_data_frame = FALSE`.
-#' @export
-#' @examples
-#' \donttest{
-#' tf <- tempfile()
-#' on.exit(unlink(tf))
-#' writeLines('
-#' { "hello": 3.5, "world": false, "yo": "thing" }
-#' { "hello": 3.25, "world": null }
-#' { "hello": 0.0, "world": true, "yo": null }
-#' ', tf, useBytes=TRUE)
-#' df <- read_json_arrow(tf)
-#' }
-read_json_arrow <- function(file,
- col_select = NULL,
- as_data_frame = TRUE,
- schema = NULL,
- ...) {
- if (!inherits(file, "InputStream")) {
- file <- make_readable_file(file)
- on.exit(file$close())
- }
- tab <- JsonTableReader$create(file, schema = schema, ...)$Read()
-
- col_select <- enquo(col_select)
- if (!quo_is_null(col_select)) {
- tab <- tab[vars_select(names(tab), !!col_select)]
- }
-
- if (isTRUE(as_data_frame)) {
- tab <- as.data.frame(tab)
- }
- tab
-}
-
-#' @include arrow-package.R
-#' @rdname CsvTableReader
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-JsonTableReader <- R6Class("JsonTableReader", inherit = ArrowObject,
- public = list(
- Read = function() json___TableReader__Read(self)
- )
-)
-JsonTableReader$create <- function(file,
- read_options = JsonReadOptions$create(),
- parse_options = JsonParseOptions$create(schema = schema),
- schema = NULL,
- ...) {
- assert_is(file, "InputStream")
- json___TableReader__Make(file, read_options, parse_options)
-}
-
-#' @rdname CsvReadOptions
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-JsonReadOptions <- R6Class("JsonReadOptions", inherit = ArrowObject)
-JsonReadOptions$create <- function(use_threads = option_use_threads(), block_size = 1048576L) {
- json___ReadOptions__initialize(use_threads, block_size)
-}
-
-#' @rdname CsvReadOptions
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#' @export
-JsonParseOptions <- R6Class("JsonParseOptions", inherit = ArrowObject)
-JsonParseOptions$create <- function(newlines_in_values = FALSE, schema = NULL) {
- if (is.null(schema)) {
- json___ParseOptions__initialize1(newlines_in_values)
- } else {
- json___ParseOptions__initialize2(newlines_in_values, schema)
- }
-
-}
diff --git a/r/R/memory-pool.R b/r/R/memory-pool.R
deleted file mode 100644
index 2207ed6..0000000
--- a/r/R/memory-pool.R
+++ /dev/null
@@ -1,61 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-#'
-#' @title class arrow::MemoryPool
-#'
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @section Methods:
-#'
-#' - `backend_name`: one of "jemalloc", "mimalloc", or "system". Alternative
-#' memory allocators are optionally enabled at build time. Windows builds
-#' generally have `mimalloc`, and most others have both `jemalloc` (used by
-#' default) and `mimalloc`. To change memory allocators at runtime, set the
-#' environment variable `ARROW_DEFAULT_MEMORY_POOL` to one of those strings
-#' prior to loading the `arrow` library.
-#' - `bytes_allocated`
-#' - `max_memory`
-#'
-#' @rdname MemoryPool
-#' @name MemoryPool
-#' @keywords internal
-MemoryPool <- R6Class("MemoryPool",
- inherit = ArrowObject,
- public = list(
- # TODO: Allocate
- # TODO: Reallocate
- # TODO: Free
- ),
- active = list(
- backend_name = function() MemoryPool__backend_name(self),
- bytes_allocated = function() MemoryPool__bytes_allocated(self),
- max_memory = function() MemoryPool__max_memory(self)
- )
-)
-
-#' Arrow's default [MemoryPool]
-#'
-#' @return the default [MemoryPool]
-#' @export
-#' @keywords internal
-default_memory_pool <- function() {
- MemoryPool__default()
-}
diff --git a/r/R/message.R b/r/R/message.R
deleted file mode 100644
index 6a374a2..0000000
--- a/r/R/message.R
+++ /dev/null
@@ -1,95 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @include arrow-package.R
-
-#' @title class arrow::Message
-#'
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @section Methods:
-#'
-#' TODO
-#'
-#' @rdname Message
-#' @name Message
-Message <- R6Class("Message", inherit = ArrowObject,
- public = list(
- Equals = function(other, ...) {
- inherits(other, "Message") && ipc___Message__Equals(self, other)
- },
- body_length = function() ipc___Message__body_length(self),
- Verify = function() ipc___Message__Verify(self)
- ),
- active = list(
- type = function() ipc___Message__type(self),
- metadata = function() ipc___Message__metadata(self),
- body = function() ipc___Message__body(self)
- )
-)
-
-#' @title class arrow::MessageReader
-#'
-#' @usage NULL
-#' @format NULL
-#' @docType class
-#'
-#' @section Methods:
-#'
-#' TODO
-#'
-#' @rdname MessageReader
-#' @name MessageReader
-#' @export
-MessageReader <- R6Class("MessageReader", inherit = ArrowObject,
- public = list(
- ReadNextMessage = function() ipc___MessageReader__ReadNextMessage(self)
- )
-)
-
-MessageReader$create <- function(stream) {
- if (!inherits(stream, "InputStream")) {
- stream <- BufferReader$create(stream)
- }
- ipc___MessageReader__Open(stream)
-}
-
-#' Read a Message from a stream
-#'
-#' @param stream an InputStream
-#'
-#' @export
-read_message <- function(stream) {
- UseMethod("read_message")
-}
-
-#' @export
-read_message.default <- function(stream) {
- read_message(BufferReader$create(stream))
-}
-
-#' @export
-read_message.InputStream <- function(stream) {
- ipc___ReadMessage(stream)
-}
-
-#' @export
-read_message.MessageReader <- function(stream) {
- stream$ReadNextMessage()
-}
diff --git a/r/R/metadata.R b/r/R/metadata.R
deleted file mode 100644
index d3e5e21..0000000
--- a/r/R/metadata.R
+++ /dev/null
@@ -1,132 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#' @importFrom utils object.size
-.serialize_arrow_r_metadata <- function(x) {
- assert_is(x, "list")
-
- # drop problems attributes (most likely from readr)
- x[["attributes"]][["problems"]] <- NULL
... 38468 lines suppressed ...