You are viewing a plain text version of this content. The canonical link for it is here.
Posted to github@arrow.apache.org by GitBox <gi...@apache.org> on 2020/12/29 19:52:43 UTC
[GitHub] [arrow] nealrichardson commented on a change in pull request #8947: ARROW-9187: [R] Add bindings for arithmetic kernels
nealrichardson commented on a change in pull request #8947:
URL: https://github.com/apache/arrow/pull/8947#discussion_r549807276
##########
File path: r/R/expression.R
##########
@@ -91,9 +122,20 @@ build_array_expression <- function(.Generic, e1, e2, ...) {
"<=" = "less_equal",
"&" = "and_kleene",
"|" = "or_kleene",
- "%in%" = "is_in_meta_binary"
+ "+" = "add_checked",
+ "-" = "subtract_checked",
+ "*" = "multiply_checked",
+ "/" = "divide_checked",
+ "%/%" = "divide_checked",
+ "%in%" = "is_in_meta_binary",
+ "%%" = "divide_checked"
Review comment:
maybe add a comment that we don't actually use "divide_checked" with `%%`
##########
File path: r/R/expression.R
##########
@@ -173,84 +205,84 @@ Expression <- R6Class("Expression", inherit = ArrowObject,
ToString = function() dataset___expr__ToString(self)
)
)
-
+Expression$create <- function(function_name,
+ ...,
+ args = list(...),
+ options = empty_named_list()) {
+ assert_that(is.string(function_name))
+ dataset___expr__call(function_name, args, options)
+}
Expression$field_ref <- function(name) {
- assert_is(name, "character")
- assert_that(length(name) == 1)
+ assert_that(is.string(name))
dataset___expr__field_ref(name)
}
Expression$scalar <- function(x) {
dataset___expr__scalar(Scalar$create(x))
}
-Expression$compare <- function(OP, e1, e2) {
- comp_func <- comparison_function_map[[OP]]
- if (is.null(comp_func)) {
- stop(OP, " is not a supported comparison function", call. = FALSE)
- }
- comp_func(e1, e2)
-}
-comparison_function_map <- list(
- "==" = dataset___expr__equal,
- "!=" = dataset___expr__not_equal,
- ">" = dataset___expr__greater,
- ">=" = dataset___expr__greater_equal,
- "<" = dataset___expr__less,
- "<=" = dataset___expr__less_equal
-)
-Expression$in_ <- function(x, set) {
- dataset___expr__in(x, Array$create(set))
-}
-Expression$and <- function(e1, e2) {
- dataset___expr__and(e1, e2)
-}
-Expression$or <- function(e1, e2) {
- dataset___expr__or(e1, e2)
-}
-Expression$not <- function(e1) {
- dataset___expr__not(e1)
-}
-Expression$is_valid <- function(e1) {
- dataset___expr__is_valid(e1)
+build_dataset_expression <- function(.Generic, e1, e2, ...) {
+ if (.Generic %in% names(.unary_function_map)) {
+ expr <- Expression$create(.unary_function_map[[.Generic]], e1)
+ } else if (.Generic == "%in%") {
+ # Special-case %in%, which is different from the Array function name
+ expr <- Expression$create("is_in", e1,
+ options = list(
+ value_set = Array$create(e2),
+ skip_nulls = TRUE
+ )
+ )
+ } else {
+ if (!inherits(e1, "Expression")) {
+ e1 <- Expression$scalar(e1)
+ }
+ if (!inherits(e2, "Expression")) {
+ e2 <- Expression$scalar(e2)
+ }
+
+ # In Arrow, "divide" is one function, which does integer division on
+ # integer inputs and floating-point division on floats
+ if (.Generic == "/") {
+ # TODO: omg so many ways it's wrong to assume these types
+ e1 <- Expression$create("cast", e1, options = list(to_type = float64()))
+ e2 <- Expression$create("cast", e2, options = list(to_type = float64()))
+ } else if (.Generic == "%/%") {
+ e1 <- Expression$create("cast", e1, options = list(to_type = float64()))
+ e2 <- Expression$create("cast", e2, options = list(to_type = float64()))
+ return(Expression$create("cast", Expression$create(.binary_function_map[[.Generic]], e1, e2, ...), options = list(to_type = int32(), allow_float_truncate = TRUE)))
+ } else if (.Generic == "%%") {
+ # {e1 - e2 * ( e1 %/% e2 )}
+ # TODO: there has to be a way to use the form ^^^ instead of this.
Review comment:
That should work: `Ops.Expression` should dispatch correctly because e1 and e2 are both Expressions. What happens when you do it?
##########
File path: r/R/expression.R
##########
@@ -91,9 +122,20 @@ build_array_expression <- function(.Generic, e1, e2, ...) {
"<=" = "less_equal",
"&" = "and_kleene",
"|" = "or_kleene",
- "%in%" = "is_in_meta_binary"
+ "+" = "add_checked",
+ "-" = "subtract_checked",
+ "*" = "multiply_checked",
+ "/" = "divide_checked",
+ "%/%" = "divide_checked",
+ "%in%" = "is_in_meta_binary",
+ "%%" = "divide_checked"
)
+
+# ‘"^"’
Review comment:
I'd move this up into the map above (commented out of course) and add next to it the JIRA number you create
##########
File path: r/tests/testthat/test-compute-arith.R
##########
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# TODO:
Review comment:
Are these TODOs done? Can you delete this?
##########
File path: r/tests/testthat/test-dplyr.R
##########
@@ -133,6 +133,42 @@ test_that("filtering with expression", {
)
})
+test_that("filtering with arithmetic", {
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl + 1 > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl / 2 > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl / 2L > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl %/% 2 > 3) %>%
Review comment:
All these test are with `dbl`; should we add some with `int` or other columns? Or do you think the other types are better tested elsewhere (test-compute-arith.R) and this is really just testing the dplyr NSE?
##########
File path: r/tests/testthat/test-dplyr.R
##########
@@ -133,6 +133,42 @@ test_that("filtering with expression", {
)
})
+test_that("filtering with arithmetic", {
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl + 1 > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl / 2 > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl / 2L > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl
+ )
+
+ expect_dplyr_equal(
+ input %>%
+ filter(dbl %/% 2 > 3) %>%
+ select(string = chr, int, dbl) %>%
+ collect(),
+ tbl,
+ # TODO: why are record batched versions problematic?
Review comment:
We should resolve this
##########
File path: r/R/expression.R
##########
@@ -173,84 +205,84 @@ Expression <- R6Class("Expression", inherit = ArrowObject,
ToString = function() dataset___expr__ToString(self)
)
)
-
+Expression$create <- function(function_name,
+ ...,
+ args = list(...),
+ options = empty_named_list()) {
+ assert_that(is.string(function_name))
+ dataset___expr__call(function_name, args, options)
+}
Expression$field_ref <- function(name) {
- assert_is(name, "character")
- assert_that(length(name) == 1)
+ assert_that(is.string(name))
dataset___expr__field_ref(name)
}
Expression$scalar <- function(x) {
dataset___expr__scalar(Scalar$create(x))
}
-Expression$compare <- function(OP, e1, e2) {
- comp_func <- comparison_function_map[[OP]]
- if (is.null(comp_func)) {
- stop(OP, " is not a supported comparison function", call. = FALSE)
- }
- comp_func(e1, e2)
-}
-comparison_function_map <- list(
- "==" = dataset___expr__equal,
- "!=" = dataset___expr__not_equal,
- ">" = dataset___expr__greater,
- ">=" = dataset___expr__greater_equal,
- "<" = dataset___expr__less,
- "<=" = dataset___expr__less_equal
-)
-Expression$in_ <- function(x, set) {
- dataset___expr__in(x, Array$create(set))
-}
-Expression$and <- function(e1, e2) {
- dataset___expr__and(e1, e2)
-}
-Expression$or <- function(e1, e2) {
- dataset___expr__or(e1, e2)
-}
-Expression$not <- function(e1) {
- dataset___expr__not(e1)
-}
-Expression$is_valid <- function(e1) {
- dataset___expr__is_valid(e1)
+build_dataset_expression <- function(.Generic, e1, e2, ...) {
+ if (.Generic %in% names(.unary_function_map)) {
+ expr <- Expression$create(.unary_function_map[[.Generic]], e1)
+ } else if (.Generic == "%in%") {
+ # Special-case %in%, which is different from the Array function name
+ expr <- Expression$create("is_in", e1,
+ options = list(
+ value_set = Array$create(e2),
+ skip_nulls = TRUE
+ )
+ )
+ } else {
+ if (!inherits(e1, "Expression")) {
+ e1 <- Expression$scalar(e1)
+ }
+ if (!inherits(e2, "Expression")) {
+ e2 <- Expression$scalar(e2)
+ }
+
+ # In Arrow, "divide" is one function, which does integer division on
+ # integer inputs and floating-point division on floats
+ if (.Generic == "/") {
+ # TODO: omg so many ways it's wrong to assume these types
+ e1 <- Expression$create("cast", e1, options = list(to_type = float64()))
Review comment:
You could define a `cast` method on Expression, `function(to_type, ...)`, and then you could rewrite these as `e1 <- e1$cast(float64())`, which I think reads nicer
##########
File path: r/tests/testthat/test-compute-arith.R
##########
@@ -0,0 +1,77 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# TODO:
+# * More tests for edge cases, esp. with division; add test helpers here?
+# * Is there a better "autocasting" solution? See what rules C++ Datasets do
+# * test-dplyr tests (Added one addition, and one summarize, but check to see if
+# we can make summarize route through arrow need more?)
+# * then, dataset tests, special casing for division
+
+test_that("Addition", {
+ a <- Array$create(c(1:4, NA_integer_))
+ expect_type_equal(a, int32())
+ expect_type_equal(a + 4, int32())
+ expect_equal(a + 4, Array$create(c(5:8, NA_integer_)))
+ expect_identical(as.vector(a + 4), c(5:8, NA_integer_))
+ expect_equal(a + 4L, Array$create(c(5:8, NA_integer_)))
+ expect_vector(a + 4L, c(5:8, NA_integer_))
+ expect_equal(a + NA_integer_, Array$create(rep(NA_integer_, 5)))
+
+ # overflow errors — this is slightly different from R's `NA` coercion when
+ # overflowing, but better than the alternative of silently restarting
+ casted <- a$cast(int8())
+ expect_error(casted + 257)
+
+ skip("autocasting should happen in compute kernels; R workaround fails on this")
Review comment:
From what you showed me in the dataset cpp code, the implicit casting it does (cast scalar to type of the column) also wouldn't promote as expected here. Might be worth making a dataset test that shows that, skipping it, and reporting a JIRA for Ben.
##########
File path: r/R/expression.R
##########
@@ -173,84 +205,84 @@ Expression <- R6Class("Expression", inherit = ArrowObject,
ToString = function() dataset___expr__ToString(self)
)
)
-
+Expression$create <- function(function_name,
+ ...,
+ args = list(...),
+ options = empty_named_list()) {
+ assert_that(is.string(function_name))
+ dataset___expr__call(function_name, args, options)
+}
Expression$field_ref <- function(name) {
- assert_is(name, "character")
- assert_that(length(name) == 1)
+ assert_that(is.string(name))
dataset___expr__field_ref(name)
}
Expression$scalar <- function(x) {
dataset___expr__scalar(Scalar$create(x))
}
-Expression$compare <- function(OP, e1, e2) {
- comp_func <- comparison_function_map[[OP]]
- if (is.null(comp_func)) {
- stop(OP, " is not a supported comparison function", call. = FALSE)
- }
- comp_func(e1, e2)
-}
-comparison_function_map <- list(
- "==" = dataset___expr__equal,
- "!=" = dataset___expr__not_equal,
- ">" = dataset___expr__greater,
- ">=" = dataset___expr__greater_equal,
- "<" = dataset___expr__less,
- "<=" = dataset___expr__less_equal
-)
-Expression$in_ <- function(x, set) {
- dataset___expr__in(x, Array$create(set))
-}
-Expression$and <- function(e1, e2) {
- dataset___expr__and(e1, e2)
-}
-Expression$or <- function(e1, e2) {
- dataset___expr__or(e1, e2)
-}
-Expression$not <- function(e1) {
- dataset___expr__not(e1)
-}
-Expression$is_valid <- function(e1) {
- dataset___expr__is_valid(e1)
+build_dataset_expression <- function(.Generic, e1, e2, ...) {
+ if (.Generic %in% names(.unary_function_map)) {
+ expr <- Expression$create(.unary_function_map[[.Generic]], e1)
+ } else if (.Generic == "%in%") {
+ # Special-case %in%, which is different from the Array function name
+ expr <- Expression$create("is_in", e1,
+ options = list(
+ value_set = Array$create(e2),
+ skip_nulls = TRUE
+ )
+ )
+ } else {
+ if (!inherits(e1, "Expression")) {
+ e1 <- Expression$scalar(e1)
+ }
+ if (!inherits(e2, "Expression")) {
+ e2 <- Expression$scalar(e2)
+ }
+
+ # In Arrow, "divide" is one function, which does integer division on
+ # integer inputs and floating-point division on floats
+ if (.Generic == "/") {
+ # TODO: omg so many ways it's wrong to assume these types
+ e1 <- Expression$create("cast", e1, options = list(to_type = float64()))
+ e2 <- Expression$create("cast", e2, options = list(to_type = float64()))
+ } else if (.Generic == "%/%") {
+ e1 <- Expression$create("cast", e1, options = list(to_type = float64()))
+ e2 <- Expression$create("cast", e2, options = list(to_type = float64()))
+ return(Expression$create("cast", Expression$create(.binary_function_map[[.Generic]], e1, e2, ...), options = list(to_type = int32(), allow_float_truncate = TRUE)))
Review comment:
```suggestion
# In R, integer division works like floor(float division)
out <- build_dataset_expression("/", e1, e2)
return(out$cast(int32(), allow_float_truncate = TRUE))
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org