You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@toree.apache.org by ch...@apache.org on 2017/01/23 01:06:28 UTC
[5/7] incubator-toree git commit: Remove SparkR fork (mariusvniekerk
via chipsenkbeil) closes #87
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2eb26cd3/sparkr-interpreter/src/main/resources/R/pkg/R/context.R
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/resources/R/pkg/R/context.R b/sparkr-interpreter/src/main/resources/R/pkg/R/context.R
deleted file mode 100644
index 4c026a5..0000000
--- a/sparkr-interpreter/src/main/resources/R/pkg/R/context.R
+++ /dev/null
@@ -1,225 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# context.R: SparkContext driven functions
-
-getMinPartitions <- function(sc, minPartitions) {
- if (is.null(minPartitions)) {
- defaultParallelism <- callJMethod(sc, "defaultParallelism")
- minPartitions <- min(defaultParallelism, 2)
- }
- as.integer(minPartitions)
-}
-
-# Create an RDD from a text file.
-#
-# This function reads a text file from HDFS, a local file system (available on all
-# nodes), or any Hadoop-supported file system URI, and creates an
-# RDD of strings from it.
-#
-# @param sc SparkContext to use
-# @param path Path of file to read. A vector of multiple paths is allowed.
-# @param minPartitions Minimum number of partitions to be created. If NULL, the default
-# value is chosen based on available parallelism.
-# @return RDD where each item is of type \code{character}
-# @export
-# @examples
-#\dontrun{
-# sc <- sparkR.init()
-# lines <- textFile(sc, "myfile.txt")
-#}
-textFile <- function(sc, path, minPartitions = NULL) {
- # Allow the user to have a more flexible definition of the text file path
- path <- suppressWarnings(normalizePath(path))
- # Convert a string vector of paths to a string containing comma separated paths
- path <- paste(path, collapse = ",")
-
- jrdd <- callJMethod(sc, "textFile", path, getMinPartitions(sc, minPartitions))
- # jrdd is of type JavaRDD[String]
- RDD(jrdd, "string")
-}
-
-# Load an RDD saved as a SequenceFile containing serialized objects.
-#
-# The file to be loaded should be one that was previously generated by calling
-# saveAsObjectFile() of the RDD class.
-#
-# @param sc SparkContext to use
-# @param path Path of file to read. A vector of multiple paths is allowed.
-# @param minPartitions Minimum number of partitions to be created. If NULL, the default
-# value is chosen based on available parallelism.
-# @return RDD containing serialized R objects.
-# @seealso saveAsObjectFile
-# @export
-# @examples
-#\dontrun{
-# sc <- sparkR.init()
-# rdd <- objectFile(sc, "myfile")
-#}
-objectFile <- function(sc, path, minPartitions = NULL) {
- # Allow the user to have a more flexible definition of the text file path
- path <- suppressWarnings(normalizePath(path))
- # Convert a string vector of paths to a string containing comma separated paths
- path <- paste(path, collapse = ",")
-
- jrdd <- callJMethod(sc, "objectFile", path, getMinPartitions(sc, minPartitions))
- # Assume the RDD contains serialized R objects.
- RDD(jrdd, "byte")
-}
-
-# Create an RDD from a homogeneous list or vector.
-#
-# This function creates an RDD from a local homogeneous list in R. The elements
-# in the list are split into \code{numSlices} slices and distributed to nodes
-# in the cluster.
-#
-# @param sc SparkContext to use
-# @param coll collection to parallelize
-# @param numSlices number of partitions to create in the RDD
-# @return an RDD created from this collection
-# @export
-# @examples
-#\dontrun{
-# sc <- sparkR.init()
-# rdd <- parallelize(sc, 1:10, 2)
-# # The RDD should contain 10 elements
-# length(rdd)
-#}
-parallelize <- function(sc, coll, numSlices = 1) {
- # TODO: bound/safeguard numSlices
- # TODO: unit tests for if the split works for all primitives
- # TODO: support matrix, data frame, etc
- if ((!is.list(coll) && !is.vector(coll)) || is.data.frame(coll)) {
- if (is.data.frame(coll)) {
- message(paste("context.R: A data frame is parallelized by columns."))
- } else {
- if (is.matrix(coll)) {
- message(paste("context.R: A matrix is parallelized by elements."))
- } else {
- message(paste("context.R: parallelize() currently only supports lists and vectors.",
- "Calling as.list() to coerce coll into a list."))
- }
- }
- coll <- as.list(coll)
- }
-
- if (numSlices > length(coll))
- numSlices <- length(coll)
-
- sliceLen <- ceiling(length(coll) / numSlices)
- slices <- split(coll, rep(1: (numSlices + 1), each = sliceLen)[1:length(coll)])
-
- # Serialize each slice: obtain a list of raws, or a list of lists (slices) of
- # 2-tuples of raws
- serializedSlices <- lapply(slices, serialize, connection = NULL)
-
- jrdd <- callJStatic("org.apache.spark.api.r.RRDD",
- "createRDDFromArray", sc, serializedSlices)
-
- RDD(jrdd, "byte")
-}
-
-# Include this specified package on all workers
-#
-# This function can be used to include a package on all workers before the
-# user's code is executed. This is useful in scenarios where other R package
-# functions are used in a function passed to functions like \code{lapply}.
-# NOTE: The package is assumed to be installed on every node in the Spark
-# cluster.
-#
-# @param sc SparkContext to use
-# @param pkg Package name
-#
-# @export
-# @examples
-#\dontrun{
-# library(Matrix)
-#
-# sc <- sparkR.init()
-# # Include the matrix library we will be using
-# includePackage(sc, Matrix)
-#
-# generateSparse <- function(x) {
-# sparseMatrix(i=c(1, 2, 3), j=c(1, 2, 3), x=c(1, 2, 3))
-# }
-#
-# rdd <- lapplyPartition(parallelize(sc, 1:2, 2L), generateSparse)
-# collect(rdd)
-#}
-includePackage <- function(sc, pkg) {
- pkg <- as.character(substitute(pkg))
- if (exists(".packages", .sparkREnv)) {
- packages <- .sparkREnv$.packages
- } else {
- packages <- list()
- }
- packages <- c(packages, pkg)
- .sparkREnv$.packages <- packages
-}
-
-# @title Broadcast a variable to all workers
-#
-# @description
-# Broadcast a read-only variable to the cluster, returning a \code{Broadcast}
-# object for reading it in distributed functions.
-#
-# @param sc Spark Context to use
-# @param object Object to be broadcast
-# @export
-# @examples
-#\dontrun{
-# sc <- sparkR.init()
-# rdd <- parallelize(sc, 1:2, 2L)
-#
-# # Large Matrix object that we want to broadcast
-# randomMat <- matrix(nrow=100, ncol=10, data=rnorm(1000))
-# randomMatBr <- broadcast(sc, randomMat)
-#
-# # Use the broadcast variable inside the function
-# useBroadcast <- function(x) {
-# sum(value(randomMatBr) * x)
-# }
-# sumRDD <- lapply(rdd, useBroadcast)
-#}
-broadcast <- function(sc, object) {
- objName <- as.character(substitute(object))
- serializedObj <- serialize(object, connection = NULL)
-
- jBroadcast <- callJMethod(sc, "broadcast", serializedObj)
- id <- as.character(callJMethod(jBroadcast, "id"))
-
- Broadcast(id, object, jBroadcast, objName)
-}
-
-# @title Set the checkpoint directory
-#
-# Set the directory under which RDDs are going to be checkpointed. The
-# directory must be a HDFS path if running on a cluster.
-#
-# @param sc Spark Context to use
-# @param dirName Directory path
-# @export
-# @examples
-#\dontrun{
-# sc <- sparkR.init()
-# setCheckpointDir(sc, "~/checkpoint")
-# rdd <- parallelize(sc, 1:2, 2L)
-# checkpoint(rdd)
-#}
-setCheckpointDir <- function(sc, dirName) {
- invisible(callJMethod(sc, "setCheckpointDir", suppressWarnings(normalizePath(dirName))))
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2eb26cd3/sparkr-interpreter/src/main/resources/R/pkg/R/deserialize.R
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/resources/R/pkg/R/deserialize.R b/sparkr-interpreter/src/main/resources/R/pkg/R/deserialize.R
deleted file mode 100644
index c5bbd61..0000000
--- a/sparkr-interpreter/src/main/resources/R/pkg/R/deserialize.R
+++ /dev/null
@@ -1,189 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Utility functions to deserialize objects from Java.
-
-# Type mapping from Java to R
-#
-# void -> NULL
-# Int -> integer
-# String -> character
-# Boolean -> logical
-# Float -> double
-# Double -> double
-# Long -> double
-# Array[Byte] -> raw
-# Date -> Date
-# Time -> POSIXct
-#
-# Array[T] -> list()
-# Object -> jobj
-
-readObject <- function(con) {
- # Read type first
- type <- readType(con)
- readTypedObject(con, type)
-}
-
-readTypedObject <- function(con, type) {
- switch (type,
- "i" = readInt(con),
- "c" = readString(con),
- "b" = readBoolean(con),
- "d" = readDouble(con),
- "r" = readRaw(con),
- "D" = readDate(con),
- "t" = readTime(con),
- "l" = readList(con),
- "n" = NULL,
- "j" = getJobj(readString(con)),
- stop(paste("Unsupported type for deserialization", type)))
-}
-
-readString <- function(con) {
- stringLen <- readInt(con)
- string <- readBin(con, raw(), stringLen, endian = "big")
- rawToChar(string)
-}
-
-readInt <- function(con) {
- readBin(con, integer(), n = 1, endian = "big")
-}
-
-readDouble <- function(con) {
- readBin(con, double(), n = 1, endian = "big")
-}
-
-readBoolean <- function(con) {
- as.logical(readInt(con))
-}
-
-readType <- function(con) {
- rawToChar(readBin(con, "raw", n = 1L))
-}
-
-readDate <- function(con) {
- as.Date(readString(con))
-}
-
-readTime <- function(con) {
- t <- readDouble(con)
- as.POSIXct(t, origin = "1970-01-01")
-}
-
-# We only support lists where all elements are of same type
-readList <- function(con) {
- type <- readType(con)
- len <- readInt(con)
- if (len > 0) {
- l <- vector("list", len)
- for (i in 1:len) {
- l[[i]] <- readTypedObject(con, type)
- }
- l
- } else {
- list()
- }
-}
-
-readRaw <- function(con) {
- dataLen <- readInt(con)
- readBin(con, raw(), as.integer(dataLen), endian = "big")
-}
-
-readRawLen <- function(con, dataLen) {
- readBin(con, raw(), as.integer(dataLen), endian = "big")
-}
-
-readDeserialize <- function(con) {
- # We have two cases that are possible - In one, the entire partition is
- # encoded as a byte array, so we have only one value to read. If so just
- # return firstData
- dataLen <- readInt(con)
- firstData <- unserialize(
- readBin(con, raw(), as.integer(dataLen), endian = "big"))
-
- # Else, read things into a list
- dataLen <- readInt(con)
- if (length(dataLen) > 0 && dataLen > 0) {
- data <- list(firstData)
- while (length(dataLen) > 0 && dataLen > 0) {
- data[[length(data) + 1L]] <- unserialize(
- readBin(con, raw(), as.integer(dataLen), endian = "big"))
- dataLen <- readInt(con)
- }
- unlist(data, recursive = FALSE)
- } else {
- firstData
- }
-}
-
-readDeserializeRows <- function(inputCon) {
- # readDeserializeRows will deserialize a DataOutputStream composed of
- # a list of lists. Since the DOS is one continuous stream and
- # the number of rows varies, we put the readRow function in a while loop
- # that terminates when the next row is empty.
- data <- list()
- while(TRUE) {
- row <- readRow(inputCon)
- if (length(row) == 0) {
- break
- }
- data[[length(data) + 1L]] <- row
- }
- data # this is a list of named lists now
-}
-
-readRowList <- function(obj) {
- # readRowList is meant for use inside an lapply. As a result, it is
- # necessary to open a standalone connection for the row and consume
- # the numCols bytes inside the read function in order to correctly
- # deserialize the row.
- rawObj <- rawConnection(obj, "r+")
- on.exit(close(rawObj))
- readRow(rawObj)
-}
-
-readRow <- function(inputCon) {
- numCols <- readInt(inputCon)
- if (length(numCols) > 0 && numCols > 0) {
- lapply(1:numCols, function(x) {
- obj <- readObject(inputCon)
- if (is.null(obj)) {
- NA
- } else {
- obj
- }
- }) # each row is a list now
- } else {
- list()
- }
-}
-
-# Take a single column as Array[Byte] and deserialize it into an atomic vector
-readCol <- function(inputCon, numRows) {
- if (numRows > 0) {
- # sapply can not work with POSIXlt
- do.call(c, lapply(1:numRows, function(x) {
- value <- readObject(inputCon)
- # Replace NULL with NA so we can coerce to vectors
- if (is.null(value)) NA else value
- }))
- } else {
- vector()
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-toree/blob/2eb26cd3/sparkr-interpreter/src/main/resources/R/pkg/R/functions.R
----------------------------------------------------------------------
diff --git a/sparkr-interpreter/src/main/resources/R/pkg/R/functions.R b/sparkr-interpreter/src/main/resources/R/pkg/R/functions.R
deleted file mode 100644
index d848730..0000000
--- a/sparkr-interpreter/src/main/resources/R/pkg/R/functions.R
+++ /dev/null
@@ -1,1988 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-#' @include generics.R column.R
-NULL
-
-#' Creates a \code{Column} of literal value.
-#'
-#' The passed in object is returned directly if it is already a \linkS4class{Column}.
-#' If the object is a Scala Symbol, it is converted into a \linkS4class{Column} also.
-#' Otherwise, a new \linkS4class{Column} is created to represent the literal value.
-#'
-#' @family normal_funcs
-#' @rdname lit
-#' @name lit
-#' @export
-setMethod("lit", signature("ANY"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "lit",
- ifelse(class(x) == "Column", x@jc, x))
- column(jc)
- })
-
-#' abs
-#'
-#' Computes the absolute value.
-#'
-#' @rdname abs
-#' @name abs
-#' @family normal_funcs
-#' @export
-#' @examples \dontrun{abs(df$c)}
-setMethod("abs",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "abs", x@jc)
- column(jc)
- })
-
-#' acos
-#'
-#' Computes the cosine inverse of the given value; the returned angle is in the range
-#' 0.0 through pi.
-#'
-#' @rdname acos
-#' @name acos
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{acos(df$c)}
-setMethod("acos",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "acos", x@jc)
- column(jc)
- })
-
-#' approxCountDistinct
-#'
-#' Aggregate function: returns the approximate number of distinct items in a group.
-#'
-#' @rdname approxCountDistinct
-#' @name approxCountDistinct
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{approxCountDistinct(df$c)}
-setMethod("approxCountDistinct",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "approxCountDistinct", x@jc)
- column(jc)
- })
-
-#' ascii
-#'
-#' Computes the numeric value of the first character of the string column, and returns the
-#' result as a int column.
-#'
-#' @rdname ascii
-#' @name ascii
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{\dontrun{ascii(df$c)}}
-setMethod("ascii",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "ascii", x@jc)
- column(jc)
- })
-
-#' asin
-#'
-#' Computes the sine inverse of the given value; the returned angle is in the range
-#' -pi/2 through pi/2.
-#'
-#' @rdname asin
-#' @name asin
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{asin(df$c)}
-setMethod("asin",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "asin", x@jc)
- column(jc)
- })
-
-#' atan
-#'
-#' Computes the tangent inverse of the given value.
-#'
-#' @rdname atan
-#' @name atan
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{atan(df$c)}
-setMethod("atan",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "atan", x@jc)
- column(jc)
- })
-
-#' avg
-#'
-#' Aggregate function: returns the average of the values in a group.
-#'
-#' @rdname avg
-#' @name avg
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{avg(df$c)}
-setMethod("avg",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "avg", x@jc)
- column(jc)
- })
-
-#' base64
-#'
-#' Computes the BASE64 encoding of a binary column and returns it as a string column.
-#' This is the reverse of unbase64.
-#'
-#' @rdname base64
-#' @name base64
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{base64(df$c)}
-setMethod("base64",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "base64", x@jc)
- column(jc)
- })
-
-#' bin
-#'
-#' An expression that returns the string representation of the binary value of the given long
-#' column. For example, bin("12") returns "1100".
-#'
-#' @rdname bin
-#' @name bin
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{bin(df$c)}
-setMethod("bin",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "bin", x@jc)
- column(jc)
- })
-
-#' bitwiseNOT
-#'
-#' Computes bitwise NOT.
-#'
-#' @rdname bitwiseNOT
-#' @name bitwiseNOT
-#' @family normal_funcs
-#' @export
-#' @examples \dontrun{bitwiseNOT(df$c)}
-setMethod("bitwiseNOT",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "bitwiseNOT", x@jc)
- column(jc)
- })
-
-#' cbrt
-#'
-#' Computes the cube-root of the given value.
-#'
-#' @rdname cbrt
-#' @name cbrt
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{cbrt(df$c)}
-setMethod("cbrt",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "cbrt", x@jc)
- column(jc)
- })
-
-#' ceil
-#'
-#' Computes the ceiling of the given value.
-#'
-#' @rdname ceil
-#' @name ceil
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{ceil(df$c)}
-setMethod("ceil",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "ceil", x@jc)
- column(jc)
- })
-
-#' cos
-#'
-#' Computes the cosine of the given value.
-#'
-#' @rdname cos
-#' @name cos
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{cos(df$c)}
-setMethod("cos",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "cos", x@jc)
- column(jc)
- })
-
-#' cosh
-#'
-#' Computes the hyperbolic cosine of the given value.
-#'
-#' @rdname cosh
-#' @name cosh
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{cosh(df$c)}
-setMethod("cosh",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "cosh", x@jc)
- column(jc)
- })
-
-#' count
-#'
-#' Aggregate function: returns the number of items in a group.
-#'
-#' @rdname count
-#' @name count
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{count(df$c)}
-setMethod("count",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "count", x@jc)
- column(jc)
- })
-
-#' crc32
-#'
-#' Calculates the cyclic redundancy check value (CRC32) of a binary column and
-#' returns the value as a bigint.
-#'
-#' @rdname crc32
-#' @name crc32
-#' @family misc_funcs
-#' @export
-#' @examples \dontrun{crc32(df$c)}
-setMethod("crc32",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "crc32", x@jc)
- column(jc)
- })
-
-#' dayofmonth
-#'
-#' Extracts the day of the month as an integer from a given date/timestamp/string.
-#'
-#' @rdname dayofmonth
-#' @name dayofmonth
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{dayofmonth(df$c)}
-setMethod("dayofmonth",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "dayofmonth", x@jc)
- column(jc)
- })
-
-#' dayofyear
-#'
-#' Extracts the day of the year as an integer from a given date/timestamp/string.
-#'
-#' @rdname dayofyear
-#' @name dayofyear
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{dayofyear(df$c)}
-setMethod("dayofyear",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "dayofyear", x@jc)
- column(jc)
- })
-
-#' exp
-#'
-#' Computes the exponential of the given value.
-#'
-#' @rdname exp
-#' @name exp
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{exp(df$c)}
-setMethod("exp",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "exp", x@jc)
- column(jc)
- })
-
-#' explode
-#'
-#' Creates a new row for each element in the given array or map column.
-#'
-#' @rdname explode
-#' @name explode
-#' @family collection_funcs
-#' @export
-#' @examples \dontrun{explode(df$c)}
-setMethod("explode",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "explode", x@jc)
- column(jc)
- })
-
-#' expm1
-#'
-#' Computes the exponential of the given value minus one.
-#'
-#' @rdname expm1
-#' @name expm1
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{expm1(df$c)}
-setMethod("expm1",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "expm1", x@jc)
- column(jc)
- })
-
-#' factorial
-#'
-#' Computes the factorial of the given value.
-#'
-#' @rdname factorial
-#' @name factorial
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{factorial(df$c)}
-setMethod("factorial",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "factorial", x@jc)
- column(jc)
- })
-
-#' first
-#'
-#' Aggregate function: returns the first value in a group.
-#'
-#' @rdname first
-#' @name first
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{first(df$c)}
-setMethod("first",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "first", x@jc)
- column(jc)
- })
-
-#' floor
-#'
-#' Computes the floor of the given value.
-#'
-#' @rdname floor
-#' @name floor
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{floor(df$c)}
-setMethod("floor",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "floor", x@jc)
- column(jc)
- })
-
-#' hex
-#'
-#' Computes hex value of the given column.
-#'
-#' @rdname hex
-#' @name hex
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{hex(df$c)}
-setMethod("hex",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "hex", x@jc)
- column(jc)
- })
-
-#' hour
-#'
-#' Extracts the hours as an integer from a given date/timestamp/string.
-#'
-#' @rdname hour
-#' @name hour
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{hour(df$c)}
-setMethod("hour",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "hour", x@jc)
- column(jc)
- })
-
-#' initcap
-#'
-#' Returns a new string column by converting the first letter of each word to uppercase.
-#' Words are delimited by whitespace.
-#'
-#' For example, "hello world" will become "Hello World".
-#'
-#' @rdname initcap
-#' @name initcap
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{initcap(df$c)}
-setMethod("initcap",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "initcap", x@jc)
- column(jc)
- })
-
-#' isNaN
-#'
-#' Return true iff the column is NaN.
-#'
-#' @rdname isNaN
-#' @name isNaN
-#' @family normal_funcs
-#' @export
-#' @examples \dontrun{isNaN(df$c)}
-setMethod("isNaN",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "isNaN", x@jc)
- column(jc)
- })
-
-#' last
-#'
-#' Aggregate function: returns the last value in a group.
-#'
-#' @rdname last
-#' @name last
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{last(df$c)}
-setMethod("last",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "last", x@jc)
- column(jc)
- })
-
-#' last_day
-#'
-#' Given a date column, returns the last day of the month which the given date belongs to.
-#' For example, input "2015-07-27" returns "2015-07-31" since July 31 is the last day of the
-#' month in July 2015.
-#'
-#' @rdname last_day
-#' @name last_day
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{last_day(df$c)}
-setMethod("last_day",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "last_day", x@jc)
- column(jc)
- })
-
-#' length
-#'
-#' Computes the length of a given string or binary column.
-#'
-#' @rdname length
-#' @name length
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{length(df$c)}
-setMethod("length",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "length", x@jc)
- column(jc)
- })
-
-#' log
-#'
-#' Computes the natural logarithm of the given value.
-#'
-#' @rdname log
-#' @name log
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{log(df$c)}
-setMethod("log",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "log", x@jc)
- column(jc)
- })
-
-#' log10
-#'
-#' Computes the logarithm of the given value in base 10.
-#'
-#' @rdname log10
-#' @name log10
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{log10(df$c)}
-setMethod("log10",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "log10", x@jc)
- column(jc)
- })
-
-#' log1p
-#'
-#' Computes the natural logarithm of the given value plus one.
-#'
-#' @rdname log1p
-#' @name log1p
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{log1p(df$c)}
-setMethod("log1p",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "log1p", x@jc)
- column(jc)
- })
-
-#' log2
-#'
-#' Computes the logarithm of the given column in base 2.
-#'
-#' @rdname log2
-#' @name log2
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{log2(df$c)}
-setMethod("log2",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "log2", x@jc)
- column(jc)
- })
-
-#' lower
-#'
-#' Converts a string column to lower case.
-#'
-#' @rdname lower
-#' @name lower
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{lower(df$c)}
-setMethod("lower",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "lower", x@jc)
- column(jc)
- })
-
-#' ltrim
-#'
-#' Trim the spaces from left end for the specified string value.
-#'
-#' @rdname ltrim
-#' @name ltrim
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{ltrim(df$c)}
-setMethod("ltrim",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "ltrim", x@jc)
- column(jc)
- })
-
-#' max
-#'
-#' Aggregate function: returns the maximum value of the expression in a group.
-#'
-#' @rdname max
-#' @name max
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{max(df$c)}
-setMethod("max",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "max", x@jc)
- column(jc)
- })
-
-#' md5
-#'
-#' Calculates the MD5 digest of a binary column and returns the value
-#' as a 32 character hex string.
-#'
-#' @rdname md5
-#' @name md5
-#' @family misc_funcs
-#' @export
-#' @examples \dontrun{md5(df$c)}
-setMethod("md5",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "md5", x@jc)
- column(jc)
- })
-
-#' mean
-#'
-#' Aggregate function: returns the average of the values in a group.
-#' Alias for avg.
-#'
-#' @rdname mean
-#' @name mean
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{mean(df$c)}
-setMethod("mean",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "mean", x@jc)
- column(jc)
- })
-
-#' min
-#'
-#' Aggregate function: returns the minimum value of the expression in a group.
-#'
-#' @rdname min
-#' @name min
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{min(df$c)}
-setMethod("min",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "min", x@jc)
- column(jc)
- })
-
-#' minute
-#'
-#' Extracts the minutes as an integer from a given date/timestamp/string.
-#'
-#' @rdname minute
-#' @name minute
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{minute(df$c)}
-setMethod("minute",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "minute", x@jc)
- column(jc)
- })
-
-#' month
-#'
-#' Extracts the month as an integer from a given date/timestamp/string.
-#'
-#' @rdname month
-#' @name month
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{month(df$c)}
-setMethod("month",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "month", x@jc)
- column(jc)
- })
-
-#' negate
-#'
-#' Unary minus, i.e. negate the expression.
-#'
-#' @rdname negate
-#' @name negate
-#' @family normal_funcs
-#' @export
-#' @examples \dontrun{negate(df$c)}
-setMethod("negate",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "negate", x@jc)
- column(jc)
- })
-
-#' quarter
-#'
-#' Extracts the quarter as an integer from a given date/timestamp/string.
-#'
-#' @rdname quarter
-#' @name quarter
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{quarter(df$c)}
-setMethod("quarter",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "quarter", x@jc)
- column(jc)
- })
-
-#' reverse
-#'
-#' Reverses the string column and returns it as a new string column.
-#'
-#' @rdname reverse
-#' @name reverse
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{reverse(df$c)}
-setMethod("reverse",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "reverse", x@jc)
- column(jc)
- })
-
-#' rint
-#'
-#' Returns the double value that is closest in value to the argument and
-#' is equal to a mathematical integer.
-#'
-#' @rdname rint
-#' @name rint
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{rint(df$c)}
-setMethod("rint",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "rint", x@jc)
- column(jc)
- })
-
-#' round
-#'
-#' Returns the value of the column `e` rounded to 0 decimal places.
-#'
-#' @rdname round
-#' @name round
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{round(df$c)}
-setMethod("round",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "round", x@jc)
- column(jc)
- })
-
-#' rtrim
-#'
-#' Trim the spaces from right end for the specified string value.
-#'
-#' @rdname rtrim
-#' @name rtrim
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{rtrim(df$c)}
-setMethod("rtrim",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "rtrim", x@jc)
- column(jc)
- })
-
-#' second
-#'
-#' Extracts the seconds as an integer from a given date/timestamp/string.
-#'
-#' @rdname second
-#' @name second
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{second(df$c)}
-setMethod("second",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "second", x@jc)
- column(jc)
- })
-
-#' sha1
-#'
-#' Calculates the SHA-1 digest of a binary column and returns the value
-#' as a 40 character hex string.
-#'
-#' @rdname sha1
-#' @name sha1
-#' @family misc_funcs
-#' @export
-#' @examples \dontrun{sha1(df$c)}
-setMethod("sha1",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sha1", x@jc)
- column(jc)
- })
-
-#' signum
-#'
-#' Computes the signum of the given value.
-#'
-#' @rdname signum
-#' @name signum
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{signum(df$c)}
-setMethod("signum",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "signum", x@jc)
- column(jc)
- })
-
-#' sin
-#'
-#' Computes the sine of the given value.
-#'
-#' @rdname sin
-#' @name sin
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{sin(df$c)}
-setMethod("sin",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sin", x@jc)
- column(jc)
- })
-
-#' sinh
-#'
-#' Computes the hyperbolic sine of the given value.
-#'
-#' @rdname sinh
-#' @name sinh
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{sinh(df$c)}
-setMethod("sinh",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sinh", x@jc)
- column(jc)
- })
-
-#' size
-#'
-#' Returns length of array or map.
-#'
-#' @rdname size
-#' @name size
-#' @family collection_funcs
-#' @export
-#' @examples \dontrun{size(df$c)}
-setMethod("size",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "size", x@jc)
- column(jc)
- })
-
-#' soundex
-#'
-#' Return the soundex code for the specified expression.
-#'
-#' @rdname soundex
-#' @name soundex
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{soundex(df$c)}
-setMethod("soundex",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "soundex", x@jc)
- column(jc)
- })
-
-#' sqrt
-#'
-#' Computes the square root of the specified float value.
-#'
-#' @rdname sqrt
-#' @name sqrt
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{sqrt(df$c)}
-setMethod("sqrt",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sqrt", x@jc)
- column(jc)
- })
-
-#' sum
-#'
-#' Aggregate function: returns the sum of all values in the expression.
-#'
-#' @rdname sum
-#' @name sum
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{sum(df$c)}
-setMethod("sum",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sum", x@jc)
- column(jc)
- })
-
-#' sumDistinct
-#'
-#' Aggregate function: returns the sum of distinct values in the expression.
-#'
-#' @rdname sumDistinct
-#' @name sumDistinct
-#' @family agg_funcs
-#' @export
-#' @examples \dontrun{sumDistinct(df$c)}
-setMethod("sumDistinct",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sumDistinct", x@jc)
- column(jc)
- })
-
-#' tan
-#'
-#' Computes the tangent of the given value.
-#'
-#' @rdname tan
-#' @name tan
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{tan(df$c)}
-setMethod("tan",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "tan", x@jc)
- column(jc)
- })
-
-#' tanh
-#'
-#' Computes the hyperbolic tangent of the given value.
-#'
-#' @rdname tanh
-#' @name tanh
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{tanh(df$c)}
-setMethod("tanh",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "tanh", x@jc)
- column(jc)
- })
-
-#' toDegrees
-#'
-#' Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
-#'
-#' @rdname toDegrees
-#' @name toDegrees
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{toDegrees(df$c)}
-setMethod("toDegrees",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "toDegrees", x@jc)
- column(jc)
- })
-
-#' toRadians
-#'
-#' Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
-#'
-#' @rdname toRadians
-#' @name toRadians
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{toRadians(df$c)}
-setMethod("toRadians",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "toRadians", x@jc)
- column(jc)
- })
-
-#' to_date
-#'
-#' Converts the column into DateType.
-#'
-#' @rdname to_date
-#' @name to_date
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{to_date(df$c)}
-setMethod("to_date",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "to_date", x@jc)
- column(jc)
- })
-
-#' trim
-#'
-#' Trim the spaces from both ends for the specified string column.
-#'
-#' @rdname trim
-#' @name trim
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{trim(df$c)}
-setMethod("trim",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "trim", x@jc)
- column(jc)
- })
-
-#' unbase64
-#'
-#' Decodes a BASE64 encoded string column and returns it as a binary column.
-#' This is the reverse of base64.
-#'
-#' @rdname unbase64
-#' @name unbase64
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{unbase64(df$c)}
-setMethod("unbase64",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "unbase64", x@jc)
- column(jc)
- })
-
-#' unhex
-#'
-#' Inverse of hex. Interprets each pair of characters as a hexadecimal number
-#' and converts to the byte representation of number.
-#'
-#' @rdname unhex
-#' @name unhex
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{unhex(df$c)}
-setMethod("unhex",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "unhex", x@jc)
- column(jc)
- })
-
-#' upper
-#'
-#' Converts a string column to upper case.
-#'
-#' @rdname upper
-#' @name upper
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{upper(df$c)}
-setMethod("upper",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "upper", x@jc)
- column(jc)
- })
-
-#' weekofyear
-#'
-#' Extracts the week number as an integer from a given date/timestamp/string.
-#'
-#' @rdname weekofyear
-#' @name weekofyear
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{weekofyear(df$c)}
-setMethod("weekofyear",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "weekofyear", x@jc)
- column(jc)
- })
-
-#' year
-#'
-#' Extracts the year as an integer from a given date/timestamp/string.
-#'
-#' @rdname year
-#' @name year
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{year(df$c)}
-setMethod("year",
- signature(x = "Column"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "year", x@jc)
- column(jc)
- })
-
-#' atan2
-#'
-#' Returns the angle theta from the conversion of rectangular coordinates (x, y) to
-#' polar coordinates (r, theta).
-#'
-#' @rdname atan2
-#' @name atan2
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{atan2(df$c, x)}
-setMethod("atan2", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "atan2", y@jc, x)
- column(jc)
- })
-
-#' datediff
-#'
-#' Returns the number of days from `start` to `end`.
-#'
-#' @rdname datediff
-#' @name datediff
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{datediff(df$c, x)}
-setMethod("datediff", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "datediff", y@jc, x)
- column(jc)
- })
-
-#' hypot
-#'
-#' Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
-#'
-#' @rdname hypot
-#' @name hypot
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{hypot(df$c, x)}
-setMethod("hypot", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "hypot", y@jc, x)
- column(jc)
- })
-
-#' levenshtein
-#'
-#' Computes the Levenshtein distance of the two given string columns.
-#'
-#' @rdname levenshtein
-#' @name levenshtein
-#' @family string_funcs
-#' @export
-#' @examples \dontrun{levenshtein(df$c, x)}
-setMethod("levenshtein", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "levenshtein", y@jc, x)
- column(jc)
- })
-
-#' months_between
-#'
-#' Returns number of months between dates `date1` and `date2`.
-#'
-#' @rdname months_between
-#' @name months_between
-#' @family datetime_funcs
-#' @export
-#' @examples \dontrun{months_between(df$c, x)}
-setMethod("months_between", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "months_between", y@jc, x)
- column(jc)
- })
-
-#' nanvl
-#'
-#' Returns col1 if it is not NaN, or col2 if col1 is NaN.
-#' hhBoth inputs should be floating point columns (DoubleType or FloatType).
-#'
-#' @rdname nanvl
-#' @name nanvl
-#' @family normal_funcs
-#' @export
-#' @examples \dontrun{nanvl(df$c, x)}
-setMethod("nanvl", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "nanvl", y@jc, x)
- column(jc)
- })
-
-#' pmod
-#'
-#' Returns the positive value of dividend mod divisor.
-#'
-#' @rdname pmod
-#' @name pmod
-#' @docType methods
-#' @family math_funcs
-#' @export
-#' @examples \dontrun{pmod(df$c, x)}
-setMethod("pmod", signature(y = "Column"),
- function(y, x) {
- if (class(x) == "Column") {
- x <- x@jc
- }
- jc <- callJStatic("org.apache.spark.sql.functions", "pmod", y@jc, x)
- column(jc)
- })
-
-
-#' Approx Count Distinct
-#'
-#' @family agg_funcs
-#' @rdname approxCountDistinct
-#' @name approxCountDistinct
-#' @return the approximate number of distinct items in a group.
-#' @export
-setMethod("approxCountDistinct",
- signature(x = "Column"),
- function(x, rsd = 0.95) {
- jc <- callJStatic("org.apache.spark.sql.functions", "approxCountDistinct", x@jc, rsd)
- column(jc)
- })
-
-#' Count Distinct
-#'
-#' @family agg_funcs
-#' @rdname countDistinct
-#' @name countDistinct
-#' @return the number of distinct items in a group.
-#' @export
-setMethod("countDistinct",
- signature(x = "Column"),
- function(x, ...) {
- jcol <- lapply(list(...), function (x) {
- x@jc
- })
- jc <- callJStatic("org.apache.spark.sql.functions", "countDistinct", x@jc,
- listToSeq(jcol))
- column(jc)
- })
-
-
-#' concat
-#'
-#' Concatenates multiple input string columns together into a single string column.
-#'
-#' @family string_funcs
-#' @rdname concat
-#' @name concat
-#' @export
-setMethod("concat",
- signature(x = "Column"),
- function(x, ...) {
- jcols <- lapply(list(x, ...), function(x) { x@jc })
- jc <- callJStatic("org.apache.spark.sql.functions", "concat", listToSeq(jcols))
- column(jc)
- })
-
-#' greatest
-#'
-#' Returns the greatest value of the list of column names, skipping null values.
-#' This function takes at least 2 parameters. It will return null if all parameters are null.
-#'
-#' @family normal_funcs
-#' @rdname greatest
-#' @name greatest
-#' @export
-setMethod("greatest",
- signature(x = "Column"),
- function(x, ...) {
- stopifnot(length(list(...)) > 0)
- jcols <- lapply(list(x, ...), function(x) { x@jc })
- jc <- callJStatic("org.apache.spark.sql.functions", "greatest", listToSeq(jcols))
- column(jc)
- })
-
-#' least
-#'
-#' Returns the least value of the list of column names, skipping null values.
-#' This function takes at least 2 parameters. It will return null iff all parameters are null.
-#'
-#' @family normal_funcs
-#' @rdname least
-#' @name least
-#' @export
-setMethod("least",
- signature(x = "Column"),
- function(x, ...) {
- stopifnot(length(list(...)) > 0)
- jcols <- lapply(list(x, ...), function(x) { x@jc })
- jc <- callJStatic("org.apache.spark.sql.functions", "least", listToSeq(jcols))
- column(jc)
- })
-
-#' ceiling
-#'
-#' Computes the ceiling of the given value.
-#'
-#' @family math_funcs
-#' @rdname ceil
-#' @name ceil
-#' @aliases ceil
-#' @export
-setMethod("ceiling",
- signature(x = "Column"),
- function(x) {
- ceil(x)
- })
-
-#' sign
-#'
-#' Computes the signum of the given value.
-#'
-#' @family math_funcs
-#' @rdname signum
-#' @name signum
-#' @aliases signum
-#' @export
-setMethod("sign", signature(x = "Column"),
- function(x) {
- signum(x)
- })
-
-#' n_distinct
-#'
-#' Aggregate function: returns the number of distinct items in a group.
-#'
-#' @family agg_funcs
-#' @rdname countDistinct
-#' @name countDistinct
-#' @aliases countDistinct
-#' @export
-setMethod("n_distinct", signature(x = "Column"),
- function(x, ...) {
- countDistinct(x, ...)
- })
-
-#' n
-#'
-#' Aggregate function: returns the number of items in a group.
-#'
-#' @family agg_funcs
-#' @rdname count
-#' @name count
-#' @aliases count
-#' @export
-setMethod("n", signature(x = "Column"),
- function(x) {
- count(x)
- })
-
-#' date_format
-#'
-#' Converts a date/timestamp/string to a value of string in the format specified by the date
-#' format given by the second argument.
-#'
-#' A pattern could be for instance \preformatted{dd.MM.yyyy} and could return a string like '18.03.1993'. All
-#' pattern letters of \code{java.text.SimpleDateFormat} can be used.
-#'
-#' NOTE: Use when ever possible specialized functions like \code{year}. These benefit from a
-#' specialized implementation.
-#'
-#' @family datetime_funcs
-#' @rdname date_format
-#' @name date_format
-#' @export
-setMethod("date_format", signature(y = "Column", x = "character"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "date_format", y@jc, x)
- column(jc)
- })
-
-#' from_utc_timestamp
-#'
-#' Assumes given timestamp is UTC and converts to given timezone.
-#'
-#' @family datetime_funcs
-#' @rdname from_utc_timestamp
-#' @name from_utc_timestamp
-#' @export
-setMethod("from_utc_timestamp", signature(y = "Column", x = "character"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "from_utc_timestamp", y@jc, x)
- column(jc)
- })
-
-#' instr
-#'
-#' Locate the position of the first occurrence of substr column in the given string.
-#' Returns null if either of the arguments are null.
-#'
-#' NOTE: The position is not zero based, but 1 based index, returns 0 if substr
-#' could not be found in str.
-#'
-#' @family string_funcs
-#' @rdname instr
-#' @name instr
-#' @export
-setMethod("instr", signature(y = "Column", x = "character"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "instr", y@jc, x)
- column(jc)
- })
-
-#' next_day
-#'
-#' Given a date column, returns the first date which is later than the value of the date column
-#' that is on the specified day of the week.
-#'
-#' For example, \code{next_day('2015-07-27', "Sunday")} returns 2015-08-02 because that is the first
-#' Sunday after 2015-07-27.
-#'
-#' Day of the week parameter is case insensitive, and accepts:
-#' "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun".
-#'
-#' @family datetime_funcs
-#' @rdname next_day
-#' @name next_day
-#' @export
-setMethod("next_day", signature(y = "Column", x = "character"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "next_day", y@jc, x)
- column(jc)
- })
-
-#' to_utc_timestamp
-#'
-#' Assumes given timestamp is in given timezone and converts to UTC.
-#'
-#' @family datetime_funcs
-#' @rdname to_utc_timestamp
-#' @name to_utc_timestamp
-#' @export
-setMethod("to_utc_timestamp", signature(y = "Column", x = "character"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "to_utc_timestamp", y@jc, x)
- column(jc)
- })
-
-#' add_months
-#'
-#' Returns the date that is numMonths after startDate.
-#'
-#' @name add_months
-#' @family datetime_funcs
-#' @rdname add_months
-#' @name add_months
-#' @export
-setMethod("add_months", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "add_months", y@jc, as.integer(x))
- column(jc)
- })
-
-#' date_add
-#'
-#' Returns the date that is `days` days after `start`
-#'
-#' @family datetime_funcs
-#' @rdname date_add
-#' @name date_add
-#' @export
-setMethod("date_add", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "date_add", y@jc, as.integer(x))
- column(jc)
- })
-
-#' date_sub
-#'
-#' Returns the date that is `days` days before `start`
-#'
-#' @family datetime_funcs
-#' @rdname date_sub
-#' @name date_sub
-#' @export
-setMethod("date_sub", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "date_sub", y@jc, as.integer(x))
- column(jc)
- })
-
-#' format_number
-#'
-#' Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places,
-#' and returns the result as a string column.
-#'
-#' If d is 0, the result has no decimal point or fractional part.
-#' If d < 0, the result will be null.'
-#'
-#' @family string_funcs
-#' @rdname format_number
-#' @name format_number
-#' @export
-setMethod("format_number", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "format_number",
- y@jc, as.integer(x))
- column(jc)
- })
-
-#' sha2
-#'
-#' Calculates the SHA-2 family of hash functions of a binary column and
-#' returns the value as a hex string.
-#'
-#' @param y column to compute SHA-2 on.
-#' @param x one of 224, 256, 384, or 512.
-#' @family misc_funcs
-#' @rdname sha2
-#' @name sha2
-#' @export
-setMethod("sha2", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "sha2", y@jc, as.integer(x))
- column(jc)
- })
-
-#' shiftLeft
-#'
-#' Shift the the given value numBits left. If the given value is a long value, this function
-#' will return a long value else it will return an integer value.
-#'
-#' @family math_funcs
-#' @rdname shiftLeft
-#' @name shiftLeft
-#' @export
-setMethod("shiftLeft", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "shiftLeft",
- y@jc, as.integer(x))
- column(jc)
- })
-
-#' shiftRight
-#'
-#' Shift the the given value numBits right. If the given value is a long value, it will return
-#' a long value else it will return an integer value.
-#'
-#' @family math_funcs
-#' @rdname shiftRight
-#' @name shiftRight
-#' @export
-setMethod("shiftRight", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "shiftRight",
- y@jc, as.integer(x))
- column(jc)
- })
-
-#' shiftRightUnsigned
-#'
-#' Unsigned shift the the given value numBits right. If the given value is a long value,
-#' it will return a long value else it will return an integer value.
-#'
-#' @family math_funcs
-#' @rdname shiftRightUnsigned
-#' @name shiftRightUnsigned
-#' @export
-setMethod("shiftRightUnsigned", signature(y = "Column", x = "numeric"),
- function(y, x) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "shiftRightUnsigned",
- y@jc, as.integer(x))
- column(jc)
- })
-
-#' concat_ws
-#'
-#' Concatenates multiple input string columns together into a single string column,
-#' using the given separator.
-#'
-#' @family string_funcs
-#' @rdname concat_ws
-#' @name concat_ws
-#' @export
-setMethod("concat_ws", signature(sep = "character", x = "Column"),
- function(sep, x, ...) {
- jcols <- listToSeq(lapply(list(x, ...), function(x) { x@jc }))
- jc <- callJStatic("org.apache.spark.sql.functions", "concat_ws", sep, jcols)
- column(jc)
- })
-
-#' conv
-#'
-#' Convert a number in a string column from one base to another.
-#'
-#' @family math_funcs
-#' @rdname conv
-#' @name conv
-#' @export
-setMethod("conv", signature(x = "Column", fromBase = "numeric", toBase = "numeric"),
- function(x, fromBase, toBase) {
- fromBase <- as.integer(fromBase)
- toBase <- as.integer(toBase)
- jc <- callJStatic("org.apache.spark.sql.functions",
- "conv",
- x@jc, fromBase, toBase)
- column(jc)
- })
-
-#' expr
-#'
-#' Parses the expression string into the column that it represents, similar to
-#' DataFrame.selectExpr
-#'
-#' @family normal_funcs
-#' @rdname expr
-#' @name expr
-#' @export
-setMethod("expr", signature(x = "character"),
- function(x) {
- jc <- callJStatic("org.apache.spark.sql.functions", "expr", x)
- column(jc)
- })
-
-#' format_string
-#'
-#' Formats the arguments in printf-style and returns the result as a string column.
-#'
-#' @family string_funcs
-#' @rdname format_string
-#' @name format_string
-#' @export
-setMethod("format_string", signature(format = "character", x = "Column"),
- function(format, x, ...) {
- jcols <- listToSeq(lapply(list(x, ...), function(arg) { arg@jc }))
- jc <- callJStatic("org.apache.spark.sql.functions",
- "format_string",
- format, jcols)
- column(jc)
- })
-
-#' from_unixtime
-#'
-#' Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string
-#' representing the timestamp of that moment in the current system time zone in the given
-#' format.
-#'
-#' @family datetime_funcs
-#' @rdname from_unixtime
-#' @name from_unixtime
-#' @export
-setMethod("from_unixtime", signature(x = "Column"),
- function(x, format = "yyyy-MM-dd HH:mm:ss") {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "from_unixtime",
- x@jc, format)
- column(jc)
- })
-
-#' locate
-#'
-#' Locate the position of the first occurrence of substr.
-#' NOTE: The position is not zero based, but 1 based index, returns 0 if substr
-#' could not be found in str.
-#'
-#' @family string_funcs
-#' @rdname locate
-#' @name locate
-#' @export
-setMethod("locate", signature(substr = "character", str = "Column"),
- function(substr, str, pos = 0) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "locate",
- substr, str@jc, as.integer(pos))
- column(jc)
- })
-
-#' lpad
-#'
-#' Left-pad the string column with
-#'
-#' @family string_funcs
-#' @rdname lpad
-#' @name lpad
-#' @export
-setMethod("lpad", signature(x = "Column", len = "numeric", pad = "character"),
- function(x, len, pad) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "lpad",
- x@jc, as.integer(len), pad)
- column(jc)
- })
-
-#' rand
-#'
-#' Generate a random column with i.i.d. samples from U[0.0, 1.0].
-#'
-#' @family normal_funcs
-#' @rdname rand
-#' @name rand
-#' @export
-setMethod("rand", signature(seed = "missing"),
- function(seed) {
- jc <- callJStatic("org.apache.spark.sql.functions", "rand")
- column(jc)
- })
-#' @family normal_funcs
-#' @rdname rand
-#' @name rand
-#' @export
-setMethod("rand", signature(seed = "numeric"),
- function(seed) {
- jc <- callJStatic("org.apache.spark.sql.functions", "rand", as.integer(seed))
- column(jc)
- })
-
-#' randn
-#'
-#' Generate a column with i.i.d. samples from the standard normal distribution.
-#'
-#' @family normal_funcs
-#' @rdname randn
-#' @name randn
-#' @export
-setMethod("randn", signature(seed = "missing"),
- function(seed) {
- jc <- callJStatic("org.apache.spark.sql.functions", "randn")
- column(jc)
- })
-#' @family normal_funcs
-#' @rdname randn
-#' @name randn
-#' @export
-setMethod("randn", signature(seed = "numeric"),
- function(seed) {
- jc <- callJStatic("org.apache.spark.sql.functions", "randn", as.integer(seed))
- column(jc)
- })
-
-#' regexp_extract
-#'
-#' Extract a specific(idx) group identified by a java regex, from the specified string column.
-#'
-#' @family string_funcs
-#' @rdname regexp_extract
-#' @name regexp_extract
-#' @export
-setMethod("regexp_extract",
- signature(x = "Column", pattern = "character", idx = "numeric"),
- function(x, pattern, idx) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "regexp_extract",
- x@jc, pattern, as.integer(idx))
- column(jc)
- })
-
-#' regexp_replace
-#'
-#' Replace all substrings of the specified string value that match regexp with rep.
-#'
-#' @family string_funcs
-#' @rdname regexp_replace
-#' @name regexp_replace
-#' @export
-setMethod("regexp_replace",
- signature(x = "Column", pattern = "character", replacement = "character"),
- function(x, pattern, replacement) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "regexp_replace",
- x@jc, pattern, replacement)
- column(jc)
- })
-
-#' rpad
-#'
-#' Right-padded with pad to a length of len.
-#'
-#' @family string_funcs
-#' @rdname rpad
-#' @name rpad
-#' @export
-setMethod("rpad", signature(x = "Column", len = "numeric", pad = "character"),
- function(x, len, pad) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "rpad",
- x@jc, as.integer(len), pad)
- column(jc)
- })
-
-#' substring_index
-#'
-#' Returns the substring from string str before count occurrences of the delimiter delim.
-#' If count is positive, everything the left of the final delimiter (counting from left) is
-#' returned. If count is negative, every to the right of the final delimiter (counting from the
-#' right) is returned. substring <- index performs a case-sensitive match when searching for delim.
-#'
-#' @family string_funcs
-#' @rdname substring_index
-#' @name substring_index
-#' @export
-setMethod("substring_index",
- signature(x = "Column", delim = "character", count = "numeric"),
- function(x, delim, count) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "substring_index",
- x@jc, delim, as.integer(count))
- column(jc)
- })
-
-#' translate
-#'
-#' Translate any character in the src by a character in replaceString.
-#' The characters in replaceString is corresponding to the characters in matchingString.
-#' The translate will happen when any character in the string matching with the character
-#' in the matchingString.
-#'
-#' @family string_funcs
-#' @rdname translate
-#' @name translate
-#' @export
-setMethod("translate",
- signature(x = "Column", matchingString = "character", replaceString = "character"),
- function(x, matchingString, replaceString) {
- jc <- callJStatic("org.apache.spark.sql.functions",
- "translate", x@jc, matchingString, replaceString)
- column(jc)
- })
-
-#' unix_timestamp
-#'
-#' Gets current Unix timestamp in seconds.
-#'
-#' @family datetime_funcs
-#' @rdname unix_timestamp
-#' @name unix_timestamp
-#' @export
-setMethod("unix_timestamp", signature(x = "missing", format = "missing"),
- function(x, format) {
- jc <- callJStatic("org.apache.spark.sql.functions", "unix_timestamp")
- column(jc)
- })
-#' @family datetime_funcs
-#' @rdname unix_timestamp
-#' @name unix_timestamp
-#' @export
-setMethod("unix_timestamp", signature(x = "Column", format = "missing"),
- function(x, format) {
- jc <- callJStatic("org.apache.spark.sql.functions", "unix_timestamp", x@jc)
- column(jc)
- })
-#' @family datetime_funcs
-#' @rdname unix_timestamp
-#' @name unix_timestamp
-#' @export
-setMethod("unix_timestamp", signature(x = "Column", format = "character"),
- function(x, format = "yyyy-MM-dd HH:mm:ss") {
- jc <- callJStatic("org.apache.spark.sql.functions", "unix_timestamp", x@jc, format)
- column(jc)
- })
-#' when
-#'
-#' Evaluates a list of conditions and returns one of multiple possible result expressions.
-#' For unmatched expressions null is returned.
-#'
-#' @family normal_funcs
-#' @rdname when
-#' @name when
-#' @export
-setMethod("when", signature(condition = "Column", value = "ANY"),
- function(condition, value) {
- condition <- condition@jc
- value <- ifelse(class(value) == "Column", value@jc, value)
- jc <- callJStatic("org.apache.spark.sql.functions", "when", condition, value)
- column(jc)
- })
-
-#' ifelse
-#'
-#' Evaluates a list of conditions and returns \code{yes} if the conditions are satisfied.
-#' Otherwise \code{no} is returned for unmatched conditions.
-#'
-#' @family normal_funcs
-#' @rdname ifelse
-#' @name ifelse
-#' @export
-setMethod("ifelse",
- signature(test = "Column", yes = "ANY", no = "ANY"),
- function(test, yes, no) {
- test <- test@jc
- yes <- ifelse(class(yes) == "Column", yes@jc, yes)
- no <- ifelse(class(no) == "Column", no@jc, no)
- jc <- callJMethod(callJStatic("org.apache.spark.sql.functions",
- "when",
- test, yes),
- "otherwise", no)
- column(jc)
- })