You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/01/18 13:33:08 UTC
spark git commit: [SPARK-23141][SQL][PYSPARK] Support data type
string as a returnType for registerJavaFunction.
Repository: spark
Updated Branches:
refs/heads/master e28eb4311 -> 5063b7481
[SPARK-23141][SQL][PYSPARK] Support data type string as a returnType for registerJavaFunction.
## What changes were proposed in this pull request?
Currently `UDFRegistration.registerJavaFunction` doesn't support data type string as a `returnType` whereas `UDFRegistration.register`, `udf`, or `pandas_udf` does.
We can support it for `UDFRegistration.registerJavaFunction` as well.
## How was this patch tested?
Added a doctest and existing tests.
Author: Takuya UESHIN <ue...@databricks.com>
Closes #20307 from ueshin/issues/SPARK-23141.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5063b748
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5063b748
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5063b748
Branch: refs/heads/master
Commit: 5063b7481173ad72bd0dc941b5cf3c9b26a591e4
Parents: e28eb43
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Thu Jan 18 22:33:04 2018 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Thu Jan 18 22:33:04 2018 +0900
----------------------------------------------------------------------
python/pyspark/sql/functions.py | 6 ++++--
python/pyspark/sql/udf.py | 14 ++++++++++++--
2 files changed, 16 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5063b748/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 988c1d2..961b326 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -2108,7 +2108,8 @@ def udf(f=None, returnType=StringType()):
can fail on special rows, the workaround is to incorporate the condition into the functions.
:param f: python function if used as a standalone function
- :param returnType: a :class:`pyspark.sql.types.DataType` object
+ :param returnType: the return type of the user-defined function. The value can be either a
+ :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
>>> from pyspark.sql.types import IntegerType
>>> slen = udf(lambda s: len(s), IntegerType())
@@ -2148,7 +2149,8 @@ def pandas_udf(f=None, returnType=None, functionType=None):
Creates a vectorized user defined function (UDF).
:param f: user-defined function. A python function if used as a standalone function
- :param returnType: a :class:`pyspark.sql.types.DataType` object
+ :param returnType: the return type of the user-defined function. The value can be either a
+ :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
:param functionType: an enum value in :class:`pyspark.sql.functions.PandasUDFType`.
Default: SCALAR.
http://git-wip-us.apache.org/repos/asf/spark/blob/5063b748/python/pyspark/sql/udf.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/udf.py b/python/pyspark/sql/udf.py
index 1943bb7..c77f19f8 100644
--- a/python/pyspark/sql/udf.py
+++ b/python/pyspark/sql/udf.py
@@ -206,7 +206,8 @@ class UDFRegistration(object):
:param f: a Python function, or a user-defined function. The user-defined function can
be either row-at-a-time or vectorized. See :meth:`pyspark.sql.functions.udf` and
:meth:`pyspark.sql.functions.pandas_udf`.
- :param returnType: the return type of the registered user-defined function.
+ :param returnType: the return type of the registered user-defined function. The value can
+ be either a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
:return: a user-defined function.
`returnType` can be optionally specified when `f` is a Python function but not
@@ -303,21 +304,30 @@ class UDFRegistration(object):
:param name: name of the user-defined function
:param javaClassName: fully qualified name of java class
- :param returnType: a :class:`pyspark.sql.types.DataType` object
+ :param returnType: the return type of the registered Java function. The value can be either
+ a :class:`pyspark.sql.types.DataType` object or a DDL-formatted type string.
>>> from pyspark.sql.types import IntegerType
>>> spark.udf.registerJavaFunction(
... "javaStringLength", "test.org.apache.spark.sql.JavaStringLength", IntegerType())
>>> spark.sql("SELECT javaStringLength('test')").collect()
[Row(UDF:javaStringLength(test)=4)]
+
>>> spark.udf.registerJavaFunction(
... "javaStringLength2", "test.org.apache.spark.sql.JavaStringLength")
>>> spark.sql("SELECT javaStringLength2('test')").collect()
[Row(UDF:javaStringLength2(test)=4)]
+
+ >>> spark.udf.registerJavaFunction(
+ ... "javaStringLength3", "test.org.apache.spark.sql.JavaStringLength", "integer")
+ >>> spark.sql("SELECT javaStringLength3('test')").collect()
+ [Row(UDF:javaStringLength3(test)=4)]
"""
jdt = None
if returnType is not None:
+ if not isinstance(returnType, DataType):
+ returnType = _parse_datatype_string(returnType)
jdt = self.sparkSession._jsparkSession.parseDataType(returnType.json())
self.sparkSession._jsparkSession.udf().registerJava(name, javaClassName, jdt)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org