You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/02/04 08:53:36 UTC
spark git commit: [SPARK-23256][ML][PYTHON] Add columnSchema method
to PySpark image reader
Repository: spark
Updated Branches:
refs/heads/master 551dff2bc -> 715047b02
[SPARK-23256][ML][PYTHON] Add columnSchema method to PySpark image reader
## What changes were proposed in this pull request?
This PR proposes to add `columnSchema` in Python side too.
```python
>>> from pyspark.ml.image import ImageSchema
>>> ImageSchema.columnSchema.simpleString()
'struct<origin:string,height:int,width:int,nChannels:int,mode:int,data:binary>'
```
## How was this patch tested?
Manually tested and unittest was added in `python/pyspark/ml/tests.py`.
Author: hyukjinkwon <gu...@gmail.com>
Closes #20475 from HyukjinKwon/SPARK-23256.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/715047b0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/715047b0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/715047b0
Branch: refs/heads/master
Commit: 715047b02df0ac9ec16ab2a73481ab7f36ffc6ca
Parents: 551dff2
Author: hyukjinkwon <gu...@gmail.com>
Authored: Sun Feb 4 17:53:31 2018 +0900
Committer: hyukjinkwon <gu...@gmail.com>
Committed: Sun Feb 4 17:53:31 2018 +0900
----------------------------------------------------------------------
python/pyspark/ml/image.py | 20 +++++++++++++++++++-
python/pyspark/ml/tests.py | 1 +
2 files changed, 20 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/715047b0/python/pyspark/ml/image.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/image.py b/python/pyspark/ml/image.py
index 2d86c7f..45c9366 100644
--- a/python/pyspark/ml/image.py
+++ b/python/pyspark/ml/image.py
@@ -40,6 +40,7 @@ class _ImageSchema(object):
def __init__(self):
self._imageSchema = None
self._ocvTypes = None
+ self._columnSchema = None
self._imageFields = None
self._undefinedImageType = None
@@ -49,7 +50,7 @@ class _ImageSchema(object):
Returns the image schema.
:return: a :class:`StructType` with a single column of images
- named "image" (nullable).
+ named "image" (nullable) and having the same type returned by :meth:`columnSchema`.
.. versionadded:: 2.3.0
"""
@@ -76,6 +77,23 @@ class _ImageSchema(object):
return self._ocvTypes
@property
+ def columnSchema(self):
+ """
+ Returns the schema for the image column.
+
+ :return: a :class:`StructType` for image column,
+ ``struct<origin:string, height:int, width:int, nChannels:int, mode:int, data:binary>``.
+
+ .. versionadded:: 2.4.0
+ """
+
+ if self._columnSchema is None:
+ ctx = SparkContext._active_spark_context
+ jschema = ctx._jvm.org.apache.spark.ml.image.ImageSchema.columnSchema()
+ self._columnSchema = _parse_datatype_json_string(jschema.json())
+ return self._columnSchema
+
+ @property
def imageFields(self):
"""
Returns field names of image columns.
http://git-wip-us.apache.org/repos/asf/spark/blob/715047b0/python/pyspark/ml/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/ml/tests.py b/python/pyspark/ml/tests.py
index 1af2b91..75d0478 100755
--- a/python/pyspark/ml/tests.py
+++ b/python/pyspark/ml/tests.py
@@ -1852,6 +1852,7 @@ class ImageReaderTest(SparkSessionTestCase):
self.assertEqual(len(array), first_row[1])
self.assertEqual(ImageSchema.toImage(array, origin=first_row[0]), first_row)
self.assertEqual(df.schema, ImageSchema.imageSchema)
+ self.assertEqual(df.schema["image"].dataType, ImageSchema.columnSchema)
expected = {'CV_8UC3': 16, 'Undefined': -1, 'CV_8U': 0, 'CV_8UC1': 0, 'CV_8UC4': 24}
self.assertEqual(ImageSchema.ocvTypes, expected)
expected = ['origin', 'height', 'width', 'nChannels', 'mode', 'data']
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org