You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/10/08 23:45:09 UTC
spark git commit: [SPARK-25659][PYTHON][TEST] Test type inference
specification for createDataFrame in PySpark
Repository: spark
Updated Branches:
refs/heads/master f9935a3f8 -> f3fed2823
[SPARK-25659][PYTHON][TEST] Test type inference specification for createDataFrame in PySpark
## What changes were proposed in this pull request?
This PR proposes to specify type inference and simple e2e tests. Looks we are not cleanly testing those logics.
For instance, see https://github.com/apache/spark/blob/08c76b5d39127ae207d9d1fff99c2551e6ce2581/python/pyspark/sql/types.py#L894-L905
Looks we intended to support datetime.time and None for type inference too but it does not work:
```
>>> spark.createDataFrame([[datetime.time()]])
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/.../spark/python/pyspark/sql/session.py", line 751, in createDataFrame
rdd, schema = self._createFromLocal(map(prepare, data), schema)
File "/.../spark/python/pyspark/sql/session.py", line 432, in _createFromLocal
data = [schema.toInternal(row) for row in data]
File "/.../spark/python/pyspark/sql/types.py", line 604, in toInternal
for f, v, c in zip(self.fields, obj, self._needConversion))
File "/.../spark/python/pyspark/sql/types.py", line 604, in <genexpr>
for f, v, c in zip(self.fields, obj, self._needConversion))
File "/.../spark/python/pyspark/sql/types.py", line 442, in toInternal
return self.dataType.toInternal(obj)
File "/.../spark/python/pyspark/sql/types.py", line 193, in toInternal
else time.mktime(dt.timetuple()))
AttributeError: 'datetime.time' object has no attribute 'timetuple'
>>> spark.createDataFrame([[None]])
Traceback (most recent call last):
File "<stdin>", line 1, in <module>
File "/.../spark/python/pyspark/sql/session.py", line 751, in createDataFrame
rdd, schema = self._createFromLocal(map(prepare, data), schema)
File "/.../spark/python/pyspark/sql/session.py", line 419, in _createFromLocal
struct = self._inferSchemaFromList(data, names=schema)
File "/.../python/pyspark/sql/session.py", line 353, in _inferSchemaFromList
raise ValueError("Some of types cannot be determined after inferring")
ValueError: Some of types cannot be determined after inferring
```
## How was this patch tested?
Manual tests and unit tests were added.
Closes #22653 from HyukjinKwon/SPARK-25659.
Authored-by: hyukjinkwon <gu...@apache.org>
Signed-off-by: hyukjinkwon <gu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f3fed282
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f3fed282
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f3fed282
Branch: refs/heads/master
Commit: f3fed28230e4e5e08d182715e8cf901daf8f3b73
Parents: f9935a3
Author: hyukjinkwon <gu...@apache.org>
Authored: Tue Oct 9 07:45:02 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Tue Oct 9 07:45:02 2018 +0800
----------------------------------------------------------------------
python/pyspark/sql/tests.py | 69 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 69 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f3fed282/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index ac87ccd..85712df 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -1149,6 +1149,75 @@ class SQLTests(ReusedSQLTestCase):
result = self.spark.sql("SELECT l[0].a from test2 where d['key'].d = '2'")
self.assertEqual(1, result.head()[0])
+ def test_infer_schema_specification(self):
+ from decimal import Decimal
+
+ class A(object):
+ def __init__(self):
+ self.a = 1
+
+ data = [
+ True,
+ 1,
+ "a",
+ u"a",
+ datetime.date(1970, 1, 1),
+ datetime.datetime(1970, 1, 1, 0, 0),
+ 1.0,
+ array.array("d", [1]),
+ [1],
+ (1, ),
+ {"a": 1},
+ bytearray(1),
+ Decimal(1),
+ Row(a=1),
+ Row("a")(1),
+ A(),
+ ]
+
+ df = self.spark.createDataFrame([data])
+ actual = list(map(lambda x: x.dataType.simpleString(), df.schema))
+ expected = [
+ 'boolean',
+ 'bigint',
+ 'string',
+ 'string',
+ 'date',
+ 'timestamp',
+ 'double',
+ 'array<double>',
+ 'array<bigint>',
+ 'struct<_1:bigint>',
+ 'map<string,bigint>',
+ 'binary',
+ 'decimal(38,18)',
+ 'struct<a:bigint>',
+ 'struct<a:bigint>',
+ 'struct<a:bigint>',
+ ]
+ self.assertEqual(actual, expected)
+
+ actual = list(df.first())
+ expected = [
+ True,
+ 1,
+ 'a',
+ u"a",
+ datetime.date(1970, 1, 1),
+ datetime.datetime(1970, 1, 1, 0, 0),
+ 1.0,
+ [1.0],
+ [1],
+ Row(_1=1),
+ {"a": 1},
+ bytearray(b'\x00'),
+ Decimal('1.000000000000000000'),
+ Row(a=1),
+ Row(a=1),
+ Row(a=1),
+ ]
+ self.assertEqual(actual, expected)
+
def test_infer_schema_not_enough_names(self):
df = self.spark.createDataFrame([["a", "b"]], ["col1"])
self.assertEqual(df.columns, ['col1', '_2'])
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org