You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2015/09/01 23:58:53 UTC
spark git commit: [SPARK-10392] [SQL] Pyspark - Wrong DateType
support on JDBC connection
Repository: spark
Updated Branches:
refs/heads/master bf550a4b5 -> 00d9af5e1
[SPARK-10392] [SQL] Pyspark - Wrong DateType support on JDBC connection
This PR addresses issue [SPARK-10392](https://issues.apache.org/jira/browse/SPARK-10392)
The problem is that for "start of epoch" date (01 Jan 1970) PySpark class DateType returns 0 instead of the `datetime.date` due to implementation of its return statement
Issue reproduction on master:
```
>>> from pyspark.sql.types import *
>>> a = DateType()
>>> a.fromInternal(0)
0
>>> a.fromInternal(1)
datetime.date(1970, 1, 2)
```
Author: 0x0FFF <pr...@gmail.com>
Closes #8556 from 0x0FFF/SPARK-10392.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/00d9af5e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/00d9af5e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/00d9af5e
Branch: refs/heads/master
Commit: 00d9af5e190475affffb8b50467fcddfc40f50dc
Parents: bf550a4
Author: 0x0FFF <pr...@gmail.com>
Authored: Tue Sep 1 14:58:49 2015 -0700
Committer: Davies Liu <da...@gmail.com>
Committed: Tue Sep 1 14:58:49 2015 -0700
----------------------------------------------------------------------
python/pyspark/sql/tests.py | 5 +++++
python/pyspark/sql/types.py | 6 ++++--
2 files changed, 9 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/00d9af5e/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 59a891b..fc77863 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -168,6 +168,11 @@ class DataTypeTests(unittest.TestCase):
t3 = DecimalType(8)
self.assertNotEqual(t2, t3)
+ # regression test for SPARK-10392
+ def test_datetype_equal_zero(self):
+ dt = DateType()
+ self.assertEqual(dt.fromInternal(0), datetime.date(1970, 1, 1))
+
class SQLTests(ReusedPySparkTestCase):
http://git-wip-us.apache.org/repos/asf/spark/blob/00d9af5e/python/pyspark/sql/types.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/types.py b/python/pyspark/sql/types.py
index f84d08d..8bd58d6 100644
--- a/python/pyspark/sql/types.py
+++ b/python/pyspark/sql/types.py
@@ -168,10 +168,12 @@ class DateType(AtomicType):
return True
def toInternal(self, d):
- return d and d.toordinal() - self.EPOCH_ORDINAL
+ if d is not None:
+ return d.toordinal() - self.EPOCH_ORDINAL
def fromInternal(self, v):
- return v and datetime.date.fromordinal(v + self.EPOCH_ORDINAL)
+ if v is not None:
+ return datetime.date.fromordinal(v + self.EPOCH_ORDINAL)
class TimestampType(AtomicType):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org