You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2017/11/09 07:56:58 UTC
spark git commit: [SPARK-22417][PYTHON][FOLLOWUP][BRANCH-2.2] Fix for
createDataFrame from pandas.DataFrame with timestamp
Repository: spark
Updated Branches:
refs/heads/branch-2.2 efaf73fcd -> 0e97c8eef
[SPARK-22417][PYTHON][FOLLOWUP][BRANCH-2.2] Fix for createDataFrame from pandas.DataFrame with timestamp
## What changes were proposed in this pull request?
This is a follow-up of #19646 for branch-2.2.
The original pr breaks branch-2.2 because the cherry-picked patch doesn't include some code which exists in master.
## How was this patch tested?
Existing tests.
Author: Takuya UESHIN <ue...@databricks.com>
Closes #19704 from ueshin/issues/SPARK-22417_2.2/fup1.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0e97c8ee
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0e97c8ee
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0e97c8ee
Branch: refs/heads/branch-2.2
Commit: 0e97c8eef9fd011465dfaf28886f2873b3c6681d
Parents: efaf73f
Author: Takuya UESHIN <ue...@databricks.com>
Authored: Thu Nov 9 16:56:50 2017 +0900
Committer: Takuya UESHIN <ue...@databricks.com>
Committed: Thu Nov 9 16:56:50 2017 +0900
----------------------------------------------------------------------
python/pyspark/sql/tests.py | 11 ++++++++++-
1 file changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0e97c8ee/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index 6d7b20e..0926112 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -46,6 +46,14 @@ if sys.version_info[:2] <= (2, 6):
else:
import unittest
+_have_pandas = False
+try:
+ import pandas
+ _have_pandas = True
+except:
+ # No Pandas, but that's okay, we'll skip those tests
+ pass
+
from pyspark import SparkContext
from pyspark.sql import SparkSession, SQLContext, HiveContext, Column, Row
from pyspark.sql.types import *
@@ -2208,10 +2216,11 @@ class SQLTests(ReusedPySparkTestCase):
self.assertTrue(isinstance(df.schema['ts'].dataType, TimestampType))
self.assertTrue(isinstance(df.schema['d'].dataType, DateType))
# test with schema will accept pdf as input
- df = self.spark.createDataFrame(pdf, schema="d date, ts timestamp")
+ df = self.spark.createDataFrame(pdf, schema="d: date, ts: timestamp")
self.assertTrue(isinstance(df.schema['ts'].dataType, TimestampType))
self.assertTrue(isinstance(df.schema['d'].dataType, DateType))
+
class HiveSparkSubmitTests(SparkSubmitTests):
def test_hivecontext(self):
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org