You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/06/12 00:54:49 UTC
git commit: HOTFIX: PySpark tests should be order insensitive.
Repository: spark
Updated Branches:
refs/heads/master fe78b8b6f -> 14e6dc94f
HOTFIX: PySpark tests should be order insensitive.
This has been messing up the SQL PySpark tests on Jenkins.
Author: Patrick Wendell <pw...@gmail.com>
Closes #1054 from pwendell/pyspark and squashes the following commits:
1eb5487 [Patrick Wendell] False change
06f062d [Patrick Wendell] HOTFIX: PySpark tests should be order insensitive
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/14e6dc94
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/14e6dc94
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/14e6dc94
Branch: refs/heads/master
Commit: 14e6dc94f68e57de82841c4ebbb573797a53869c
Parents: fe78b8b
Author: Patrick Wendell <pw...@gmail.com>
Authored: Wed Jun 11 15:54:41 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Jun 11 15:54:41 2014 -0700
----------------------------------------------------------------------
python/pyspark/sql.py | 8 ++++----
sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala | 1 +
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/14e6dc94/python/pyspark/sql.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql.py b/python/pyspark/sql.py
index b4e9618..960d0a8 100644
--- a/python/pyspark/sql.py
+++ b/python/pyspark/sql.py
@@ -117,7 +117,7 @@ class SQLContext:
>>> srdd = sqlCtx.inferSchema(rdd)
>>> srdd.saveAsParquetFile(parquetFile)
>>> srdd2 = sqlCtx.parquetFile(parquetFile)
- >>> srdd.collect() == srdd2.collect()
+ >>> sorted(srdd.collect()) == sorted(srdd2.collect())
True
"""
jschema_rdd = self._ssql_ctx.parquetFile(path)
@@ -141,7 +141,7 @@ class SQLContext:
>>> srdd = sqlCtx.inferSchema(rdd)
>>> sqlCtx.registerRDDAsTable(srdd, "table1")
>>> srdd2 = sqlCtx.table("table1")
- >>> srdd.collect() == srdd2.collect()
+ >>> sorted(srdd.collect()) == sorted(srdd2.collect())
True
"""
return SchemaRDD(self._ssql_ctx.table(tableName), self)
@@ -293,7 +293,7 @@ class SchemaRDD(RDD):
>>> srdd = sqlCtx.inferSchema(rdd)
>>> srdd.saveAsParquetFile(parquetFile)
>>> srdd2 = sqlCtx.parquetFile(parquetFile)
- >>> srdd2.collect() == srdd.collect()
+ >>> sorted(srdd2.collect()) == sorted(srdd.collect())
True
"""
self._jschema_rdd.saveAsParquetFile(path)
@@ -307,7 +307,7 @@ class SchemaRDD(RDD):
>>> srdd = sqlCtx.inferSchema(rdd)
>>> srdd.registerAsTable("test")
>>> srdd2 = sqlCtx.sql("select * from test")
- >>> srdd.collect() == srdd2.collect()
+ >>> sorted(srdd.collect()) == sorted(srdd2.collect())
True
"""
self._jschema_rdd.registerAsTable(name)
http://git-wip-us.apache.org/repos/asf/spark/blob/14e6dc94/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
index 7ad8edf..44b19bc 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SchemaRDD.scala
@@ -33,6 +33,7 @@ import org.apache.spark.api.java.JavaRDD
import java.util.{Map => JMap}
/**
+ * ***FALSE CHANGE***
* :: AlphaComponent ::
* An RDD of [[Row]] objects that has an associated schema. In addition to standard RDD functions,
* SchemaRDDs can be used in relational queries, as shown in the examples below.