You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/02/05 07:43:47 UTC
spark git commit: [SPARK-13187][SQL] Add boolean/long/double options
in DataFrameReader/Writer
Repository: spark
Updated Branches:
refs/heads/master 352102ed0 -> 82d84ff2d
[SPARK-13187][SQL] Add boolean/long/double options in DataFrameReader/Writer
This patch adds option function for boolean, long, and double types. This makes it slightly easier for Spark users to specify options without turning them into strings. Using the JSON data source as an example.
Before this patch:
```scala
sqlContext.read.option("primitivesAsString", "true").json("/path/to/json")
```
After this patch:
Before this patch:
```scala
sqlContext.read.option("primitivesAsString", true).json("/path/to/json")
```
Author: Reynold Xin <rx...@databricks.com>
Closes #11072 from rxin/SPARK-13187.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/82d84ff2
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/82d84ff2
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/82d84ff2
Branch: refs/heads/master
Commit: 82d84ff2dd3efb3bda20b529f09a4022586fb722
Parents: 352102e
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Feb 4 22:43:44 2016 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Feb 4 22:43:44 2016 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/DataFrameReader.scala | 21 ++++++++++++++++
.../org/apache/spark/sql/DataFrameWriter.scala | 21 ++++++++++++++++
.../sql/streaming/DataStreamReaderSuite.scala | 25 ++++++++++++++++++++
3 files changed, 67 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/82d84ff2/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
index a58643a..962fdad 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameReader.scala
@@ -79,6 +79,27 @@ class DataFrameReader private[sql](sqlContext: SQLContext) extends Logging {
}
/**
+ * Adds an input option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Boolean): DataFrameReader = option(key, value.toString)
+
+ /**
+ * Adds an input option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Long): DataFrameReader = option(key, value.toString)
+
+ /**
+ * Adds an input option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Double): DataFrameReader = option(key, value.toString)
+
+ /**
* (Scala-specific) Adds input options for the underlying data source.
*
* @since 1.4.0
http://git-wip-us.apache.org/repos/asf/spark/blob/82d84ff2/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
index 80447fe..8060198 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrameWriter.scala
@@ -96,6 +96,27 @@ final class DataFrameWriter private[sql](df: DataFrame) {
}
/**
+ * Adds an output option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Boolean): DataFrameWriter = option(key, value.toString)
+
+ /**
+ * Adds an output option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Long): DataFrameWriter = option(key, value.toString)
+
+ /**
+ * Adds an output option for the underlying data source.
+ *
+ * @since 2.0.0
+ */
+ def option(key: String, value: Double): DataFrameWriter = option(key, value.toString)
+
+ /**
* (Scala-specific) Adds output options for the underlying data source.
*
* @since 1.4.0
http://git-wip-us.apache.org/repos/asf/spark/blob/82d84ff2/sql/core/src/test/scala/org/apache/spark/sql/streaming/DataStreamReaderSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/streaming/DataStreamReaderSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/streaming/DataStreamReaderSuite.scala
index b36b41c..95a17f3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/streaming/DataStreamReaderSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/streaming/DataStreamReaderSuite.scala
@@ -162,4 +162,29 @@ class DataStreamReaderWriterSuite extends StreamTest with SharedSQLContext {
assert(LastOptions.parameters("path") == "/test")
}
+ test("test different data types for options") {
+ val df = sqlContext.read
+ .format("org.apache.spark.sql.streaming.test")
+ .option("intOpt", 56)
+ .option("boolOpt", false)
+ .option("doubleOpt", 6.7)
+ .stream("/test")
+
+ assert(LastOptions.parameters("intOpt") == "56")
+ assert(LastOptions.parameters("boolOpt") == "false")
+ assert(LastOptions.parameters("doubleOpt") == "6.7")
+
+ LastOptions.parameters = null
+ df.write
+ .format("org.apache.spark.sql.streaming.test")
+ .option("intOpt", 56)
+ .option("boolOpt", false)
+ .option("doubleOpt", 6.7)
+ .stream("/test")
+ .stop()
+
+ assert(LastOptions.parameters("intOpt") == "56")
+ assert(LastOptions.parameters("boolOpt") == "false")
+ assert(LastOptions.parameters("doubleOpt") == "6.7")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org