You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/05/13 12:56:16 UTC
spark git commit: [SPARK-18772][SQL] Avoid unnecessary conversion try
for special floats in JSON
Repository: spark
Updated Branches:
refs/heads/master b84ff7eb6 -> 3f98375d8
[SPARK-18772][SQL] Avoid unnecessary conversion try for special floats in JSON
## What changes were proposed in this pull request?
This PR is based on https://github.com/apache/spark/pull/16199 and extracts the valid change from https://github.com/apache/spark/pull/9759 to resolve SPARK-18772
This avoids additional conversion try with `toFloat` and `toDouble`.
For avoiding additional conversions, please refer the codes below:
**Before**
```scala
scala> import org.apache.spark.sql.types._
import org.apache.spark.sql.types._
scala> spark.read.schema(StructType(Seq(StructField("a", DoubleType)))).option("mode", "FAILFAST").json(Seq("""{"a": "nan"}""").toDS).show()
17/05/12 11:30:41 ERROR Executor: Exception in task 0.0 in stage 2.0 (TID 2)
java.lang.NumberFormatException: For input string: "nan"
...
```
**After**
```scala
scala> import org.apache.spark.sql.types._
import org.apache.spark.sql.types._
scala> spark.read.schema(StructType(Seq(StructField("a", DoubleType)))).option("mode", "FAILFAST").json(Seq("""{"a": "nan"}""").toDS).show()
17/05/12 11:44:30 ERROR Executor: Exception in task 0.0 in stage 0.0 (TID 0)
java.lang.RuntimeException: Cannot parse nan as DoubleType.
...
```
## How was this patch tested?
Unit tests added in `JsonSuite`.
Closes #16199
Author: hyukjinkwon <gu...@gmail.com>
Author: Nathan Howell <nh...@godaddy.com>
Closes #17956 from HyukjinKwon/SPARK-18772.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3f98375d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3f98375d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3f98375d
Branch: refs/heads/master
Commit: 3f98375d8a0f3d48041042d356915a5c298b1c8e
Parents: b84ff7e
Author: hyukjinkwon <gu...@gmail.com>
Authored: Sat May 13 20:56:04 2017 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Sat May 13 20:56:04 2017 +0800
----------------------------------------------------------------------
.../spark/sql/catalyst/json/JacksonParser.scala | 31 +++++----------
.../execution/datasources/json/JsonSuite.scala | 40 ++++++++++++++++++++
2 files changed, 50 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/3f98375d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
index ff6c93a..4ed6728 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/json/JacksonParser.scala
@@ -18,7 +18,6 @@
package org.apache.spark.sql.catalyst.json
import java.io.ByteArrayOutputStream
-import java.util.Locale
import scala.collection.mutable.ArrayBuffer
import scala.util.Try
@@ -126,16 +125,11 @@ class JacksonParser(
case VALUE_STRING =>
// Special case handling for NaN and Infinity.
- val value = parser.getText
- val lowerCaseValue = value.toLowerCase(Locale.ROOT)
- if (lowerCaseValue.equals("nan") ||
- lowerCaseValue.equals("infinity") ||
- lowerCaseValue.equals("-infinity") ||
- lowerCaseValue.equals("inf") ||
- lowerCaseValue.equals("-inf")) {
- value.toFloat
- } else {
- throw new RuntimeException(s"Cannot parse $value as FloatType.")
+ parser.getText match {
+ case "NaN" => Float.NaN
+ case "Infinity" => Float.PositiveInfinity
+ case "-Infinity" => Float.NegativeInfinity
+ case other => throw new RuntimeException(s"Cannot parse $other as FloatType.")
}
}
@@ -146,16 +140,11 @@ class JacksonParser(
case VALUE_STRING =>
// Special case handling for NaN and Infinity.
- val value = parser.getText
- val lowerCaseValue = value.toLowerCase(Locale.ROOT)
- if (lowerCaseValue.equals("nan") ||
- lowerCaseValue.equals("infinity") ||
- lowerCaseValue.equals("-infinity") ||
- lowerCaseValue.equals("inf") ||
- lowerCaseValue.equals("-inf")) {
- value.toDouble
- } else {
- throw new RuntimeException(s"Cannot parse $value as DoubleType.")
+ parser.getText match {
+ case "NaN" => Double.NaN
+ case "Infinity" => Double.PositiveInfinity
+ case "-Infinity" => Double.NegativeInfinity
+ case other => throw new RuntimeException(s"Cannot parse $other as DoubleType.")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/3f98375d/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
index 5e7f794..e66a60d 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.execution.datasources.json
import java.io.{File, StringWriter}
import java.nio.charset.StandardCharsets
import java.sql.{Date, Timestamp}
+import java.util.Locale
import com.fasterxml.jackson.core.JsonFactory
import org.apache.hadoop.fs.{Path, PathFilter}
@@ -1988,4 +1989,43 @@ class JsonSuite extends QueryTest with SharedSQLContext with TestJsonData {
assert(errMsg.startsWith("The field for corrupt records must be string type and nullable"))
}
}
+
+ test("SPARK-18772: Parse special floats correctly") {
+ val jsons = Seq(
+ """{"a": "NaN"}""",
+ """{"a": "Infinity"}""",
+ """{"a": "-Infinity"}""")
+
+ // positive cases
+ val checks: Seq[Double => Boolean] = Seq(
+ _.isNaN,
+ _.isPosInfinity,
+ _.isNegInfinity)
+
+ Seq(FloatType, DoubleType).foreach { dt =>
+ jsons.zip(checks).foreach { case (json, check) =>
+ val ds = spark.read
+ .schema(StructType(Seq(StructField("a", dt))))
+ .json(Seq(json).toDS())
+ .select($"a".cast(DoubleType)).as[Double]
+ assert(check(ds.first()))
+ }
+ }
+
+ // negative cases
+ Seq(FloatType, DoubleType).foreach { dt =>
+ val lowerCasedJsons = jsons.map(_.toLowerCase(Locale.ROOT))
+ // The special floats are case-sensitive so these cases below throw exceptions.
+ lowerCasedJsons.foreach { lowerCasedJson =>
+ val e = intercept[SparkException] {
+ spark.read
+ .option("mode", "FAILFAST")
+ .schema(StructType(Seq(StructField("a", dt))))
+ .json(Seq(lowerCasedJson).toDS())
+ .collect()
+ }
+ assert(e.getMessage.contains("Cannot parse"))
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org