You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/09/19 05:39:34 UTC
spark git commit: [SPARK-23173][SQL] rename
spark.sql.fromJsonForceNullableSchema
Repository: spark
Updated Branches:
refs/heads/master 497f00f62 -> 6c7db7fd1
[SPARK-23173][SQL] rename spark.sql.fromJsonForceNullableSchema
## What changes were proposed in this pull request?
`spark.sql.fromJsonForceNullableSchema` -> `spark.sql.function.fromJson.forceNullable`
## How was this patch tested?
Made sure there are no more references to `spark.sql.fromJsonForceNullableSchema`.
Closes #22459 from rxin/SPARK-23173.
Authored-by: Reynold Xin <rx...@databricks.com>
Signed-off-by: gatorsmile <ga...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c7db7fd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c7db7fd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c7db7fd
Branch: refs/heads/master
Commit: 6c7db7fd1ced1d143b1389d09990a620fc16be46
Parents: 497f00f
Author: Reynold Xin <rx...@databricks.com>
Authored: Tue Sep 18 22:39:29 2018 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Tue Sep 18 22:39:29 2018 -0700
----------------------------------------------------------------------
.../sql/catalyst/expressions/jsonExpressions.scala | 4 ++--
.../org/apache/spark/sql/internal/SQLConf.scala | 16 ++++++++--------
2 files changed, 10 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6c7db7fd/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
index bd9090a..ade10ab 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/jsonExpressions.scala
@@ -517,12 +517,12 @@ case class JsonToStructs(
timeZoneId: Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression with CodegenFallback with ExpectsInputTypes {
- val forceNullableSchema = SQLConf.get.getConf(SQLConf.FROM_JSON_FORCE_NULLABLE_SCHEMA)
+ val forceNullableSchema: Boolean = SQLConf.get.getConf(SQLConf.FROM_JSON_FORCE_NULLABLE_SCHEMA)
// The JSON input data might be missing certain fields. We force the nullability
// of the user-provided schema to avoid data corruptions. In particular, the parquet-mr encoder
// can generate incorrect files if values are missing in columns declared as non-nullable.
- val nullableSchema = if (forceNullableSchema) schema.asNullable else schema
+ val nullableSchema: DataType = if (forceNullableSchema) schema.asNullable else schema
override def nullable: Boolean = true
http://git-wip-us.apache.org/repos/asf/spark/blob/6c7db7fd/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 4928560..bdc4007 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -624,14 +624,6 @@ object SQLConf {
.stringConf
.createWithDefault("_corrupt_record")
- val FROM_JSON_FORCE_NULLABLE_SCHEMA = buildConf("spark.sql.fromJsonForceNullableSchema")
- .internal()
- .doc("When true, force the output schema of the from_json() function to be nullable " +
- "(including all the fields). Otherwise, the schema might not be compatible with" +
- "actual data, which leads to curruptions.")
- .booleanConf
- .createWithDefault(true)
-
val BROADCAST_TIMEOUT = buildConf("spark.sql.broadcastTimeout")
.doc("Timeout in seconds for the broadcast wait time in broadcast joins.")
.timeConf(TimeUnit.SECONDS)
@@ -1354,6 +1346,14 @@ object SQLConf {
"When this conf is not set, the value from `spark.redaction.string.regex` is used.")
.fallbackConf(org.apache.spark.internal.config.STRING_REDACTION_PATTERN)
+ val FROM_JSON_FORCE_NULLABLE_SCHEMA = buildConf("spark.sql.function.fromJson.forceNullable")
+ .internal()
+ .doc("When true, force the output schema of the from_json() function to be nullable " +
+ "(including all the fields). Otherwise, the schema might not be compatible with" +
+ "actual data, which leads to corruptions.")
+ .booleanConf
+ .createWithDefault(true)
+
val CONCAT_BINARY_AS_STRING = buildConf("spark.sql.function.concatBinaryAsString")
.doc("When this option is set to false and all inputs are binary, `functions.concat` returns " +
"an output as binary. Otherwise, it returns as a string. ")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org