You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/11/16 00:35:11 UTC
spark git commit: [SPARK-25883][BACKPORT][SQL][MINOR] Override method
`prettyName` in `from_avro`/`to_avro`
Repository: spark
Updated Branches:
refs/heads/branch-2.4 96834fb77 -> 6148a77a5
[SPARK-25883][BACKPORT][SQL][MINOR] Override method `prettyName` in `from_avro`/`to_avro`
Back port https://github.com/apache/spark/pull/22890 to branch-2.4.
It is a bug fix for this issue:
https://issues.apache.org/jira/browse/SPARK-26063
## What changes were proposed in this pull request?
Previously in from_avro/to_avro, we override the method `simpleString` and `sql` for the string output. However, the override only affects the alias naming:
```
Project [from_avro('col,
...
, (mode,PERMISSIVE)) AS from_avro(col, struct<col1:bigint,col2:double>, Map(mode -> PERMISSIVE))#11]
```
It only makes the alias name quite long: `from_avro(col, struct<col1:bigint,col2:double>, Map(mode -> PERMISSIVE))`).
We should follow `from_csv`/`from_json` here, to override the method prettyName only, and we will get a clean alias name
```
... AS from_avro(col)#11
```
## How was this patch tested?
Manual check
Closes #23047 from gengliangwang/backport_avro_pretty_name.
Authored-by: Gengliang Wang <ge...@databricks.com>
Signed-off-by: hyukjinkwon <gu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6148a77a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6148a77a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6148a77a
Branch: refs/heads/branch-2.4
Commit: 6148a77a5da9ca33fb115269f1cba29cddfc652e
Parents: 96834fb
Author: Gengliang Wang <ge...@databricks.com>
Authored: Fri Nov 16 08:35:00 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Fri Nov 16 08:35:00 2018 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala | 8 +-------
.../scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala | 8 +-------
2 files changed, 2 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6148a77a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
----------------------------------------------------------------------
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
index 915769f..8641b9f 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/AvroDataToCatalyst.scala
@@ -51,13 +51,7 @@ case class AvroDataToCatalyst(child: Expression, jsonFormatSchema: String)
deserializer.deserialize(result)
}
- override def simpleString: String = {
- s"from_avro(${child.sql}, ${dataType.simpleString})"
- }
-
- override def sql: String = {
- s"from_avro(${child.sql}, ${dataType.catalogString})"
- }
+ override def prettyName: String = "from_avro"
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val expr = ctx.addReferenceObj("this", this)
http://git-wip-us.apache.org/repos/asf/spark/blob/6148a77a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
----------------------------------------------------------------------
diff --git a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
index 141ff37..6ed330d 100644
--- a/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
+++ b/external/avro/src/main/scala/org/apache/spark/sql/avro/CatalystDataToAvro.scala
@@ -52,13 +52,7 @@ case class CatalystDataToAvro(child: Expression) extends UnaryExpression {
out.toByteArray
}
- override def simpleString: String = {
- s"to_avro(${child.sql}, ${child.dataType.simpleString})"
- }
-
- override def sql: String = {
- s"to_avro(${child.sql}, ${child.dataType.catalogString})"
- }
+ override def prettyName: String = "to_avro"
override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val expr = ctx.addReferenceObj("this", this)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org