You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/08/11 22:06:07 UTC
[spark] branch master updated: [SPARK-39927][BUILD] Upgrade to Avro 1.11.1
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 4394e244bbd [SPARK-39927][BUILD] Upgrade to Avro 1.11.1
4394e244bbd is described below
commit 4394e244bbd50d0b625b373351d38508f4debf41
Author: Ismaël Mejía <ie...@gmail.com>
AuthorDate: Thu Aug 11 15:05:41 2022 -0700
[SPARK-39927][BUILD] Upgrade to Avro 1.11.1
### What changes were proposed in this pull request?
Update the Avro version to 1.11.1
### Why are the changes needed?
To stay up to date with upstream
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Unit tests
Closes #37352 from iemejia/SPARK-39927-avro-1.11.1.
Authored-by: Ismaël Mejía <ie...@gmail.com>
Signed-off-by: Dongjoon Hyun <do...@apache.org>
---
.../avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala | 4 ++--
.../avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala | 5 ++---
dev/deps/spark-deps-hadoop-2-hive-2.3 | 6 +++---
dev/deps/spark-deps-hadoop-3-hive-2.3 | 6 +++---
docs/sql-data-sources-avro.md | 4 ++--
pom.xml | 2 +-
project/SparkBuild.scala | 2 +-
.../scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala | 2 +-
8 files changed, 15 insertions(+), 16 deletions(-)
diff --git a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
index 3c68cbd537a..540420974f5 100644
--- a/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
+++ b/connector/avro/src/main/scala/org/apache/spark/sql/avro/AvroOptions.scala
@@ -79,14 +79,14 @@ private[sql] class AvroOptions(
/**
* Top level record name in write result, which is required in Avro spec.
- * See https://avro.apache.org/docs/1.11.0/spec.html#schema_record .
+ * See https://avro.apache.org/docs/1.11.1/spec.html#schema_record .
* Default value is "topLevelRecord"
*/
val recordName: String = parameters.getOrElse("recordName", "topLevelRecord")
/**
* Record namespace in write result. Default value is "".
- * See Avro spec for details: https://avro.apache.org/docs/1.11.0/spec.html#schema_record .
+ * See Avro spec for details: https://avro.apache.org/docs/1.11.1/spec.html#schema_record .
*/
val recordNamespace: String = parameters.getOrElse("recordNamespace", "")
diff --git a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
index 8a088a43579..4a1749533ab 100644
--- a/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
+++ b/connector/avro/src/test/scala/org/apache/spark/sql/avro/AvroSuite.scala
@@ -875,7 +875,7 @@ abstract class AvroSuite
dfWithNull.write.format("avro")
.option("avroSchema", avroSchema).save(s"$tempDir/${UUID.randomUUID()}")
}
- assertExceptionMsg[AvroTypeException](e1, "Not an enum: null")
+ assertExceptionMsg[AvroTypeException](e1, "value null is not a SuitEnumType")
// Writing df containing data not in the enum will throw an exception
val e2 = intercept[SparkException] {
@@ -1075,8 +1075,7 @@ abstract class AvroSuite
.save(s"$tempDir/${UUID.randomUUID()}")
}.getCause.getMessage
assert(message.contains("Caused by: java.lang.NullPointerException: "))
- assert(message.contains(
- "null of string in string in field Name of test_schema in test_schema"))
+ assert(message.contains("null in string in field Name"))
}
}
diff --git a/dev/deps/spark-deps-hadoop-2-hive-2.3 b/dev/deps/spark-deps-hadoop-2-hive-2.3
index a86bbc52431..13e5e56ab0e 100644
--- a/dev/deps/spark-deps-hadoop-2-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-2-hive-2.3
@@ -23,9 +23,9 @@ arrow-memory-netty/9.0.0//arrow-memory-netty-9.0.0.jar
arrow-vector/9.0.0//arrow-vector-9.0.0.jar
audience-annotations/0.5.0//audience-annotations-0.5.0.jar
automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.11.0//avro-ipc-1.11.0.jar
-avro-mapred/1.11.0//avro-mapred-1.11.0.jar
-avro/1.11.0//avro-1.11.0.jar
+avro-ipc/1.11.1//avro-ipc-1.11.1.jar
+avro-mapred/1.11.1//avro-mapred-1.11.1.jar
+avro/1.11.1//avro-1.11.1.jar
azure-storage/2.0.0//azure-storage-2.0.0.jar
blas/2.2.1//blas-2.2.1.jar
bonecp/0.8.0.RELEASE//bonecp-0.8.0.RELEASE.jar
diff --git a/dev/deps/spark-deps-hadoop-3-hive-2.3 b/dev/deps/spark-deps-hadoop-3-hive-2.3
index 854919a9af6..c221e092806 100644
--- a/dev/deps/spark-deps-hadoop-3-hive-2.3
+++ b/dev/deps/spark-deps-hadoop-3-hive-2.3
@@ -22,9 +22,9 @@ arrow-memory-netty/9.0.0//arrow-memory-netty-9.0.0.jar
arrow-vector/9.0.0//arrow-vector-9.0.0.jar
audience-annotations/0.5.0//audience-annotations-0.5.0.jar
automaton/1.11-8//automaton-1.11-8.jar
-avro-ipc/1.11.0//avro-ipc-1.11.0.jar
-avro-mapred/1.11.0//avro-mapred-1.11.0.jar
-avro/1.11.0//avro-1.11.0.jar
+avro-ipc/1.11.1//avro-ipc-1.11.1.jar
+avro-mapred/1.11.1//avro-mapred-1.11.1.jar
+avro/1.11.1//avro-1.11.1.jar
aws-java-sdk-bundle/1.12.262//aws-java-sdk-bundle-1.12.262.jar
azure-data-lake-store-sdk/2.3.9//azure-data-lake-store-sdk-2.3.9.jar
azure-keyvault-core/1.0.0//azure-keyvault-core-1.0.0.jar
diff --git a/docs/sql-data-sources-avro.md b/docs/sql-data-sources-avro.md
index 28f41043a75..117692a7618 100644
--- a/docs/sql-data-sources-avro.md
+++ b/docs/sql-data-sources-avro.md
@@ -393,7 +393,7 @@ applications. Read the [Advanced Dependency Management](https://spark.apache
Submission Guide for more details.
## Supported types for Avro -> Spark SQL conversion
-Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.0/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.11.0/spec.html#schema_complex) under records of Avro.
+Currently Spark supports reading all [primitive types](https://avro.apache.org/docs/1.11.1/spec.html#schema_primitive) and [complex types](https://avro.apache.org/docs/1.11.1/spec.html#schema_complex) under records of Avro.
<table class="table">
<tr><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
<tr>
@@ -457,7 +457,7 @@ In addition to the types listed above, it supports reading `union` types. The fo
3. `union(something, null)`, where something is any supported Avro type. This will be mapped to the same Spark SQL type as that of something, with nullable set to true.
All other union types are considered complex. They will be mapped to StructType where field names are member0, member1, etc., in accordance with members of the union. This is consistent with the behavior when converting between Avro and Parquet.
-It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types):
+It also supports reading the following Avro [logical types](https://avro.apache.org/docs/1.11.1/spec.html#Logical+Types):
<table class="table">
<tr><th><b>Avro logical type</b></th><th><b>Avro type</b></th><th><b>Spark SQL type</b></th></tr>
diff --git a/pom.xml b/pom.xml
index 1bd36133c62..c197987cd53 100644
--- a/pom.xml
+++ b/pom.xml
@@ -144,7 +144,7 @@
-->
<codahale.metrics.version>4.2.10</codahale.metrics.version>
<!-- Should be consistent with SparkBuild.scala and docs -->
- <avro.version>1.11.0</avro.version>
+ <avro.version>1.11.1</avro.version>
<aws.kinesis.client.version>1.12.0</aws.kinesis.client.version>
<!-- Should be consistent with Kinesis client dependency -->
<aws.java.sdk.version>1.11.655</aws.java.sdk.version>
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 6a33aae6b41..21ab6f9f636 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -709,7 +709,7 @@ object DependencyOverrides {
dependencyOverrides += "com.google.guava" % "guava" % guavaVersion,
dependencyOverrides += "xerces" % "xercesImpl" % "2.12.0",
dependencyOverrides += "jline" % "jline" % "2.14.6",
- dependencyOverrides += "org.apache.avro" % "avro" % "1.11.0")
+ dependencyOverrides += "org.apache.avro" % "avro" % "1.11.1")
}
/**
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
index ad0f9a56a82..22698b91cb6 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveClientSuite.scala
@@ -895,7 +895,7 @@ class HiveClientSuite(version: String, allVersions: Seq[String])
test("Decimal support of Avro Hive serde") {
val tableName = "tab1"
// TODO: add the other logical types. For details, see the link:
- // https://avro.apache.org/docs/1.11.0/spec.html#Logical+Types
+ // https://avro.apache.org/docs/1.11.1/spec.html#Logical+Types
val avroSchema =
"""{
| "name": "test_record",
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org