You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by yu...@apache.org on 2022/09/30 06:39:12 UTC
[hudi] 05/10: [HUDI-4885] Adding org.apache.avro to hudi-hive-sync bundle (#6729)
This is an automated email from the ASF dual-hosted git repository.
yuzhaojing pushed a commit to branch release-0.12.1
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 26bbcec8756c2a7ecdf03267421117acfe4e0f2c
Author: Sivabalan Narayanan <n....@gmail.com>
AuthorDate: Thu Sep 29 02:06:32 2022 -0700
[HUDI-4885] Adding org.apache.avro to hudi-hive-sync bundle (#6729)
---
.../src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala | 6 +++++-
packaging/hudi-hive-sync-bundle/pom.xml | 9 +++++++++
pom.xml | 4 ++--
3 files changed, 16 insertions(+), 3 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
index 3da7a3ddd4..89d8faaf2e 100644
--- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
+++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/TestHoodieSparkUtils.scala
@@ -196,7 +196,11 @@ class TestHoodieSparkUtils {
fail("createRdd should fail, because records don't have a column which is not nullable in the passed in schema")
} catch {
case e: Exception =>
- assertTrue(e.getMessage.contains("null of string in field new_nested_col of test_namespace.test_struct_name.nullableInnerStruct of union"))
+ if (HoodieSparkUtils.gteqSpark3_3) {
+ assertTrue(e.getMessage.contains("null value for (non-nullable) string at test_struct_name.nullableInnerStruct[nullableInnerStruct].new_nested_col"))
+ } else {
+ assertTrue(e.getMessage.contains("null of string in field new_nested_col of test_namespace.test_struct_name.nullableInnerStruct of union"))
+ }
}
spark.stop()
}
diff --git a/packaging/hudi-hive-sync-bundle/pom.xml b/packaging/hudi-hive-sync-bundle/pom.xml
index 87ae223a92..9c42d86287 100644
--- a/packaging/hudi-hive-sync-bundle/pom.xml
+++ b/packaging/hudi-hive-sync-bundle/pom.xml
@@ -72,6 +72,7 @@
<include>org.apache.hudi:hudi-hive-sync</include>
<include>com.beust:jcommander</include>
+ <include>org.apache.avro:avro</include>
<include>org.apache.hbase:hbase-common</include>
<include>org.apache.hbase:hbase-client</include>
<include>org.apache.hbase:hbase-hadoop-compat</include>
@@ -256,5 +257,13 @@
<scope>compile</scope>
</dependency>
+ <!-- Avro -->
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ <version>${avro.version}</version>
+ <scope>compile</scope>
+ </dependency>
+
</dependencies>
</project>
diff --git a/pom.xml b/pom.xml
index 7f591f8d3b..b92245c953 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1958,7 +1958,7 @@
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
- <avro.version>1.11.0</avro.version>
+ <avro.version>1.11.1</avro.version>
<orc.version>1.7.4</orc.version>
<antlr.version>4.8</antlr.version>
<fasterxml.spark3.version>2.13.3</fasterxml.spark3.version>
@@ -2067,7 +2067,7 @@
<scalatest.version>${scalatest.spark3.version}</scalatest.version>
<kafka.version>${kafka.spark3.version}</kafka.version>
<parquet.version>1.12.2</parquet.version>
- <avro.version>1.11.0</avro.version>
+ <avro.version>1.11.1</avro.version>
<orc.version>1.7.4</orc.version>
<antlr.version>4.8</antlr.version>
<fasterxml.spark3.version>2.13.3</fasterxml.spark3.version>