You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by ga...@apache.org on 2021/01/21 15:04:43 UTC
[hudi] branch master updated: [HUDI-1512] Fix spark 2 unit tests
failure with Spark 3 (#2412)
This is an automated email from the ASF dual-hosted git repository.
garyli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 976420c [HUDI-1512] Fix spark 2 unit tests failure with Spark 3 (#2412)
976420c is described below
commit 976420c49a3fe764f7cecd30b6cdb32861be5537
Author: wenningd <we...@gmail.com>
AuthorDate: Thu Jan 21 07:04:28 2021 -0800
[HUDI-1512] Fix spark 2 unit tests failure with Spark 3 (#2412)
* [HUDI-1512] Fix spark 2 unit tests failure with Spark 3
* resolve comments
Co-authored-by: Wenning Ding <we...@amazon.com>
---
hudi-spark-datasource/hudi-spark2/pom.xml | 9 ++++++++-
pom.xml | 9 ++++++---
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/hudi-spark-datasource/hudi-spark2/pom.xml b/hudi-spark-datasource/hudi-spark2/pom.xml
index 8668984..1f6f792 100644
--- a/hudi-spark-datasource/hudi-spark2/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2/pom.xml
@@ -125,6 +125,13 @@
</configuration>
</plugin>
<plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <skipTests>${skip.hudi-spark2.unit.tests}</skipTests>
+ </configuration>
+ </plugin>
+ <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
</plugin>
@@ -144,7 +151,7 @@
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
- <version>${scala.version}</version>
+ <version>${scala11.version}</version>
</dependency>
<!-- Hoodie -->
diff --git a/pom.xml b/pom.xml
index 6a39cef..4f8b153 100644
--- a/pom.xml
+++ b/pom.xml
@@ -105,14 +105,15 @@
<metrics.version>4.1.1</metrics.version>
<prometheus.version>0.8.0</prometheus.version>
<http.version>4.4.1</http.version>
- <spark.version>2.4.4</spark.version>
+ <spark.version>${spark2.version}</spark.version>
<flink.version>1.12.0</flink.version>
<spark2.version>2.4.4</spark2.version>
<spark3.version>3.0.0</spark3.version>
<avro.version>1.8.2</avro.version>
- <scala.version>2.11.12</scala.version>
- <scala.binary.version>2.11</scala.binary.version>
+ <scala11.version>2.11.12</scala11.version>
<scala12.version>2.12.10</scala12.version>
+ <scala.version>${scala11.version}</scala.version>
+ <scala.binary.version>2.11</scala.binary.version>
<apache-rat-plugin.version>0.12</apache-rat-plugin.version>
<scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
<scalatest.version>3.0.1</scalatest.version>
@@ -140,6 +141,7 @@
<presto.bundle.bootstrap.scope>compile</presto.bundle.bootstrap.scope>
<presto.bundle.bootstrap.shade.prefix>org.apache.hudi.</presto.bundle.bootstrap.shade.prefix>
<shadeSources>true</shadeSources>
+ <skip.hudi-spark2.unit.tests>false</skip.hudi-spark2.unit.tests>
</properties>
<scm>
@@ -1361,6 +1363,7 @@
<fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
<fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
<fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+ <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
</properties>
<activation>
<property>