(hudi) branch master updated: [MINOR] Fix unit tests (#10362)

[yi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

yihua pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 76cbca3e8dd [MINOR] Fix unit tests (#10362)
76cbca3e8dd is described below

commit 76cbca3e8ddffb2afe928eb6a13c8f00528f399a
Author: Geser Dugarov, PhD <ge...@gmail.com>
AuthorDate: Wed Jan 10 23:52:36 2024 +0700

    [MINOR] Fix unit tests (#10362)
---
 .../java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java    | 4 ++++
 .../hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java    | 5 +++++
 .../apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java | 4 +++-
 3 files changed, 12 insertions(+), 1 deletion(-)

diff --git a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
index e7b51e50bbb..42242fdfa32 100644
--- a/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
+++ b/hudi-client/hudi-java-client/src/test/java/org/apache/hudi/client/TestJavaHoodieBackedMetadata.java
@@ -71,6 +71,7 @@ import org.apache.hudi.common.testutils.HoodieTestDataGenerator;
 import org.apache.hudi.common.testutils.HoodieTestTable;
 import org.apache.hudi.common.testutils.HoodieTestUtils;
 import org.apache.hudi.common.util.HoodieTimer;
+import org.apache.hudi.common.util.JsonUtils;
 import org.apache.hudi.common.util.Option;
 import org.apache.hudi.common.util.collection.ClosableIterator;
 import org.apache.hudi.common.util.collection.ExternalSpillableMap;
@@ -492,6 +493,9 @@ public class TestJavaHoodieBackedMetadata extends TestHoodieMetadataBase {
             .withMaxNumDeltaCommitsBeforeCompaction(12) // cannot restore to before the oldest compaction on MDT as there are no base files before that time
             .build())
         .build();
+    // module com.fasterxml.jackson.datatype:jackson-datatype-jsr310 is needed for proper column stats processing for Jackson >= 2.11 (Spark >= 3.3)
+    // Java 8 date/time type `java.time.LocalDate` is not supported by default
+    JsonUtils.registerModules();
     init(tableType, writeConfig);
     testTableOperationsForMetaIndexImpl(writeConfig);
   }
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
index 594fe3d1028..cb9c2b85aa3 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/HoodieDeltaStreamerTestBase.java
@@ -245,6 +245,11 @@ public class HoodieDeltaStreamerTestBase extends UtilitiesTestBase {
   }
 
   @AfterAll
+  public static void tearDown() {
+    cleanupKafkaTestUtils();
+    UtilitiesTestBase.cleanUpUtilitiesTestServices();
+  }
+
   public static void cleanupKafkaTestUtils() {
     if (testUtils != null) {
       testUtils.teardown();
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
index b39c5f260df..7217993cc9c 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestHoodieDeltaStreamer.java
@@ -2291,7 +2291,9 @@ public class TestHoodieDeltaStreamer extends HoodieDeltaStreamerTestBase {
       testCsvDFSSource(false, '\t', false, Collections.singletonList(TripsWithDistanceTransformer.class.getName()));
     }, "Should error out when doing the transformation.");
     LOG.debug("Expected error during transformation", e);
-    assertTrue(e.getMessage().contains("cannot resolve 'begin_lat' given input columns:"));
+    // first version for Spark >= 3.3, the second one is for Spark < 3.3
+    assertTrue(e.getMessage().contains("Column 'begin_lat' does not exist. Did you mean one of the following?")
+        || e.getMessage().contains("cannot resolve 'begin_lat' given input columns:"));
   }
 
   @Test