You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2022/07/25 04:42:20 UTC
[hudi] branch master updated: [HUDI-4456] Close FileSystem in SparkClientFunctionalTestHarness (#6201)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 76a28daeb0 [HUDI-4456] Close FileSystem in SparkClientFunctionalTestHarness (#6201)
76a28daeb0 is described below
commit 76a28daeb08e7192d75dfc447624c827643bef0d
Author: Tim Brown <ti...@onehouse.ai>
AuthorDate: Sun Jul 24 21:42:15 2022 -0700
[HUDI-4456] Close FileSystem in SparkClientFunctionalTestHarness (#6201)
---
.../hudi/testutils/SparkClientFunctionalTestHarness.java | 15 ++++++++++++++-
1 file changed, 14 insertions(+), 1 deletion(-)
diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
index f9676c6c47..c58dd178dc 100644
--- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
+++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/testutils/SparkClientFunctionalTestHarness.java
@@ -67,6 +67,7 @@ import org.apache.spark.sql.Row;
import org.apache.spark.sql.SQLContext;
import org.apache.spark.sql.SparkSession;
import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.io.TempDir;
@@ -96,6 +97,7 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
private static transient JavaSparkContext jsc;
private static transient HoodieSparkEngineContext context;
private static transient TimelineService timelineService;
+ private FileSystem fileSystem;
/**
* An indicator of the initialization status.
@@ -128,7 +130,10 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
}
public FileSystem fs() {
- return FSUtils.getFs(basePath(), hadoopConf());
+ if (fileSystem == null) {
+ fileSystem = FSUtils.getFs(basePath(), hadoopConf());
+ }
+ return fileSystem;
}
@Override
@@ -208,6 +213,14 @@ public class SparkClientFunctionalTestHarness implements SparkProvider, HoodieMe
}
}
+ @AfterEach
+ public void closeFileSystem() throws IOException {
+ if (fileSystem != null) {
+ fileSystem.close();
+ fileSystem = null;
+ }
+ }
+
protected JavaRDD<HoodieRecord> tagLocation(
HoodieIndex index, JavaRDD<HoodieRecord> records, HoodieTable table) {
return HoodieJavaRDD.getJavaRDD(