You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by si...@apache.org on 2023/04/08 21:52:17 UTC
[hudi] branch release-0.12.3 updated: Fixing closing of resourcs in TestHoodieLogFormat
This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch release-0.12.3
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/release-0.12.3 by this push:
new 266936114d1 Fixing closing of resourcs in TestHoodieLogFormat
266936114d1 is described below
commit 266936114d12ea6e7a807b90c9621eee6350ee8d
Author: sivabalan <n....@gmail.com>
AuthorDate: Sat Apr 8 14:52:05 2023 -0700
Fixing closing of resourcs in TestHoodieLogFormat
---
.../common/functional/TestHoodieLogFormat.java | 26 ++++++++++++++++++----
1 file changed, 22 insertions(+), 4 deletions(-)
diff --git a/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
index 4c8bea022bb..94a517dd80d 100755
--- a/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
+++ b/hudi-common/src/test/java/org/apache/hudi/common/functional/TestHoodieLogFormat.java
@@ -103,7 +103,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.params.provider.Arguments.arguments;
/**
- * Tests hoodie log format {@link HoodieLogFormat}
+ * Tests hoodie log format {@link HoodieLogFormat}.
*/
@SuppressWarnings("Duplicates")
public class TestHoodieLogFormat extends HoodieCommonTestHarness {
@@ -149,6 +149,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertEquals(0, writer.getCurrentSize(), "Just created this log, size should be 0");
assertTrue(writer.getLogFile().getFileName().startsWith("."), "Check all log files should start with a .");
assertEquals(1, writer.getLogFile().getLogVersion(), "Version should be 1 for new log created");
+ writer.close();
}
@ParameterizedTest
@@ -404,7 +405,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
reader.close();
}
- // @Test
+ @Test
public void testHugeLogFileWrite() throws IOException, URISyntaxException, InterruptedException {
Writer writer =
HoodieLogFormat.newWriterBuilder().onParentPath(partitionPath).withFileExtension(HoodieLogFile.DELTA_EXTENSION)
@@ -604,7 +605,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertEquals(scannedRecords.size(), allRecords.stream().mapToLong(Collection::size).sum(),
"Scanner records count should be the same as appended records");
-
+ scanner.close();
}
@Test
@@ -837,6 +838,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
copyOfRecords1.stream().map(s -> ((GenericRecord) s).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString())
.collect(Collectors.toSet());
assertEquals(originalKeys, readKeys, "CompositeAvroLogReader should return 200 records from 2 versions");
+ scanner.close();
}
@ParameterizedTest
@@ -915,6 +917,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
copyOfRecords1.stream().map(s -> ((GenericRecord) s).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString())
.collect(Collectors.toSet());
assertEquals(originalKeys, readKeys, "CompositeAvroLogReader should return 200 records from 2 versions");
+ scanner.close();
}
@ParameterizedTest
@@ -1002,6 +1005,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
copyOfRecords1.stream().map(s -> ((GenericRecord) s).get(HoodieRecord.RECORD_KEY_METADATA_FIELD).toString())
.collect(Collectors.toSet());
assertEquals(originalKeys, readKeys, "CompositeAvroLogReader should return 200 records from 2 versions");
+ scanner.close();
}
@ParameterizedTest
@@ -1103,6 +1107,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
FileCreateUtils.deleteDeltaCommit(basePath, "102", fs);
readKeys.clear();
+ scanner.close();
scanner = HoodieMergedLogRecordScanner.newBuilder()
.withFileSystem(fs)
.withBasePath(basePath)
@@ -1119,6 +1124,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
.build();
scanner.forEach(s -> readKeys.add(s.getKey().getRecordKey()));
assertEquals(200, readKeys.size(), "Stream collect should return all 200 records after rollback of delete");
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1236,6 +1243,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
Collections.sort(originalKeys);
Collections.sort(readKeys);
assertEquals(originalKeys, readKeys, "HoodieMergedLogRecordScanner should return 180 records from 4 versions");
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1321,6 +1330,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
scanner.forEach(s -> readKeys.add(s.getKey().getRecordKey()));
assertEquals(0, readKeys.size(), "Stream collect should return all 0 records");
FileCreateUtils.deleteDeltaCommit(basePath, "100", fs);
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1385,6 +1396,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
.build();
assertEquals(0, scanner.getTotalLogRecords(), "We would read 0 records");
FileCreateUtils.deleteDeltaCommit(basePath, "100", fs);
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1438,6 +1451,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
final List<String> readKeys = new ArrayList<>(100);
scanner.forEach(s -> readKeys.add(s.getKey().getRecordKey()));
assertEquals(100, readKeys.size(), "Stream collect should return all 150 records");
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1505,6 +1520,8 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
.withBitCaskDiskMapCompressionEnabled(isCompressionEnabled)
.build();
assertEquals(0, scanner.getTotalLogRecords(), "We would read 0 records");
+ writer.close();
+ scanner.close();
}
@ParameterizedTest
@@ -1612,6 +1629,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
.build();
assertEquals(0, scanner.getTotalLogRecords(), "We would read 0 records");
FileCreateUtils.deleteDeltaCommit(basePath, "100", fs);
+ scanner.close();
}
/*
@@ -1687,7 +1705,7 @@ public class TestHoodieLogFormat extends HoodieCommonTestHarness {
assertEquals(Math.max(numRecordsInLog1, numRecordsInLog2), scanner.getNumMergedRecordsInLog(),
"We would read 100 records");
-
+ scanner.close();
} catch (Exception e) {
e.printStackTrace();
}