You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2026/09/03 20:18:05 UTC
(kafka) branch trunk updated: KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)
This is an automated email from the ASF dual-hosted git repository.
junrao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 18cdc642d8c KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)
18cdc642d8c is described below
commit 18cdc642d8ce7aff66f5db9b682e552c09e2c796
Author: Tushar saini <39...@users.noreply.github.com>
AuthorDate: Fri Sep 4 01:47:58 2026 +0530
KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)
When `log.preallocate=true`, a new log segment is created at the configured initial file size. The physical file is larger than the logical data until `FileRecords.trim()` truncates it to `sizeInBytes()` on segment roll or
close.
`FileRecords.close()` previously called `flush()` before `trim()`. The fsync persisted message data but not the smaller file length set by `truncate()`, because truncation happened after the fsync. If the broker shut down
cleanly and the process crashed before the OS flushed file metadata, the on-disk `.log` file could still report the preallocated length.
On restart, recovery is skipped for a clean shutdown, so `readNextOffset()` can scan past valid records into uninitialized or zero-filled bytes and fail.
Reviewers: Jun Rao <ju...@gmail.com>
---
.../kafka/common/record/internal/FileRecords.java | 4 +++-
.../common/record/internal/FileRecordsTest.java | 22 ++++++++++++++++++++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
index 7a7ae0f4e00..29adde39595 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
@@ -215,8 +215,10 @@ public class FileRecords extends AbstractRecords implements Closeable {
return;
}
- flush();
trim();
+ // flush() must run after trim() so the truncated file length is included in the fsync.
+ // A flush before trim only persists message data, not the smaller size set by trim().
+ flush();
channel.close();
}
diff --git a/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
index f0831f556d9..f8845d5c987 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.InOrder;
import org.mockito.Mockito;
import java.io.File;
@@ -59,6 +60,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -388,6 +390,26 @@ public class FileRecordsTest {
assertEquals(0, temp.length());
}
+ /**
+ * Closing a preallocated file must fsync after trimming so the truncated length is durable.
+ */
+ @Test
+ public void testCloseFlushesAfterTrim() throws IOException {
+ FileChannel channelMock = mock(FileChannel.class);
+
+ when(channelMock.size()).thenReturn(1024L);
+ when(channelMock.isOpen()).thenReturn(true);
+ when(channelMock.truncate(anyLong())).thenReturn(channelMock);
+ when(channelMock.position(anyLong())).thenReturn(channelMock);
+
+ FileRecords records = new FileRecords(tempFile(), channelMock, 100);
+ records.close();
+
+ InOrder inOrder = inOrder(channelMock);
+ inOrder.verify(channelMock).truncate(100L);
+ inOrder.verify(channelMock).force(true);
+ }
+
/**
* Test the new FileRecords with pre allocate as true and file has been clearly shut down, the file will be truncate to end of valid data.
*/