You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kafka.apache.org by ju...@apache.org on 2026/09/03 20:18:05 UTC

(kafka) branch trunk updated: KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)

This is an automated email from the ASF dual-hosted git repository.

junrao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git


The following commit(s) were added to refs/heads/trunk by this push:
     new 18cdc642d8c KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)
18cdc642d8c is described below

commit 18cdc642d8ce7aff66f5db9b682e552c09e2c796
Author: Tushar saini <39...@users.noreply.github.com>
AuthorDate: Fri Sep 4 01:47:58 2026 +0530

    KAFKA-21000: Ensure FileRecords trim is fsynced when shutting down (#23303)
    
    When `log.preallocate=true`, a new log segment is created at the configured initial file size. The physical file is larger than the logical data until `FileRecords.trim()` truncates it to `sizeInBytes()` on segment roll or
    close.
    
    `FileRecords.close()` previously called `flush()` before `trim()`. The fsync persisted message data but not the smaller file length set by `truncate()`, because truncation happened after the fsync. If the broker shut down
    cleanly and the process crashed before the OS flushed file metadata, the on-disk `.log` file could still report the preallocated length.
    
    On restart, recovery is skipped for a clean shutdown, so `readNextOffset()` can scan past valid records into uninitialized or zero-filled bytes and fail.
    
    Reviewers: Jun Rao <ju...@gmail.com>
---
 .../kafka/common/record/internal/FileRecords.java  |  4 +++-
 .../common/record/internal/FileRecordsTest.java    | 22 ++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java b/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
index 7a7ae0f4e00..29adde39595 100644
--- a/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
+++ b/clients/src/main/java/org/apache/kafka/common/record/internal/FileRecords.java
@@ -215,8 +215,10 @@ public class FileRecords extends AbstractRecords implements Closeable {
             return;
         }
 
-        flush();
         trim();
+        // flush() must run after trim() so the truncated file length is included in the fsync.
+        // A flush before trim only persists message data, not the smaller size set by trim().
+        flush();
         channel.close();
     }
 
diff --git a/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java b/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
index f0831f556d9..f8845d5c987 100644
--- a/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
+++ b/clients/src/test/java/org/apache/kafka/common/record/internal/FileRecordsTest.java
@@ -29,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.ValueSource;
+import org.mockito.InOrder;
 import org.mockito.Mockito;
 
 import java.io.File;
@@ -59,6 +60,7 @@ import static org.mockito.ArgumentMatchers.anyLong;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.atLeastOnce;
 import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.inOrder;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -388,6 +390,26 @@ public class FileRecordsTest {
         assertEquals(0, temp.length());
     }
 
+    /**
+     * Closing a preallocated file must fsync after trimming so the truncated length is durable.
+     */
+    @Test
+    public void testCloseFlushesAfterTrim() throws IOException {
+        FileChannel channelMock = mock(FileChannel.class);
+
+        when(channelMock.size()).thenReturn(1024L);
+        when(channelMock.isOpen()).thenReturn(true);
+        when(channelMock.truncate(anyLong())).thenReturn(channelMock);
+        when(channelMock.position(anyLong())).thenReturn(channelMock);
+
+        FileRecords records = new FileRecords(tempFile(), channelMock, 100);
+        records.close();
+
+        InOrder inOrder = inOrder(channelMock);
+        inOrder.verify(channelMock).truncate(100L);
+        inOrder.verify(channelMock).force(true);
+    }
+
     /**
      * Test the new FileRecords with pre allocate as true and file has been clearly shut down, the file will be truncate to end of valid data.
      */