You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2026/03/10 15:51:37 UTC

(jackrabbit-oak) branch use_exisitng_fs_to_initiate_new_fs created (now 30c505ad00)

This is an automated email from the ASF dual-hosted git repository.

miroslav pushed a change to branch use_exisitng_fs_to_initiate_new_fs
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git


      at 30c505ad00 init new filestore from old one

This branch includes the following new commits:

     new 30c505ad00 init new filestore from old one

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.



(jackrabbit-oak) 01/01: init new filestore from old one

Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

miroslav pushed a commit to branch use_exisitng_fs_to_initiate_new_fs
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git

commit 30c505ad00088cf10b0e9bc436cc37e4875baffa
Author: smiroslav <mi...@apache.com>
AuthorDate: Tue Mar 10 16:42:34 2026 +0100

    init new filestore from old one
---
 .../jackrabbit/oak/segment/file/FileStore.java     |   7 +-
 .../oak/segment/file/FileStoreBuilder.java         |  27 +++++
 .../jackrabbit/oak/segment/file/tar/TarFiles.java  |  50 +++++++++
 .../oak/segment/file/FileStoreInitFromTest.java    | 116 +++++++++++++++++++++
 .../oak/segment/file/tar/TarFilesTest.java         |  76 ++++++++++++++
 5 files changed, 275 insertions(+), 1 deletion(-)

diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index 68a2c3a6cb..0143ab90e7 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -172,7 +172,12 @@ public class FileStore extends AbstractFileStore {
                 .withInitialisedReadersAndWriters(false);
 
         this.tarFiles = tarFilesBuilder.build();
-        this.tarFiles.init();
+        TarFiles existingTarFiles = builder.getExistingTarFiles();
+        if (existingTarFiles != null) {
+            this.tarFiles.initFrom(existingTarFiles);
+        } else {
+            this.tarFiles.init();
+        }
 
         long size = this.tarFiles.size();
         this.stats.init(size);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
index e37e764f3a..3dafc038e6 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
@@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.segment.WriterCacheManager;
 import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
 import org.apache.jackrabbit.oak.segment.file.proc.Proc.Backend;
 import org.apache.jackrabbit.oak.segment.spi.persistence.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
 import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
 import org.apache.jackrabbit.oak.segment.spi.monitor.*;
 import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -149,6 +150,9 @@ public class FileStoreBuilder {
 
     private boolean eagerSegmentCaching;
 
+    @Nullable
+    private TarFiles existingTarFiles;
+
     private boolean built;
 
     /**
@@ -434,6 +438,24 @@ public class FileStoreBuilder {
         return this;
     }
 
+    /**
+     * Provide an existing {@link TarFiles} instance whose TAR readers will be
+     * transferred to the new store instead of re-opening them from disk.
+     * This avoids the expensive I/O of re-reading all TAR file indices.
+     * <p>
+     * After the new store is built, the existing {@code TarFiles} instance
+     * will have its readers detached and can be safely closed without
+     * affecting the new store.
+     *
+     * @param existingTarFiles the existing TarFiles to transfer readers from
+     * @return this instance
+     */
+    @NotNull
+    public FileStoreBuilder withExistingTarFiles(@NotNull TarFiles existingTarFiles) {
+        this.existingTarFiles = requireNonNull(existingTarFiles);
+        return this;
+    }
+
     /**
      * Sets the threshold under which binaries are inlined in data segments.
      * @param binariesInlineThreshold the threshold
@@ -622,6 +644,11 @@ public class FileStoreBuilder {
         return eagerSegmentCaching;
     }
 
+    @Nullable
+    TarFiles getExistingTarFiles() {
+        return existingTarFiles;
+    }
+
     int getBinariesInlineThreshold() {
         return binariesInlineThreshold;
     }
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index ed9f1fd770..ea653fb7ca 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -406,6 +406,56 @@ public class TarFiles implements Closeable {
         }
     }
 
+    /**
+     * Initializes this instance by transferring the existing TAR readers from
+     * the given {@code TarFiles} instance, instead of opening new ones from
+     * disk. This avoids the expensive I/O of re-reading all TAR file indices.
+     * <p>
+     * The existing instance's readers are atomically detached — after this call,
+     * the existing instance will have no readers and can be safely closed
+     * without affecting the transferred readers.
+     * <p>
+     * If this instance is not read-only, a new {@link TarWriter} is created.
+     *
+     * @param existing the {@code TarFiles} instance to transfer readers from
+     * @throws IOException if an error occurs while creating the TAR writer
+     */
+    public void initFrom(TarFiles existing) throws IOException {
+        Validate.checkState(!initialised, "TarFiles already initialised");
+
+        Node existingReaders;
+
+        // Atomically detach readers from the existing TarFiles
+        existing.lock.writeLock().lock();
+        try {
+            existingReaders = existing.readers;
+            existing.readers = null;
+        } finally {
+            existing.lock.writeLock().unlock();
+        }
+
+        // Adopt the transferred readers
+        this.readers = existingReaders;
+        for (TarReader reader : iterable(readers)) {
+            segmentCount.inc(getSegmentCount(reader));
+            readerCount.inc();
+        }
+
+        // Create writer if not read-only
+        if (!readOnly) {
+            Map<Integer, Map<Character, String>> map = collectFiles(archiveManager);
+            Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
+            Arrays.sort(indices);
+            int writeNumber = 0;
+            if (indices.length > 0) {
+                writeNumber = indices[indices.length - 1] + 1;
+            }
+            writer = new TarWriter(archiveManager, writeNumber, segmentCount);
+        }
+
+        initialised = true;
+    }
+
     public void init() throws IOException {
         Map<Integer, Map<Character, String>> map = collectFiles(archiveManager);
         Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java
new file mode 100644
index 0000000000..4671a1cde4
--- /dev/null
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Tests creating a writable {@link FileStore} from an existing
+ * {@link ReadOnlyFileStore}, reusing the already-opened TAR readers.
+ */
+public class FileStoreInitFromTest {
+
+    @Rule
+    public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+    @Test
+    public void testUpgradeReadOnlyToWritable() throws Exception {
+        File directory = folder.getRoot();
+
+        // Step 1: Create a writable FileStore and write some nodes and properties
+        try (FileStore original = fileStoreBuilder(directory)
+                .withMaxFileSize(1)
+                .withMemoryMapping(false)
+                .build()) {
+            SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(original).build();
+            NodeBuilder builder = nodeStore.getRoot().builder();
+            builder.child("content").setProperty("title", "Hello");
+            builder.child("content").child("page1").setProperty("text", "First page");
+            builder.child("content").child("page2").setProperty("text", "Second page");
+            nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+            original.flush();
+        }
+
+        // Step 2: Open as read-only and verify the content is there
+        ReadOnlyFileStore readOnly = fileStoreBuilder(directory)
+                .withMemoryMapping(false)
+                .buildReadOnly();
+
+        TarFiles readOnlyTarFiles = readOnly.getTarFiles();
+        int readerCount = readOnlyTarFiles.readerCount();
+        assertTrue("Expected at least one TAR reader", readerCount > 0);
+
+        NodeState roRoot = readOnly.getHead().getChildNode("root");
+        assertEquals("Hello", roRoot.getChildNode("content").getString("title"));
+
+        // Step 3: Create a new writable FileStore, transferring readers from read-only
+        FileStore writable = fileStoreBuilder(directory)
+                .withMaxFileSize(1)
+                .withMemoryMapping(false)
+                .withExistingTarFiles(readOnlyTarFiles)
+                .build();
+
+        // Verify readers were transferred, not re-opened
+        assertEquals(readerCount, writable.readerCount());
+        assertEquals(0, readOnlyTarFiles.readerCount());
+
+        // Close the read-only store — transferred readers must remain unaffected
+        readOnly.close();
+
+        // Step 4: Verify old content is accessible from the new writable store
+        SegmentNodeStore rwNodeStore = SegmentNodeStoreBuilders.builder(writable).build();
+        NodeState rwRoot = rwNodeStore.getRoot();
+        assertNotNull(rwRoot.getChildNode("content"));
+        assertEquals("Hello", rwRoot.getChildNode("content").getString("title"));
+        assertEquals("First page", rwRoot.getChildNode("content").getChildNode("page1").getString("text"));
+        assertEquals("Second page", rwRoot.getChildNode("content").getChildNode("page2").getString("text"));
+
+        // Step 5: Write new content
+        NodeBuilder rwBuilder = rwNodeStore.getRoot().builder();
+        rwBuilder.child("content").child("page3").setProperty("text", "Third page");
+        rwBuilder.child("config").setProperty("version", 2);
+        rwNodeStore.merge(rwBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+        writable.flush();
+
+        // Step 6: Verify both old and new content are present
+        NodeState finalRoot = rwNodeStore.getRoot();
+        assertEquals("Hello", finalRoot.getChildNode("content").getString("title"));
+        assertEquals("First page", finalRoot.getChildNode("content").getChildNode("page1").getString("text"));
+        assertEquals("Second page", finalRoot.getChildNode("content").getChildNode("page2").getString("text"));
+        assertEquals("Third page", finalRoot.getChildNode("content").getChildNode("page3").getString("text"));
+        assertNotNull(finalRoot.getChildNode("config").getProperty("version"));
+        assertEquals(2, (long) finalRoot.getChildNode("config").getProperty("version").getValue(org.apache.jackrabbit.oak.api.Type.LONG));
+
+        writable.close();
+    }
+}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
index 80789efdf1..6353860a9b 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
@@ -504,4 +504,80 @@ public class TarFilesTest {
 
         assertEquals(TOTAL_ENTRIES / WRITER_ENTRY_LIMIT, tarFiles.readerCount());
     }
+
+    @Test
+    public void testInitFromTransfersReaders() throws Exception {
+        // Write segments to create TAR files, then close
+        UUID id1 = randomUUID();
+        UUID id2 = randomUUID();
+        byte[] data1 = randomData();
+        byte[] data2 = randomData();
+
+        writeSegment(id1, data1);
+        writeSegment(id2, data2);
+        tarFiles.close();
+        tarFiles = null;
+
+        // Open read-only TarFiles
+        TarFiles readOnlyTarFiles = TarFiles.builder()
+                .withDirectory(folder.getRoot())
+                .withTarRecovery((id, data, recovery) -> {})
+                .withIOMonitor(new IOMonitorAdapter())
+                .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+                .withReadOnly()
+                .build();
+
+        int readerCount = readOnlyTarFiles.readerCount();
+        int segmentCount = readOnlyTarFiles.segmentCount();
+        assertTrue("Expected at least one reader", readerCount > 0);
+        assertTrue("Expected at least two segments", segmentCount >= 2);
+
+        // Create new writable TarFiles, transferring readers from read-only
+        TarFiles writableTarFiles = TarFiles.builder()
+                .withDirectory(folder.getRoot())
+                .withTarRecovery((id, data, recovery) -> {})
+                .withIOMonitor(new IOMonitorAdapter())
+                .withFileStoreMonitor(new FileStoreMonitorAdapter())
+                .withMaxFileSize(MAX_FILE_SIZE)
+                .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+                .withInitialisedReadersAndWriters(false)
+                .build();
+        writableTarFiles.initFrom(readOnlyTarFiles);
+
+        // Verify readers were transferred
+        assertEquals(readerCount, writableTarFiles.readerCount());
+        assertEquals(segmentCount, writableTarFiles.segmentCount());
+
+        // Verify read-only instance has no readers left
+        assertEquals(0, readOnlyTarFiles.readerCount());
+
+        // Verify existing segments are readable from the new instance
+        assertTrue(writableTarFiles.containsSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits()));
+        assertTrue(writableTarFiles.containsSegment(id2.getMostSignificantBits(), id2.getLeastSignificantBits()));
+
+        Buffer buf1 = writableTarFiles.readSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits());
+        byte[] read1 = new byte[buf1.remaining()];
+        buf1.get(read1);
+        assertArrayEquals(data1, read1);
+
+        Buffer buf2 = writableTarFiles.readSegment(id2.getMostSignificantBits(), id2.getLeastSignificantBits());
+        byte[] read2 = new byte[buf2.remaining()];
+        buf2.get(read2);
+        assertArrayEquals(data2, read2);
+
+        // Verify writing new segments works
+        UUID id3 = randomUUID();
+        byte[] data3 = randomData();
+        writableTarFiles.writeSegment(id3, data3, 0, data3.length, newGCGeneration(1, 1, false), emptySet(), emptySet());
+        assertTrue(writableTarFiles.containsSegment(id3.getMostSignificantBits(), id3.getLeastSignificantBits()));
+
+        // Close old read-only instance — should not affect transferred readers
+        readOnlyTarFiles.close();
+
+        // Verify segments are still readable after closing the old instance
+        assertTrue(writableTarFiles.containsSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits()));
+        assertTrue(writableTarFiles.containsSegment(id3.getMostSignificantBits(), id3.getLeastSignificantBits()));
+
+        writableTarFiles.close();
+    }
 }