You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2026/03/10 15:51:38 UTC
(jackrabbit-oak) 01/01: init new filestore from old one
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch use_exisitng_fs_to_initiate_new_fs
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 30c505ad00088cf10b0e9bc436cc37e4875baffa
Author: smiroslav <mi...@apache.com>
AuthorDate: Tue Mar 10 16:42:34 2026 +0100
init new filestore from old one
---
.../jackrabbit/oak/segment/file/FileStore.java | 7 +-
.../oak/segment/file/FileStoreBuilder.java | 27 +++++
.../jackrabbit/oak/segment/file/tar/TarFiles.java | 50 +++++++++
.../oak/segment/file/FileStoreInitFromTest.java | 116 +++++++++++++++++++++
.../oak/segment/file/tar/TarFilesTest.java | 76 ++++++++++++++
5 files changed, 275 insertions(+), 1 deletion(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index 68a2c3a6cb..0143ab90e7 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -172,7 +172,12 @@ public class FileStore extends AbstractFileStore {
.withInitialisedReadersAndWriters(false);
this.tarFiles = tarFilesBuilder.build();
- this.tarFiles.init();
+ TarFiles existingTarFiles = builder.getExistingTarFiles();
+ if (existingTarFiles != null) {
+ this.tarFiles.initFrom(existingTarFiles);
+ } else {
+ this.tarFiles.init();
+ }
long size = this.tarFiles.size();
this.stats.init(size);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
index e37e764f3a..3dafc038e6 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
@@ -45,6 +45,7 @@ import org.apache.jackrabbit.oak.segment.WriterCacheManager;
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.proc.Proc.Backend;
import org.apache.jackrabbit.oak.segment.spi.persistence.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
import org.apache.jackrabbit.oak.segment.spi.monitor.*;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
@@ -149,6 +150,9 @@ public class FileStoreBuilder {
private boolean eagerSegmentCaching;
+ @Nullable
+ private TarFiles existingTarFiles;
+
private boolean built;
/**
@@ -434,6 +438,24 @@ public class FileStoreBuilder {
return this;
}
+ /**
+ * Provide an existing {@link TarFiles} instance whose TAR readers will be
+ * transferred to the new store instead of re-opening them from disk.
+ * This avoids the expensive I/O of re-reading all TAR file indices.
+ * <p>
+ * After the new store is built, the existing {@code TarFiles} instance
+ * will have its readers detached and can be safely closed without
+ * affecting the new store.
+ *
+ * @param existingTarFiles the existing TarFiles to transfer readers from
+ * @return this instance
+ */
+ @NotNull
+ public FileStoreBuilder withExistingTarFiles(@NotNull TarFiles existingTarFiles) {
+ this.existingTarFiles = requireNonNull(existingTarFiles);
+ return this;
+ }
+
/**
* Sets the threshold under which binaries are inlined in data segments.
* @param binariesInlineThreshold the threshold
@@ -622,6 +644,11 @@ public class FileStoreBuilder {
return eagerSegmentCaching;
}
+ @Nullable
+ TarFiles getExistingTarFiles() {
+ return existingTarFiles;
+ }
+
int getBinariesInlineThreshold() {
return binariesInlineThreshold;
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index ed9f1fd770..ea653fb7ca 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -406,6 +406,56 @@ public class TarFiles implements Closeable {
}
}
+ /**
+ * Initializes this instance by transferring the existing TAR readers from
+ * the given {@code TarFiles} instance, instead of opening new ones from
+ * disk. This avoids the expensive I/O of re-reading all TAR file indices.
+ * <p>
+ * The existing instance's readers are atomically detached — after this call,
+ * the existing instance will have no readers and can be safely closed
+ * without affecting the transferred readers.
+ * <p>
+ * If this instance is not read-only, a new {@link TarWriter} is created.
+ *
+ * @param existing the {@code TarFiles} instance to transfer readers from
+ * @throws IOException if an error occurs while creating the TAR writer
+ */
+ public void initFrom(TarFiles existing) throws IOException {
+ Validate.checkState(!initialised, "TarFiles already initialised");
+
+ Node existingReaders;
+
+ // Atomically detach readers from the existing TarFiles
+ existing.lock.writeLock().lock();
+ try {
+ existingReaders = existing.readers;
+ existing.readers = null;
+ } finally {
+ existing.lock.writeLock().unlock();
+ }
+
+ // Adopt the transferred readers
+ this.readers = existingReaders;
+ for (TarReader reader : iterable(readers)) {
+ segmentCount.inc(getSegmentCount(reader));
+ readerCount.inc();
+ }
+
+ // Create writer if not read-only
+ if (!readOnly) {
+ Map<Integer, Map<Character, String>> map = collectFiles(archiveManager);
+ Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
+ Arrays.sort(indices);
+ int writeNumber = 0;
+ if (indices.length > 0) {
+ writeNumber = indices[indices.length - 1] + 1;
+ }
+ writer = new TarWriter(archiveManager, writeNumber, segmentCount);
+ }
+
+ initialised = true;
+ }
+
public void init() throws IOException {
Map<Integer, Map<Character, String>> map = collectFiles(archiveManager);
Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java
new file mode 100644
index 0000000000..4671a1cde4
--- /dev/null
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreInitFromTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
+import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
+import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+/**
+ * Tests creating a writable {@link FileStore} from an existing
+ * {@link ReadOnlyFileStore}, reusing the already-opened TAR readers.
+ */
+public class FileStoreInitFromTest {
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder(new File("target"));
+
+ @Test
+ public void testUpgradeReadOnlyToWritable() throws Exception {
+ File directory = folder.getRoot();
+
+ // Step 1: Create a writable FileStore and write some nodes and properties
+ try (FileStore original = fileStoreBuilder(directory)
+ .withMaxFileSize(1)
+ .withMemoryMapping(false)
+ .build()) {
+ SegmentNodeStore nodeStore = SegmentNodeStoreBuilders.builder(original).build();
+ NodeBuilder builder = nodeStore.getRoot().builder();
+ builder.child("content").setProperty("title", "Hello");
+ builder.child("content").child("page1").setProperty("text", "First page");
+ builder.child("content").child("page2").setProperty("text", "Second page");
+ nodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ original.flush();
+ }
+
+ // Step 2: Open as read-only and verify the content is there
+ ReadOnlyFileStore readOnly = fileStoreBuilder(directory)
+ .withMemoryMapping(false)
+ .buildReadOnly();
+
+ TarFiles readOnlyTarFiles = readOnly.getTarFiles();
+ int readerCount = readOnlyTarFiles.readerCount();
+ assertTrue("Expected at least one TAR reader", readerCount > 0);
+
+ NodeState roRoot = readOnly.getHead().getChildNode("root");
+ assertEquals("Hello", roRoot.getChildNode("content").getString("title"));
+
+ // Step 3: Create a new writable FileStore, transferring readers from read-only
+ FileStore writable = fileStoreBuilder(directory)
+ .withMaxFileSize(1)
+ .withMemoryMapping(false)
+ .withExistingTarFiles(readOnlyTarFiles)
+ .build();
+
+ // Verify readers were transferred, not re-opened
+ assertEquals(readerCount, writable.readerCount());
+ assertEquals(0, readOnlyTarFiles.readerCount());
+
+ // Close the read-only store — transferred readers must remain unaffected
+ readOnly.close();
+
+ // Step 4: Verify old content is accessible from the new writable store
+ SegmentNodeStore rwNodeStore = SegmentNodeStoreBuilders.builder(writable).build();
+ NodeState rwRoot = rwNodeStore.getRoot();
+ assertNotNull(rwRoot.getChildNode("content"));
+ assertEquals("Hello", rwRoot.getChildNode("content").getString("title"));
+ assertEquals("First page", rwRoot.getChildNode("content").getChildNode("page1").getString("text"));
+ assertEquals("Second page", rwRoot.getChildNode("content").getChildNode("page2").getString("text"));
+
+ // Step 5: Write new content
+ NodeBuilder rwBuilder = rwNodeStore.getRoot().builder();
+ rwBuilder.child("content").child("page3").setProperty("text", "Third page");
+ rwBuilder.child("config").setProperty("version", 2);
+ rwNodeStore.merge(rwBuilder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ writable.flush();
+
+ // Step 6: Verify both old and new content are present
+ NodeState finalRoot = rwNodeStore.getRoot();
+ assertEquals("Hello", finalRoot.getChildNode("content").getString("title"));
+ assertEquals("First page", finalRoot.getChildNode("content").getChildNode("page1").getString("text"));
+ assertEquals("Second page", finalRoot.getChildNode("content").getChildNode("page2").getString("text"));
+ assertEquals("Third page", finalRoot.getChildNode("content").getChildNode("page3").getString("text"));
+ assertNotNull(finalRoot.getChildNode("config").getProperty("version"));
+ assertEquals(2, (long) finalRoot.getChildNode("config").getProperty("version").getValue(org.apache.jackrabbit.oak.api.Type.LONG));
+
+ writable.close();
+ }
+}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
index 80789efdf1..6353860a9b 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
@@ -504,4 +504,80 @@ public class TarFilesTest {
assertEquals(TOTAL_ENTRIES / WRITER_ENTRY_LIMIT, tarFiles.readerCount());
}
+
+ @Test
+ public void testInitFromTransfersReaders() throws Exception {
+ // Write segments to create TAR files, then close
+ UUID id1 = randomUUID();
+ UUID id2 = randomUUID();
+ byte[] data1 = randomData();
+ byte[] data2 = randomData();
+
+ writeSegment(id1, data1);
+ writeSegment(id2, data2);
+ tarFiles.close();
+ tarFiles = null;
+
+ // Open read-only TarFiles
+ TarFiles readOnlyTarFiles = TarFiles.builder()
+ .withDirectory(folder.getRoot())
+ .withTarRecovery((id, data, recovery) -> {})
+ .withIOMonitor(new IOMonitorAdapter())
+ .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+ .withReadOnly()
+ .build();
+
+ int readerCount = readOnlyTarFiles.readerCount();
+ int segmentCount = readOnlyTarFiles.segmentCount();
+ assertTrue("Expected at least one reader", readerCount > 0);
+ assertTrue("Expected at least two segments", segmentCount >= 2);
+
+ // Create new writable TarFiles, transferring readers from read-only
+ TarFiles writableTarFiles = TarFiles.builder()
+ .withDirectory(folder.getRoot())
+ .withTarRecovery((id, data, recovery) -> {})
+ .withIOMonitor(new IOMonitorAdapter())
+ .withFileStoreMonitor(new FileStoreMonitorAdapter())
+ .withMaxFileSize(MAX_FILE_SIZE)
+ .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+ .withInitialisedReadersAndWriters(false)
+ .build();
+ writableTarFiles.initFrom(readOnlyTarFiles);
+
+ // Verify readers were transferred
+ assertEquals(readerCount, writableTarFiles.readerCount());
+ assertEquals(segmentCount, writableTarFiles.segmentCount());
+
+ // Verify read-only instance has no readers left
+ assertEquals(0, readOnlyTarFiles.readerCount());
+
+ // Verify existing segments are readable from the new instance
+ assertTrue(writableTarFiles.containsSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits()));
+ assertTrue(writableTarFiles.containsSegment(id2.getMostSignificantBits(), id2.getLeastSignificantBits()));
+
+ Buffer buf1 = writableTarFiles.readSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits());
+ byte[] read1 = new byte[buf1.remaining()];
+ buf1.get(read1);
+ assertArrayEquals(data1, read1);
+
+ Buffer buf2 = writableTarFiles.readSegment(id2.getMostSignificantBits(), id2.getLeastSignificantBits());
+ byte[] read2 = new byte[buf2.remaining()];
+ buf2.get(read2);
+ assertArrayEquals(data2, read2);
+
+ // Verify writing new segments works
+ UUID id3 = randomUUID();
+ byte[] data3 = randomData();
+ writableTarFiles.writeSegment(id3, data3, 0, data3.length, newGCGeneration(1, 1, false), emptySet(), emptySet());
+ assertTrue(writableTarFiles.containsSegment(id3.getMostSignificantBits(), id3.getLeastSignificantBits()));
+
+ // Close old read-only instance — should not affect transferred readers
+ readOnlyTarFiles.close();
+
+ // Verify segments are still readable after closing the old instance
+ assertTrue(writableTarFiles.containsSegment(id1.getMostSignificantBits(), id1.getLeastSignificantBits()));
+ assertTrue(writableTarFiles.containsSegment(id3.getMostSignificantBits(), id3.getLeastSignificantBits()));
+
+ writableTarFiles.close();
+ }
}