You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by mi...@apache.org on 2024/02/06 14:03:48 UTC
(jackrabbit-oak) branch issue/OAK-10636 created (now 82021c78d6)
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a change to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
at 82021c78d6 OAK-10636 remove logger
This branch includes the following new commits:
new a34a351fdb OAK-10636 test and fist iteration for the solution
new b23cb42f48 OAK-10636 modify TarFiles builder
new 3d4bd12e81 OAK-10636 ignore .DS_Store
new 8e28d54bcf OAK-10636 ignore move segment map outside of the loop
new 2b841d3e75 OAK-10636 avoid code duplication
new 8e892b0d20 OAK-10636 check if TarFiles is initialised
new 82021c78d6 OAK-10636 remove logger
The 7 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(jackrabbit-oak) 02/07: OAK-10636 modify TarFiles builder
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit b23cb42f484dfd41596728bbd178788125e3c764
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Mon Feb 5 16:44:13 2024 +0100
OAK-10636 modify TarFiles builder
---
.../apache/jackrabbit/oak/segment/file/FileStore.java | 4 +++-
.../jackrabbit/oak/segment/file/ReadOnlyFileStore.java | 3 ++-
.../jackrabbit/oak/segment/file/tar/TarFiles.java | 11 +++++++++++
.../jackrabbit/oak/segment/IdMappingBlobStore.java | 18 +++++++++++++++++-
.../jackrabbit/oak/segment/LongIdMappingBlobStore.java | 17 +++++++++++++++++
.../jackrabbit/oak/segment/file/FileStoreTest.java | 5 ++++-
6 files changed, 54 insertions(+), 4 deletions(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index 641b9a9b83..c62304189f 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -169,10 +169,12 @@ public class FileStore extends AbstractFileStore {
.withMaxFileSize(builder.getMaxFileSize() * MB)
.withPersistence(builder.getPersistence())
.withReaderCountStats(readerCountStats)
- .withSegmentCountStats(segmentCountStats);
+ .withSegmentCountStats(segmentCountStats)
+ .withInitialisedReaders(false);
this.tarFiles = tarFilesBuilder.build();
this.tarFiles.init();
+
long size = this.tarFiles.size();
this.stats.init(size);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
index bf25f32b74..b90181637c 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
@@ -77,8 +77,9 @@ public class ReadOnlyFileStore extends AbstractFileStore {
.withMemoryMapping(memoryMapping)
.withReadOnly()
.withPersistence(builder.getPersistence())
+ .withInitialisedReaders(false)
.build();
-
+ tarFiles.init();
writer = defaultSegmentWriterBuilder("read-only").withoutCache().build(this);
gcRetainedGenerations = builder.getGcOptions().getRetainedGenerations();
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index 9a0b77cf6f..31180e4653 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -136,6 +136,8 @@ public class TarFiles implements Closeable {
private CounterStats segmentCountStats = NoopStats.INSTANCE;
+ private boolean initialisedReaders = true;
+
private Builder() {
// Prevent external instantiation.
}
@@ -201,6 +203,11 @@ public class TarFiles implements Closeable {
return this;
}
+ public Builder withInitialisedReaders(boolean initialisedReaders) {
+ this.initialisedReaders = initialisedReaders;
+ return this;
+ }
+
public TarFiles build() throws IOException {
checkState(directory != null, "Directory not specified");
checkState(tarRecovery != null, "TAR recovery strategy not specified");
@@ -384,6 +391,10 @@ public class TarFiles implements Closeable {
segmentCount = builder.segmentCountStats;
readOnly = builder.readOnly;
tarRecovery = builder.tarRecovery;
+
+ if (builder.initialisedReaders) {
+ init();
+ }
}
public void init() throws IOException {
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
index 249a37e9af..032b24d0de 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
@@ -1,6 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.jackrabbit.oak.segment;
-import org.apache.jackrabbit.guava.common.base.Strings;
import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
index 1ca7e2c0af..5bba7ebad9 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
@@ -1,3 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.jackrabbit.oak.segment;
import org.apache.jackrabbit.guava.common.base.Strings;
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
index 442ba0d3d9..903b795c9c 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
@@ -166,12 +166,15 @@ public class FileStoreTest {
builder.setProperty("foo", "bar");
Blob blob = builder.createBlob(new ZeroStream(100));
- builder.setProperty("binaryProperty", blob);
+
segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
fileStore.flush();
// create another segment
+
+ builder.setProperty("binaryProperty", blob);
+
builder.setProperty("foo1", "bar1");
segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
fileStore.flush();
(jackrabbit-oak) 05/07: OAK-10636 avoid code duplication
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 2b841d3e7517726d91d94fd7be83284d6c884f35
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Tue Feb 6 11:03:57 2024 +0100
OAK-10636 avoid code duplication
---
.../apache/jackrabbit/oak/segment/SegmentBlob.java | 47 ++++++++++------------
.../oak/segment/file/AbstractFileStore.java | 2 +-
.../oak/segment/file/tar/EntryRecovery.java | 3 ++
.../jackrabbit/oak/segment/file/tar/TarReader.java | 5 +++
4 files changed, 31 insertions(+), 26 deletions(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index e28892c6f1..05cca5caa3 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -27,14 +27,15 @@ import static org.apache.jackrabbit.oak.segment.SegmentStream.BLOCK_SIZE;
import java.io.InputStream;
import java.util.List;
+import java.util.Map;
import java.util.Set;
+import java.util.function.Function;
import org.apache.jackrabbit.oak.api.Blob;
import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
-import org.apache.jackrabbit.oak.segment.file.tar.EntryRecovery;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -165,31 +166,27 @@ public class SegmentBlob extends Record implements Blob {
}
@Nullable
- public static String readBlobId(@NotNull Segment segment, int recordNumber) {
+ private static String readBlobId(@NotNull Segment segment, int recordNumber, Function<Segment, String> readLongBlobIdFunction) {
byte head = segment.readByte(recordNumber);
if ((head & 0xf0) == 0xe0) {
// 1110 xxxx: external value, small blob ID
return readShortBlobId(segment, recordNumber, head);
} else if ((head & 0xf8) == 0xf0) {
// 1111 0xxx: external value, long blob ID
- return readLongBlobId(segment, recordNumber);
+ return readLongBlobIdFunction.apply(segment);
} else {
return null;
}
}
@Nullable
- public static String readBlobId(@NotNull Segment segment, int recordNumber, EntryRecovery recovery) {
- byte head = segment.readByte(recordNumber);
- if ((head & 0xf0) == 0xe0) {
- // 1110 xxxx: external value, small blob ID
- return readShortBlobId(segment, recordNumber, head);
- } else if ((head & 0xf8) == 0xf0) {
- // 1111 0xxx: external value, long blob ID
- return readLongBlobId(segment, recordNumber, recovery);
- } else {
- return null;
- }
+ public static String readBlobId(@NotNull Segment segment, int recordNumber) {
+ return readBlobId(segment, recordNumber, s -> readLongBlobId(s, recordNumber));
+ }
+
+ @Nullable
+ public static String readBlobId(@NotNull Segment segment, int recordNumber, Map<SegmentId, Segment> recoveredSegments) {
+ return readBlobId(segment, recordNumber, s -> readLongBlobId(s, recordNumber, recoveredSegments));
}
//------------------------------------------------------------< Object >--
@@ -244,22 +241,22 @@ public class SegmentBlob extends Record implements Blob {
return segment.readBytes(recordNumber, 2, length).decode(UTF_8).toString();
}
- private static String readLongBlobId(Segment segment, int recordNumber) {
+ private static String readLongBlobId(Segment segment, int recordNumber, Function<RecordId, Segment> getSegmentFunction) {
RecordId blobId = segment.readRecordId(recordNumber, 1);
+ Segment blobIdSegment = getSegmentFunction.apply(blobId);
- return blobId.getSegment().readString(blobId.getRecordNumber());
+ return blobIdSegment.readString(blobId.getRecordNumber());
}
- private static String readLongBlobId(Segment segment, int recordNumber, EntryRecovery recovery) {
- RecordId blobIdRecord = segment.readRecordId(recordNumber, 1);
-
- Segment blobIdSegment = recovery.getSegment(blobIdRecord.getSegmentId());
+ private static String readLongBlobId(Segment segment, int recordNumber) {
+ return readLongBlobId(segment, recordNumber, RecordId::getSegment);
+ }
- if (blobIdSegment != null) {
- return blobIdSegment.readString(blobIdRecord.getRecordNumber());
- } else {
- return blobIdRecord.getSegment().readString(blobIdRecord.getRecordNumber());
- }
+ private static String readLongBlobId(Segment segment, int recordNumber, Map<SegmentId, Segment> recoveredSegments) {
+ return readLongBlobId(segment, recordNumber, recordId -> {
+ Segment blobIdSegment = recoveredSegments.get(recordId.getSegmentId());
+ return blobIdSegment != null ? blobIdSegment : recordId.getSegment();
+ });
}
private List<RecordId> getBulkRecordIds() {
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
index 0e5b0fe12f..9f5e168653 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
@@ -252,7 +252,7 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable {
final UUID id = segment.getSegmentId().asUUID();
segment.forEachRecord((number, type, offset) -> {
if (type == RecordType.BLOB_ID) {
- w.recoverBinaryReference(generation, id, SegmentBlob.readBlobId(segment, number, w));
+ w.recoverBinaryReference(generation, id, SegmentBlob.readBlobId(segment, number, w.getRecoveredSegments()));
}
});
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
index dc928c0128..f32342581c 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
@@ -21,6 +21,7 @@ import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
import java.io.IOException;
+import java.util.Map;
import java.util.UUID;
public interface EntryRecovery {
@@ -34,4 +35,6 @@ public interface EntryRecovery {
Segment getSegment(SegmentId id);
void addSegment(Segment segment);
+
+ Map<SegmentId, Segment> getRecoveredSegments();
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index 3bf20e0be2..3a812f83f8 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -208,6 +208,11 @@ public class TarReader implements Closeable {
public void addSegment(Segment segment) {
segmentMap.put(segment.getSegmentId(), segment);
}
+
+ @Override
+ public Map<SegmentId, Segment> getRecoveredSegments() {
+ return segmentMap;
+ }
});
} catch (IOException e) {
throw new IOException(String.format("Unable to recover entry %s for file %s", entry.getKey(), file), e);
(jackrabbit-oak) 06/07: OAK-10636 check if TarFiles is initialised
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 8e892b0d2008c33b837da6f1b17a42cb3a99f9c8
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Tue Feb 6 11:51:33 2024 +0100
OAK-10636 check if TarFiles is initialised
---
.../jackrabbit/oak/segment/file/FileStore.java | 2 +-
.../oak/segment/file/ReadOnlyFileStore.java | 2 +-
.../jackrabbit/oak/segment/file/tar/TarFiles.java | 39 +++++++++++----
.../oak/segment/file/tar/TarFilesTest.java | 55 ++++++++++++++++++++--
4 files changed, 81 insertions(+), 17 deletions(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index c62304189f..6381e1129c 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -170,7 +170,7 @@ public class FileStore extends AbstractFileStore {
.withPersistence(builder.getPersistence())
.withReaderCountStats(readerCountStats)
.withSegmentCountStats(segmentCountStats)
- .withInitialisedReaders(false);
+ .withInitialisedReadersAndWriters(false);
this.tarFiles = tarFilesBuilder.build();
this.tarFiles.init();
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
index b90181637c..97b2dc7ef7 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
@@ -77,7 +77,7 @@ public class ReadOnlyFileStore extends AbstractFileStore {
.withMemoryMapping(memoryMapping)
.withReadOnly()
.withPersistence(builder.getPersistence())
- .withInitialisedReaders(false)
+ .withInitialisedReadersAndWriters(false)
.build();
tarFiles.init();
writer = defaultSegmentWriterBuilder("read-only").withoutCache().build(this);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index 31180e4653..fbd48e15fc 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -49,6 +49,7 @@ import java.util.regex.Pattern;
import org.apache.jackrabbit.guava.common.base.Predicate;
import org.apache.jackrabbit.guava.common.collect.Iterables;
+import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.file.FileReaper;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitor;
@@ -136,7 +137,7 @@ public class TarFiles implements Closeable {
private CounterStats segmentCountStats = NoopStats.INSTANCE;
- private boolean initialisedReaders = true;
+ private boolean initialiseReadersAndWriters = true;
private Builder() {
// Prevent external instantiation.
@@ -203,8 +204,8 @@ public class TarFiles implements Closeable {
return this;
}
- public Builder withInitialisedReaders(boolean initialisedReaders) {
- this.initialisedReaders = initialisedReaders;
+ public Builder withInitialisedReadersAndWriters(boolean initialiseReadersAndWriters) {
+ this.initialiseReadersAndWriters = initialiseReadersAndWriters;
return this;
}
@@ -380,6 +381,11 @@ public class TarFiles implements Closeable {
private final TarRecovery tarRecovery;
+ /**
+ * If {@code true}, the readers and writers are initialised.
+ */
+ private boolean initialised;
+
private static int getSegmentCount(TarReader reader) {
return reader.getEntries().length;
}
@@ -392,7 +398,7 @@ public class TarFiles implements Closeable {
readOnly = builder.readOnly;
tarRecovery = builder.tarRecovery;
- if (builder.initialisedReaders) {
+ if (builder.initialiseReadersAndWriters) {
init();
}
}
@@ -418,14 +424,21 @@ public class TarFiles implements Closeable {
readers = new Node(r, readers);
readerCount.inc();
}
- if (readOnly) {
- return;
+ if (!readOnly) {
+ int writeNumber = 0;
+ if (indices.length > 0) {
+ writeNumber = indices[indices.length - 1] + 1;
+ }
+ writer = new TarWriter(archiveManager, writeNumber, segmentCount);
}
- int writeNumber = 0;
- if (indices.length > 0) {
- writeNumber = indices[indices.length - 1] + 1;
+
+ initialised = true;
+ }
+
+ private void checkInitialised() {
+ if (!initialised) {
+ throw new IllegalRepositoryStateException("TarFiles not initialised");
}
- writer = new TarWriter(archiveManager, writeNumber, segmentCount);
}
@Override
@@ -549,6 +562,7 @@ public class TarFiles implements Closeable {
}
public void flush() throws IOException {
+ checkInitialised();
lock.readLock().lock();
try {
writer.flush();
@@ -611,6 +625,7 @@ public class TarFiles implements Closeable {
}
public void writeSegment(UUID id, byte[] buffer, int offset, int length, GCGeneration generation, Set<UUID> references, Set<String> binaryReferences) throws IOException {
+ checkInitialised();
lock.writeLock().lock();
try {
long size = writer.writeEntry(
@@ -662,6 +677,7 @@ public class TarFiles implements Closeable {
}
void newWriter() throws IOException {
+ checkInitialised();
lock.writeLock().lock();
try {
internalNewWriter();
@@ -671,6 +687,7 @@ public class TarFiles implements Closeable {
}
public CleanupResult cleanup(CleanupContext context) throws IOException {
+ checkInitialised();
CleanupResult result = new CleanupResult();
result.removableFiles = new ArrayList<>();
result.reclaimedSegmentIds = new HashSet<>();
@@ -822,6 +839,7 @@ public class TarFiles implements Closeable {
}
public void collectBlobReferences(Consumer<String> collector, Predicate<GCGeneration> reclaim) throws IOException {
+ checkInitialised();
Node head;
lock.writeLock().lock();
try {
@@ -908,6 +926,7 @@ public class TarFiles implements Closeable {
}
public FileReaper createFileReaper() {
+ checkInitialised();
return new FileReaper(archiveManager);
}
}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
index 39deea6ea2..37693d6ede 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
@@ -24,11 +24,7 @@ import static java.util.Collections.singleton;
import static java.util.Collections.singletonList;
import static java.util.UUID.randomUUID;
import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
@@ -40,6 +36,7 @@ import java.util.Random;
import java.util.Set;
import java.util.UUID;
+import org.apache.jackrabbit.oak.api.IllegalRepositoryStateException;
import org.apache.jackrabbit.oak.commons.Buffer;
import org.apache.jackrabbit.oak.segment.file.tar.TarFiles.CleanupResult;
import org.apache.jackrabbit.oak.segment.spi.monitor.FileStoreMonitorAdapter;
@@ -397,4 +394,52 @@ public class TarFilesTest {
assertTrue(result.getReclaimedSegmentIds().isEmpty());
assertEquals(0, result.getReclaimedSize());
}
+
+ @Test
+ public void testUninitialisedTarFiles() throws IOException {
+ tarFiles = TarFiles.builder()
+ .withDirectory(folder.getRoot())
+ .withTarRecovery((id, data, recovery) -> {
+ // Intentionally left blank
+ })
+ .withIOMonitor(new IOMonitorAdapter())
+ .withFileStoreMonitor(new FileStoreMonitorAdapter())
+ .withMaxFileSize(MAX_FILE_SIZE)
+ .withRemoteStoreMonitor(new RemoteStoreMonitorAdapter())
+ .withInitialisedReadersAndWriters(false)
+ .build();
+
+ assertEquals(0, tarFiles.readerCount());
+ assertEquals(0, tarFiles.segmentCount());
+ assertEquals(0, tarFiles.size());
+ assertFalse(tarFiles.containsSegment(0l, 0l));
+ assertNull(tarFiles.readSegment(0l, 0l));
+
+ tarFiles.toString();
+ assertThrows(IllegalRepositoryStateException.class, () -> tarFiles.flush());
+ assertThrows(IllegalRepositoryStateException.class, () -> writeSegment(randomUUID()));
+ assertThrows(IllegalRepositoryStateException.class, () -> tarFiles.cleanup(new CleanupContext() {
+ @Override
+ public Collection<UUID> initialReferences() {
+ return emptySet();
+ }
+
+ @Override
+ public boolean shouldReclaim(UUID id, GCGeneration generation, boolean referenced) {
+ return false;
+ }
+
+ @Override
+ public boolean shouldFollow(UUID from, UUID to) {
+ return false;
+ }
+ }));
+
+ assertThrows(IllegalRepositoryStateException.class, () -> tarFiles.collectBlobReferences(r -> {}, g -> false));
+ assertFalse(tarFiles.getSegmentIds().iterator().hasNext());
+ assertTrue(tarFiles.getIndices().isEmpty());
+ assertTrue(tarFiles.getGraph("").isEmpty());
+
+ assertThrows(IllegalRepositoryStateException.class, () -> tarFiles.createFileReaper());
+ }
}
(jackrabbit-oak) 01/07: OAK-10636 test and fist iteration for the solution
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit a34a351fdb21923cec5da8626ed1e68973c427ee
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Fri Feb 2 09:22:57 2024 +0100
OAK-10636 test and fist iteration for the solution
---
.../apache/jackrabbit/oak/segment/SegmentBlob.java | 26 ++++++
.../oak/segment/file/AbstractFileStore.java | 6 +-
.../jackrabbit/oak/segment/file/FileStore.java | 4 +-
.../oak/segment/file/tar/EntryRecovery.java | 6 ++
.../jackrabbit/oak/segment/file/tar/TarFiles.java | 16 +++-
.../jackrabbit/oak/segment/file/tar/TarReader.java | 12 +++
.../jackrabbit/oak/segment/IdMappingBlobStore.java | 72 +++++++++++++++
.../oak/segment/LongIdMappingBlobStore.java | 14 +++
.../jackrabbit/oak/segment/file/FileStoreTest.java | 101 ++++++++++++++++++---
9 files changed, 238 insertions(+), 19 deletions(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
index 9ab2ffea8d..e28892c6f1 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentBlob.java
@@ -34,6 +34,7 @@ import org.apache.jackrabbit.oak.commons.properties.SystemPropertySupplier;
import org.apache.jackrabbit.oak.plugins.blob.BlobStoreBlob;
import org.apache.jackrabbit.oak.plugins.blob.datastore.InMemoryDataRecord;
import org.apache.jackrabbit.oak.plugins.memory.AbstractBlob;
+import org.apache.jackrabbit.oak.segment.file.tar.EntryRecovery;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
@@ -177,6 +178,19 @@ public class SegmentBlob extends Record implements Blob {
}
}
+ @Nullable
+ public static String readBlobId(@NotNull Segment segment, int recordNumber, EntryRecovery recovery) {
+ byte head = segment.readByte(recordNumber);
+ if ((head & 0xf0) == 0xe0) {
+ // 1110 xxxx: external value, small blob ID
+ return readShortBlobId(segment, recordNumber, head);
+ } else if ((head & 0xf8) == 0xf0) {
+ // 1111 0xxx: external value, long blob ID
+ return readLongBlobId(segment, recordNumber, recovery);
+ } else {
+ return null;
+ }
+ }
//------------------------------------------------------------< Object >--
@Override
@@ -236,6 +250,18 @@ public class SegmentBlob extends Record implements Blob {
return blobId.getSegment().readString(blobId.getRecordNumber());
}
+ private static String readLongBlobId(Segment segment, int recordNumber, EntryRecovery recovery) {
+ RecordId blobIdRecord = segment.readRecordId(recordNumber, 1);
+
+ Segment blobIdSegment = recovery.getSegment(blobIdRecord.getSegmentId());
+
+ if (blobIdSegment != null) {
+ return blobIdSegment.readString(blobIdRecord.getRecordNumber());
+ } else {
+ return blobIdRecord.getSegment().readString(blobIdRecord.getRecordNumber());
+ }
+ }
+
private List<RecordId> getBulkRecordIds() {
Segment segment = getSegment();
byte head = segment.readByte(getRecordNumber());
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
index 61cea135d1..0e5b0fe12f 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
@@ -232,7 +232,9 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable {
: GCGeneration.NULL;
w.recoverEntry(msb, lsb, data, 0, data.length, generation);
if (SegmentId.isDataSegmentId(lsb)) {
- Segment segment = new Segment(tracker, segmentReader, tracker.newSegmentId(msb, lsb), buffer);
+ SegmentId segmentId = tracker.newSegmentId(msb, lsb);
+ Segment segment = new Segment(tracker, segmentReader, segmentId, buffer);
+ w.addSegment(segment);
populateTarGraph(segment, w);
populateTarBinaryReferences(segment, w);
}
@@ -250,7 +252,7 @@ public abstract class AbstractFileStore implements SegmentStore, Closeable {
final UUID id = segment.getSegmentId().asUUID();
segment.forEachRecord((number, type, offset) -> {
if (type == RecordType.BLOB_ID) {
- w.recoverBinaryReference(generation, id, SegmentBlob.readBlobId(segment, number));
+ w.recoverBinaryReference(generation, id, SegmentBlob.readBlobId(segment, number, w));
}
});
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
index 57fb74379b..641b9a9b83 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
@@ -155,6 +155,8 @@ public class FileStore extends AbstractFileStore {
this.stats = new FileStoreStats(statsProvider, this, 0);
+ this.snfeListener = builder.getSnfeListener();
+
CounterStats readerCountStats = statsProvider.getCounterStats(TAR_READER_COUNT, DEFAULT);
CounterStats segmentCountStats = statsProvider.getCounterStats(SEGMENT_COUNT, DEFAULT);
TarFiles.Builder tarFilesBuilder = TarFiles.builder()
@@ -170,6 +172,7 @@ public class FileStore extends AbstractFileStore {
.withSegmentCountStats(segmentCountStats);
this.tarFiles = tarFilesBuilder.build();
+ this.tarFiles.init();
long size = this.tarFiles.size();
this.stats.init(size);
@@ -202,7 +205,6 @@ public class FileStore extends AbstractFileStore {
.build(this)
);
- this.snfeListener = builder.getSnfeListener();
this.eagerSegmentCaching = builder.getEagerSegmentCaching();
TimerStats flushTimer = statsProvider.getTimer("oak.segment.flush", METRICS_ONLY);
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
index 34bef73a9c..dc928c0128 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/EntryRecovery.java
@@ -17,6 +17,9 @@
package org.apache.jackrabbit.oak.segment.file.tar;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+
import java.io.IOException;
import java.util.UUID;
@@ -28,4 +31,7 @@ public interface EntryRecovery {
void recoverBinaryReference(GCGeneration generation, UUID segmentId, String reference);
+ Segment getSegment(SegmentId id);
+
+ void addSegment(Segment segment);
}
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
index b3cb9fd06e..9a0b77cf6f 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
@@ -369,6 +369,10 @@ public class TarFiles implements Closeable {
*/
private final CounterStats segmentCount;
+ private final boolean readOnly;
+
+ private final TarRecovery tarRecovery;
+
private static int getSegmentCount(TarReader reader) {
return reader.getEntries().length;
}
@@ -378,7 +382,11 @@ public class TarFiles implements Closeable {
archiveManager = builder.buildArchiveManager();
readerCount = builder.readerCountStats;
segmentCount = builder.segmentCountStats;
+ readOnly = builder.readOnly;
+ tarRecovery = builder.tarRecovery;
+ }
+ public void init() throws IOException {
Map<Integer, Map<Character, String>> map = collectFiles(archiveManager);
Integer[] indices = map.keySet().toArray(new Integer[map.size()]);
Arrays.sort(indices);
@@ -390,16 +398,16 @@ public class TarFiles implements Closeable {
for (Integer index : indices) {
TarReader r;
- if (builder.readOnly) {
- r = TarReader.openRO(map.get(index), builder.tarRecovery, archiveManager);
+ if (readOnly) {
+ r = TarReader.openRO(map.get(index), tarRecovery, archiveManager);
} else {
- r = TarReader.open(map.get(index), builder.tarRecovery, archiveManager);
+ r = TarReader.open(map.get(index), tarRecovery, archiveManager);
}
segmentCount.inc(getSegmentCount(r));
readers = new Node(r, readers);
readerCount.inc();
}
- if (builder.readOnly) {
+ if (readOnly) {
return;
}
int writeNumber = 0;
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index 47f080370d..e648fe2a6b 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -30,6 +30,7 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -44,6 +45,8 @@ import java.util.stream.Collectors;
import org.apache.jackrabbit.guava.common.base.Predicate;
import org.apache.jackrabbit.oak.commons.Buffer;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndex;
import org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndexLoader;
import org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException;
@@ -182,6 +185,8 @@ public class TarReader implements Closeable {
try {
recovery.recoverEntry(entry.getKey(), entry.getValue(), new EntryRecovery() {
+ private Map<SegmentId, Segment> segmentMap = new HashMap<>(entries.size());
+
@Override
public void recoverEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation) throws IOException {
writer.writeEntry(msb, lsb, data, offset, size, generation);
@@ -197,6 +202,13 @@ public class TarReader implements Closeable {
writer.addBinaryReference(generation, segmentId, reference);
}
+ public Segment getSegment(SegmentId id) {
+ return segmentMap.get(id);
+ }
+
+ public void addSegment(Segment segment) {
+ segmentMap.put(segment.getSegmentId(), segment);
+ }
});
} catch (IOException e) {
throw new IOException(String.format("Unable to recover entry %s for file %s", entry.getKey(), file), e);
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
new file mode 100644
index 0000000000..249a37e9af
--- /dev/null
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/IdMappingBlobStore.java
@@ -0,0 +1,72 @@
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.guava.common.base.Strings;
+import org.apache.jackrabbit.oak.spi.blob.BlobOptions;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
+import org.jetbrains.annotations.NotNull;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+
+public abstract class IdMappingBlobStore implements BlobStore {
+
+ private final MemoryBlobStore bs = new MemoryBlobStore();
+
+ private final Map<String, String> ids = new HashMap<>();
+
+ @Override
+ public String writeBlob(InputStream inputStream) throws IOException {
+ String in = bs.writeBlob(inputStream);
+ String out = generateId();
+ ids.put(out, in);
+ return out;
+ }
+
+ @Override
+ public String writeBlob(InputStream inputStream, BlobOptions options) throws IOException {
+ return writeBlob(inputStream);
+ }
+
+ @Override
+ public int readBlob(String s, long l, byte[] bytes, int i, int i1) throws IOException {
+ return bs.readBlob(mapId(s), l, bytes, i, i1);
+ }
+
+ @Override
+ public long getBlobLength(String s) throws IOException {
+ return bs.getBlobLength(mapId(s));
+ }
+
+ @Override
+ public InputStream getInputStream(String s) throws IOException {
+ return bs.getInputStream(mapId(s));
+ }
+
+ @Override
+ public String getBlobId(@NotNull String s) {
+ return bs.getBlobId(s);
+ }
+
+ @Override
+ public String getReference(@NotNull String s) {
+ return bs.getBlobId(mapId(s));
+ }
+
+ @Override
+ public void close() throws Exception {
+ bs.close();
+ }
+
+ private String mapId(String in) {
+ String out = ids.get(in);
+ if (out == null) {
+ throw new IllegalArgumentException("in");
+ }
+ return out;
+ }
+
+ protected abstract String generateId();
+}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
new file mode 100644
index 0000000000..1ca7e2c0af
--- /dev/null
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
@@ -0,0 +1,14 @@
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.guava.common.base.Strings;
+
+public class LongIdMappingBlobStore extends IdMappingBlobStore {
+
+ private static int next;
+
+ @Override
+ protected String generateId() {
+ return Strings.repeat("0", Segment.BLOB_ID_SMALL_LIMIT) + Integer.toString(next++);
+ }
+
+}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
index 9043799682..442ba0d3d9 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
@@ -19,20 +19,15 @@
package org.apache.jackrabbit.oak.segment.file;
-import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
-import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertThrows;
-import static org.junit.Assert.fail;
-
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-
import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.CommitFailedException;
+import org.apache.jackrabbit.oak.api.Type;
+import org.apache.jackrabbit.oak.segment.LongIdMappingBlobStore;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentNodeBuilder;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStoreBuilders;
import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarManager;
import org.apache.jackrabbit.oak.segment.file.tar.SegmentTarWriter;
import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
@@ -41,20 +36,38 @@ import org.apache.jackrabbit.oak.segment.spi.monitor.IOMonitor;
import org.apache.jackrabbit.oak.segment.spi.monitor.RemoteStoreMonitor;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveWriter;
+import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.commit.CommitInfo;
+import org.apache.jackrabbit.oak.spi.commit.EmptyHook;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.jetbrains.annotations.NotNull;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+
+import static org.apache.jackrabbit.oak.plugins.memory.EmptyNodeState.EMPTY_NODE;
+import static org.apache.jackrabbit.oak.segment.file.FileStoreBuilder.fileStoreBuilder;
+import static org.junit.Assert.*;
public class FileStoreTest {
private static final String FAILED_TO_WRITE_ON_CLOSE = "Failed to write to the archive on closing";
+ private static Logger LOG = LoggerFactory.getLogger(FileStoreTest.class);
+
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
- private File getFileStoreFolder() {
- return folder.getRoot();
+ private File getFileStoreFolder() throws IOException {
+ return folder.newFolder("segmentstore");
}
@Test
@@ -140,6 +153,70 @@ public class FileStoreTest {
assertEquals("already shut down", closeEx.getMessage());
}
+ @Test
+ public void testRecovery_FileStore_withExternalBlobStore() throws InvalidFileStoreVersionException, IOException, CommitFailedException {
+
+ File segmentStore = getFileStoreFolder();
+ BlobStore blobStore = new LongIdMappingBlobStore();
+ FileStore fileStore = createFileStore(segmentStore, blobStore);
+
+ SegmentNodeStore segmentNodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+ NodeBuilder builder = segmentNodeStore.getRoot().builder();
+ builder.setProperty("foo", "bar");
+
+ Blob blob = builder.createBlob(new ZeroStream(100));
+ builder.setProperty("binaryProperty", blob);
+
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ // create another segment
+ builder.setProperty("foo1", "bar1");
+ segmentNodeStore.merge(builder, EmptyHook.INSTANCE, CommitInfo.EMPTY);
+ fileStore.flush();
+
+ // copy all files form segmentStore to segmentStoreClone
+ // with this, tha last tar archive will not have index, graph and binary references
+ File segmentStoreClone = folder.newFolder("segmentstore-clone");
+ Files.walk(segmentStore.toPath())
+ .forEach(source -> {
+ try {
+ Path target = segmentStoreClone.toPath().resolve(segmentStore.toPath().relativize(source));
+ Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING);
+ } catch (IOException e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ });
+
+
+ fileStore.close();
+
+ // Start new FileStore, which should be able to recover successfully despite now having the index, graph and binary references
+ try {
+ fileStore = createFileStore(segmentStoreClone, blobStore);
+ } catch (Exception e){
+ fail("Should not throw exception" + e);
+ }
+
+ segmentNodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
+
+
+ builder = segmentNodeStore.getRoot().builder();
+
+ assertNotNull(builder.getProperty("foo"));
+ assertNotNull(builder.getProperty("foo1"));
+ blob = builder.getProperty("binaryProperty").getValue(Type.BINARY);
+ assertNotNull(blob);
+ }
+
+ public FileStore createFileStore(File segmentStore, BlobStore blobStore) throws IOException, InvalidFileStoreVersionException {
+ return fileStoreBuilder(segmentStore)
+ .withBinariesInlineThreshold(0)
+ .withBlobStore(blobStore)
+ .build();
+ }
+
private static int counter = 0;
private static void writeData(FileStore fileStore, int size) throws IOException {
(jackrabbit-oak) 07/07: OAK-10636 remove logger
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 82021c78d62dc031aaf7c72345eaa55885108026
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Tue Feb 6 15:03:00 2024 +0100
OAK-10636 remove logger
---
.../org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java | 2 +-
.../java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java | 4 ----
2 files changed, 1 insertion(+), 5 deletions(-)
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
index 5bba7ebad9..2b51647bc7 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/LongIdMappingBlobStore.java
@@ -25,7 +25,7 @@ public class LongIdMappingBlobStore extends IdMappingBlobStore {
@Override
protected String generateId() {
- return Strings.repeat("0", Segment.BLOB_ID_SMALL_LIMIT) + Integer.toString(next++);
+ return Strings.repeat("0", Segment.BLOB_ID_SMALL_LIMIT) + next++;
}
}
diff --git a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
index 903b795c9c..d2e6f0d355 100644
--- a/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
+++ b/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/FileStoreTest.java
@@ -44,8 +44,6 @@ import org.jetbrains.annotations.NotNull;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -61,8 +59,6 @@ import static org.junit.Assert.*;
public class FileStoreTest {
private static final String FAILED_TO_WRITE_ON_CLOSE = "Failed to write to the archive on closing";
- private static Logger LOG = LoggerFactory.getLogger(FileStoreTest.class);
-
@Rule
public TemporaryFolder folder = new TemporaryFolder(new File("target"));
(jackrabbit-oak) 04/07: OAK-10636 ignore move segment map outside of the loop
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 8e28d54bcfd02713d71a78c039d010e009d28bb4
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Tue Feb 6 08:51:16 2024 +0100
OAK-10636 ignore move segment map outside of the loop
---
.../java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
index e648fe2a6b..3bf20e0be2 100644
--- a/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
+++ b/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
@@ -181,12 +181,11 @@ public class TarReader implements Closeable {
log.info("Regenerating tar file {}", file);
try (TarWriter writer = new TarWriter(archiveManager, file)) {
+ Map<SegmentId, Segment> segmentMap = new HashMap<>(entries.size());
+
for (Entry<UUID, byte[]> entry : entries.entrySet()) {
try {
recovery.recoverEntry(entry.getKey(), entry.getValue(), new EntryRecovery() {
-
- private Map<SegmentId, Segment> segmentMap = new HashMap<>(entries.size());
-
@Override
public void recoverEntry(long msb, long lsb, byte[] data, int offset, int size, GCGeneration generation) throws IOException {
writer.writeEntry(msb, lsb, data, offset, size, generation);
(jackrabbit-oak) 03/07: OAK-10636 ignore .DS_Store
Posted by mi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
miroslav pushed a commit to branch issue/OAK-10636
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 3d4bd12e819f0ff381c6d8d90978b0ee6ffbe681
Author: Miroslav Smiljanic <mi...@apache.com>
AuthorDate: Tue Feb 6 08:46:47 2024 +0100
OAK-10636 ignore .DS_Store
---
.gitignore | 1 +
1 file changed, 1 insertion(+)
diff --git a/.gitignore b/.gitignore
index d13284858c..c700561aa7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -12,3 +12,4 @@ atlassian-ide-plugin.xml
derby.log
.java-version
oak-shaded-guava/dependency-reduced-pom.xml
+.DS_Store
\ No newline at end of file