You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2018/02/13 11:17:43 UTC
svn commit: r1824115 [1/3] - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ oak-...
Author: tomekr
Date: Tue Feb 13 11:17:42 2018
New Revision: 1824115
URL: http://svn.apache.org/viewvc?rev=1824115&view=rev
Log:
OAK-6921: Support pluggable segment storage
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarManager.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/SegmentTarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarPersistence.java
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarConstants.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarEntry.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarFiles.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriter.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/ConsistencyChecker.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tooling/RevisionHistory.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Compact.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/History.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/GcJournalTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/JournalEntryTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/JournalReaderTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ManifestCheckerTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/ManifestTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/SizeDeltaGCEstimationTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/TarRevisionsTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFileTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarFilesTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/file/tar/TarWriterTest.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/upgrade/UpgradeIT.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/explorer/SegmentTarExplorerBackend.java Tue Feb 13 11:17:42 2018
@@ -47,11 +47,13 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.JournalFile;
import org.apache.jackrabbit.oak.segment.SegmentPropertyState;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.JournalEntry;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile;
import org.apache.jackrabbit.oak.spi.state.NodeState;
class SegmentTarExplorerBackend implements ExplorerBackend {
@@ -85,7 +87,7 @@ class SegmentTarExplorerBackend implemen
@Override
public List<String> readRevisions() {
- File journal = new File(path, "journal.log");
+ JournalFile journal = new LocalJournalFile(path, "journal.log");
if (!journal.exists()) {
return newArrayList();
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentArchiveManager.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarEntry;
+import org.apache.jackrabbit.oak.segment.file.tar.binaries.BinaryReferencesIndex;
+import org.apache.jackrabbit.oak.segment.file.tar.binaries.InvalidBinaryReferencesIndexException;
+import org.apache.jackrabbit.oak.segment.file.tar.index.Index;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+
+/**
+ * SegmentArchiveManager provides a low-level access to the segment files (eg.
+ * stored in the .tar). It allows to perform a few FS-like operations (delete,
+ * rename, copy, etc.) and also opens the segment archives either for reading
+ * or reading and writing.
+ */
+public interface SegmentArchiveManager {
+
+ /**
+ * List names of the available archives.
+ *
+ * @return archive list
+ */
+ @Nonnull
+ List<String> listArchives() throws IOException;
+
+ /**
+ * Opens a given archive for reading.
+ *
+ * @param archiveName
+ * @return the archive reader or null if the archive doesn't exist
+ */
+ @Nullable
+ SegmentArchiveReader open(@Nonnull String archiveName) throws IOException;
+
+ /**
+ * Creates a new archive.
+ *
+ * @param archiveName
+ * @return the archive writer
+ */
+ @Nonnull
+ SegmentArchiveWriter create(@Nonnull String archiveName) throws IOException;
+
+ /**
+ * Deletes the archive if exists.
+ *
+ * @param archiveName
+ * @return true if the archive was removed, false otherwise
+ */
+ boolean delete(@Nonnull String archiveName);
+
+ /**
+ * Renames the archive.
+ *
+ * @param from the existing archive
+ * @param to new name
+ * @return true if the archive was renamed, false otherwise
+ */
+ boolean renameTo(@Nonnull String from, @Nonnull String to);
+
+ /**
+ * Copies the archive with all the segments.
+ *
+ * @param from the existing archive
+ * @param to new name
+ */
+ void copyFile(@Nonnull String from, @Nonnull String to) throws IOException;
+
+ /**
+ * Check if archive exists.
+ *
+ * @param archiveName archive to check
+ * @return true if archive exists, false otherwise
+ */
+ boolean exists(@Nonnull String archiveName);
+
+ /**
+ * Finds all the segments included in the archive.
+ *
+ * @param archiveName archive to recover
+ * @param entries results will be put there, in the order of presence in the
+ * archive
+ */
+ void recoverEntries(@Nonnull String archiveName, @Nonnull LinkedHashMap<UUID, byte[]> entries) throws IOException;
+
+ /**
+ * Allows to write in the new archive.
+ */
+ interface SegmentArchiveWriter {
+
+ /**
+ * Write the new segment to the archive.
+ *
+ * @param msb
+ * @param lsb
+ * @param data
+ * @param offset
+ * @param size
+ * @param generation
+ * @return the entry representing the new segment. Can be later used for the {@link #readSegment(TarEntry)} method.
+ */
+ @Nonnull
+ TarEntry writeSegment(long msb, long lsb, @Nonnull byte[] data, int offset, int size, @Nonnull GCGeneration generation) throws IOException;
+
+ /**
+ * Read the segment.
+ *
+ * @param tarEntry
+ * @return byte buffer containing the segment data or null if segment doesn't exist
+ */
+ @Nullable
+ ByteBuffer readSegment(@Nonnull TarEntry tarEntry) throws IOException;
+
+ /**
+ * Write the index data.
+ *
+ * @param data
+ */
+ void writeIndex(@Nonnull byte[] data) throws IOException;
+
+ /**
+ * Write the graph data.
+ *
+ * @param data
+ */
+ void writeGraph(@Nonnull byte[] data) throws IOException;
+
+ /**
+ * Write the binary references data.
+ *
+ * @param data
+ */
+ void writeBinaryReferences(@Nonnull byte[] data) throws IOException;
+
+ /**
+ * Get the current length of the archive.
+ *
+ * @return length of the archive, in bytes
+ */
+ long getLength();
+
+ /**
+ * Close the archive.
+ */
+ void close() throws IOException;
+
+ /**
+ * Check if the archive has been created (eg. something has been written).
+ *
+ * @return true if the archive has been created, false otherwise
+ */
+ boolean isCreated();
+
+ /**
+ * Flush all the data to the storage.
+ */
+ void flush() throws IOException;
+
+ /**
+ * Get the name of the archive.
+ *
+ * @return archive name
+ */
+ @Nonnull
+ String getName();
+ }
+
+ interface SegmentArchiveReader {
+
+ /**
+ * Read the segment.
+ *
+ * @param msb
+ * @param lsb
+ * @return byte buffer containing the segment data or null if segment doesn't exist
+ */
+ @Nullable
+ ByteBuffer readSegment(long msb, long lsb) throws IOException;
+
+ /**
+ * Returns the index.
+ *
+ * @return segment index
+ */
+ @Nonnull
+ Index getIndex();
+
+ /**
+ * Loads and returns the graph.
+ *
+ * @return the segment graph or null if the persisted graph doesn't exist.
+ */
+ @Nullable
+ Map<UUID, List<UUID>> getGraph() throws IOException;
+
+ /**
+ * Check if the persisted graph exists.
+ *
+ * @return true if the graph exists, false otherwise
+ */
+ boolean hasGraph();
+
+ /**
+ * Loads and returns the binary references.
+ *
+ * @return binary references
+ */
+ @Nonnull
+ BinaryReferencesIndex getBinaryReferences() throws IOException, InvalidBinaryReferencesIndexException;
+
+ /**
+ * Get the current length of the archive.
+ *
+ * @return length of the archive, in bytes
+ */
+ long length();
+
+ /**
+ * Get the name of the archive.
+ *
+ * @return archive name
+ */
+ @Nonnull
+ String getName();
+
+ /**
+ * Close the archive.
+ */
+ void close() throws IOException;
+
+ /**
+ * Returns the size of the entry
+ * @param size
+ * @return
+ */
+ int getEntrySize(int size);
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreFactory.java Tue Feb 13 11:17:42 2018
@@ -75,6 +75,12 @@ public class SegmentNodeStoreFactory {
public static final String CUSTOM_BLOB_STORE = "customBlobStore";
@Property(boolValue = false,
+ label = "Custom segment store",
+ description = "Boolean value indicating that a custom (non-tar) segment store is used"
+ )
+ public static final String CUSTOM_SEGMENT_STORE = "customSegmentStore";
+
+ @Property(boolValue = false,
label = "Register JCR descriptors as OSGi services",
description="Should only be done for one factory instance")
public static final String REGISTER_DESCRIPTORS = "registerDescriptors";
@@ -89,6 +95,13 @@ public class SegmentNodeStoreFactory {
)
private volatile BlobStore blobStore;
+ @Reference(
+ cardinality = ReferenceCardinality.OPTIONAL_UNARY,
+ policy = ReferencePolicy.STATIC,
+ policyOption = ReferencePolicyOption.GREEDY
+ )
+ private volatile SegmentNodeStorePersistence segmentStore;
+
@Reference
private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP;
@@ -100,6 +113,7 @@ public class SegmentNodeStoreFactory {
// In secondaryNodeStore mode customBlobStore is always enabled
boolean isSecondaryStoreMode = "secondary".equals(role);
boolean customBlobStore = Boolean.parseBoolean(property(CUSTOM_BLOB_STORE, context)) || isSecondaryStoreMode;
+ boolean customSegmentStore = Boolean.parseBoolean(property(CUSTOM_SEGMENT_STORE, context));
boolean registerRepositoryDescriptors = Boolean.parseBoolean(property(REGISTER_DESCRIPTORS, context));
log.info("activate: SegmentNodeStore '" + role + "' starting.");
@@ -108,10 +122,15 @@ public class SegmentNodeStoreFactory {
return;
}
+ if (segmentStore == null && customSegmentStore) {
+ log.info("customSegmentStore enabled. SegmentNodeStore will be initialized once the custom segment store becomes available");
+ return;
+ }
+
if (role != null) {
registrations = Closer.create();
OsgiWhiteboard whiteboard = new OsgiWhiteboard(context.getBundleContext());
- final SegmentNodeStore store = SegmentNodeStoreService.registerSegmentStore(context, blobStore,
+ final SegmentNodeStore store = SegmentNodeStoreService.registerSegmentStore(context, blobStore, segmentStore,
statisticsProvider, registrations, whiteboard, role, registerRepositoryDescriptors);
if (store != null) {
Map<String, Object> props = new HashMap<String, Object>();
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStorePersistence.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment;
+
+import org.apache.jackrabbit.oak.segment.file.tar.FileStoreMonitor;
+import org.apache.jackrabbit.oak.segment.file.tar.IOMonitor;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+public interface SegmentNodeStorePersistence {
+
+ SegmentArchiveManager createArchiveManager(boolean memoryMapping, IOMonitor ioMonitor, FileStoreMonitor fileStoreMonitor) throws IOException;
+
+ boolean segmentFilesExist();
+
+ JournalFile getJournalFile();
+
+ GCJournalFile getGCJournalFile() throws IOException;
+
+ ManifestFile getManifestFile() throws IOException;
+
+ RepositoryLock lockRepository() throws IOException;
+
+ interface JournalFile {
+
+ JournalFileReader openJournalReader() throws IOException;
+
+ JournalFileWriter openJournalWriter() throws IOException;
+
+ String getName();
+
+ boolean exists();
+ }
+
+ interface JournalFileReader extends Closeable {
+
+ String readLine() throws IOException;
+
+ }
+
+ interface JournalFileWriter extends Closeable {
+
+ void truncate() throws IOException;
+
+ void writeLine(String line) throws IOException;
+
+ }
+
+ interface GCJournalFile {
+
+ void writeLine(String line) throws IOException;
+
+ List<String> readLines() throws IOException;
+
+ }
+
+ interface ManifestFile {
+
+ boolean exists();
+
+ Properties load() throws IOException;
+
+ void save(Properties properties) throws IOException;
+
+ }
+
+ interface RepositoryLock {
+
+ void unlock() throws IOException;
+
+ }
+}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStoreService.java Tue Feb 13 11:17:42 2018
@@ -35,6 +35,7 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.COMPACTION_RETRY_COUNT;
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.COMPACTION_SIZE_DELTA_ESTIMATION;
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_BLOB_STORE;
+import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.CUSTOM_SEGMENT_STORE;
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DEFAULT_BLOB_GC_MAX_AGE;
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.DEFAULT_BLOB_SNAPSHOT_INTERVAL;
import static org.apache.jackrabbit.oak.segment.SegmentNodeStoreService.GC_PROGRESS_LOG;
@@ -316,6 +317,12 @@ public class SegmentNodeStoreService {
)
public static final String CUSTOM_BLOB_STORE = "customBlobStore";
+ @Property(boolValue = false,
+ label = "Custom segment store",
+ description = "Boolean value indicating that a custom (non-tar) segment store is used"
+ )
+ public static final String CUSTOM_SEGMENT_STORE = "customSegmentStore";
+
@Property(
label = "Backup directory",
description = "Directory (relative to current working directory) for storing repository backups. " +
@@ -331,6 +338,13 @@ public class SegmentNodeStoreService {
)
private volatile BlobStore blobStore;
+ @Reference(
+ cardinality = ReferenceCardinality.OPTIONAL_UNARY,
+ policy = ReferencePolicy.STATIC,
+ policyOption = ReferencePolicyOption.GREEDY
+ )
+ private volatile SegmentNodeStorePersistence segmentStore;
+
@Reference
private StatisticsProvider statisticsProvider = StatisticsProvider.NOOP;
@@ -374,9 +388,14 @@ public class SegmentNodeStoreService {
"store becomes available");
return;
}
+ if (segmentStore == null && configuration.hasCustomSegmentStore()) {
+ log.info("customSegmentStore enabled. SegmentNodeStore will be initialized once the custom segment " +
+ "store becomes available");
+ return;
+ }
closer = Closer.create();
OsgiWhiteboard whiteboard = new OsgiWhiteboard(context.getBundleContext());
- registerSegmentStore(context, blobStore, statisticsProvider, closer, whiteboard, null, true);
+ registerSegmentStore(context, blobStore, segmentStore, statisticsProvider, closer, whiteboard, null, true);
}
/**
@@ -388,6 +407,8 @@ public class SegmentNodeStoreService {
* @param context An instance of {@link ComponentContext}.
* @param blobStore An instance of {@link BlobStore}. It can be
* {@code null}.
+ * @param segmentStore An instance of {@link SegmentNodeStorePersistence}. It can be
+ * {@code null}.
* @param statisticsProvider An instance of {@link StatisticsProvider}.
* @param closer An instance of {@link Closer}. It will be used
* to track every registered service or
@@ -406,6 +427,7 @@ public class SegmentNodeStoreService {
static SegmentNodeStore registerSegmentStore(
@Nonnull ComponentContext context,
@Nullable BlobStore blobStore,
+ @Nullable SegmentNodeStorePersistence segmentStore,
@Nonnull StatisticsProvider statisticsProvider,
@Nonnull Closer closer,
@Nonnull Whiteboard whiteboard,
@@ -469,6 +491,11 @@ public class SegmentNodeStoreService {
builder.withBlobStore(blobStore);
}
+ if (configuration.hasCustomSegmentStore() && segmentStore != null) {
+ log.info("Initializing SegmentNodeStore with custom persistence [{}]", segmentStore);
+ builder.withCustomPersistence(segmentStore);
+ }
+
if (configuration.isStandbyInstance()) {
builder.withSnfeListener(IGNORE_SNFE);
}
@@ -934,6 +961,10 @@ class Configuration {
return toBoolean(property(CUSTOM_BLOB_STORE), false);
}
+ boolean hasCustomSegmentStore() {
+ return toBoolean(property(CUSTOM_SEGMENT_STORE), false);
+ }
+
long getBlobGcMaxAge() {
return toLong(property(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE);
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/AbstractFileStore.java Tue Feb 13 11:17:42 2018
@@ -24,7 +24,6 @@ import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.UUID;
@@ -32,7 +31,6 @@ import java.util.UUID;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
-import org.apache.commons.io.FileUtils;
import org.apache.jackrabbit.oak.api.jmx.CacheStatsMBean;
import org.apache.jackrabbit.oak.segment.CachingSegmentReader;
import org.apache.jackrabbit.oak.segment.RecordType;
@@ -45,6 +43,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.SegmentIdFactory;
import org.apache.jackrabbit.oak.segment.SegmentIdProvider;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.SegmentReader;
import org.apache.jackrabbit.oak.segment.SegmentStore;
@@ -66,8 +65,6 @@ public abstract class AbstractFileStore
private static final Logger log = LoggerFactory.getLogger(AbstractFileStore.class);
- private static final String MANIFEST_FILE_NAME = "manifest";
-
/**
* The minimum supported store version. It is possible for an implementation
* to support in a transparent and backwards-compatible way older versions
@@ -88,20 +85,15 @@ public abstract class AbstractFileStore
*/
private static final int MAX_STORE_VERSION = 2;
- static ManifestChecker newManifestChecker(File directory, boolean strictVersionCheck) {
+ static ManifestChecker newManifestChecker(SegmentNodeStorePersistence persistence, boolean strictVersionCheck) throws IOException {
return ManifestChecker.newManifestChecker(
- new File(directory, MANIFEST_FILE_NAME),
- notEmptyDirectory(directory),
+ persistence.getManifestFile(),
+ persistence.segmentFilesExist(),
strictVersionCheck ? MAX_STORE_VERSION : MIN_STORE_VERSION,
MAX_STORE_VERSION
);
}
- private static boolean notEmptyDirectory(File path) {
- Collection<File> entries = FileUtils.listFiles(path, new String[] {"tar"}, false);
- return !entries.isEmpty();
- }
-
@Nonnull
final SegmentTracker tracker;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileReaper.java Tue Feb 13 11:17:42 2018
@@ -19,37 +19,42 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.collect.Lists.newArrayList;
-import java.io.File;
import java.io.IOException;
-import java.nio.file.Files;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import com.google.common.base.Joiner;
+import org.apache.jackrabbit.oak.segment.SegmentArchiveManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Thread-safe class tracking files to be removed.
*/
-class FileReaper {
+public class FileReaper {
private static final Logger logger = LoggerFactory.getLogger(FileReaper.class);
- private final Set<File> files = new HashSet<>();
+ private final Set<String> files = new HashSet<>();
private final Object lock = new Object();
+ private final SegmentArchiveManager archiveManager;
+
+ public FileReaper(SegmentArchiveManager archiveManager) {
+ this.archiveManager = archiveManager;
+ }
+
/**
* Add files to be removed. The same file can be added more than once.
* Duplicates are ignored.
*
* @param files group of files to be removed.
*/
- void add(Iterable<File> files) {
+ void add(Iterable<String> files) {
synchronized (lock) {
- for (File file : files) {
+ for (String file : files) {
this.files.add(file);
}
}
@@ -59,21 +64,20 @@ class FileReaper {
* Reap previously added files.
*/
void reap() {
- Set<File> reap;
+ Set<String> reap;
synchronized (lock) {
reap = new HashSet<>(files);
files.clear();
}
- Set<File> redo = new HashSet<>();
- List<File> removed = newArrayList();
- for (File file : reap) {
- try {
- Files.deleteIfExists(file.toPath());
+ Set<String> redo = new HashSet<>();
+ List<String> removed = newArrayList();
+ for (String file : reap) {
+ if (archiveManager.delete(file)) {
removed.add(file);
- } catch (IOException e) {
- logger.warn(String.format("Unable to remove file %s", file), e);
+ } else {
+ logger.warn("Unable to remove file {}", file);
redo.add(file);
}
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStore.java Tue Feb 13 11:17:42 2018
@@ -43,12 +43,8 @@ import static org.apache.jackrabbit.oak.
import static org.apache.jackrabbit.oak.segment.file.TarRevisions.EXPEDITE_OPTION;
import static org.apache.jackrabbit.oak.segment.file.TarRevisions.timeout;
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
-import java.nio.channels.FileLock;
-import java.nio.channels.OverlappingFileLockException;
import java.util.Collection;
import java.util.List;
import java.util.Set;
@@ -74,6 +70,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.Segment;
import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
import org.apache.jackrabbit.oak.segment.SegmentWriter;
@@ -85,7 +82,6 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.file.tar.CleanupContext;
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
import org.apache.jackrabbit.oak.segment.file.tar.TarFiles;
-import org.apache.jackrabbit.oak.segment.file.tar.TarFiles.CleanupResult;
import org.apache.jackrabbit.oak.spi.state.NodeBuilder;
import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.slf4j.Logger;
@@ -107,8 +103,6 @@ public class FileStore extends AbstractF
private static final int MB = 1024 * 1024;
- static final String LOCK_FILE_NAME = "repo.lock";
-
/**
* GC counter for logging purposes
*/
@@ -122,10 +116,7 @@ public class FileStore extends AbstractF
private final TarFiles tarFiles;
- private final RandomAccessFile lockFile;
-
- @Nonnull
- private final FileLock lock;
+ private final SegmentNodeStorePersistence.RepositoryLock repositoryLock;
private volatile TarRevisions revisions;
@@ -139,7 +130,7 @@ public class FileStore extends AbstractF
* not be removed immediately, because they first need to be closed, and the
* JVM needs to release the memory mapped file references.
*/
- private final FileReaper fileReaper = new FileReaper();
+ private final FileReaper fileReaper;
/**
* This flag is periodically updated by calling the {@code SegmentGCOptions}
@@ -163,13 +154,8 @@ public class FileStore extends AbstractF
FileStore(final FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException {
super(builder);
- lockFile = new RandomAccessFile(new File(directory, LOCK_FILE_NAME), "rw");
- try {
- lock = lockFile.getChannel().lock();
- } catch (OverlappingFileLockException ex) {
- throw new IllegalStateException(directory.getAbsolutePath()
- + " is in use by another store.", ex);
- }
+ SegmentNodeStorePersistence persistence = builder.getPersistence();
+ repositoryLock = persistence.lockRepository();
this.segmentWriter = defaultSegmentWriterBuilder("sys")
.withGeneration(() -> getGcGeneration().nonGC())
@@ -180,24 +166,29 @@ public class FileStore extends AbstractF
this.garbageCollector = new GarbageCollector(
builder.getGcOptions(),
builder.getGcListener(),
- new GCJournal(directory),
+ new GCJournal(persistence.getGCJournalFile()),
builder.getCacheManager()
.withAccessTracking("COMPACT", builder.getStatsProvider()));
- newManifestChecker(directory, builder.getStrictVersionCheck()).checkAndUpdateManifest();
+ newManifestChecker(persistence, builder.getStrictVersionCheck()).checkAndUpdateManifest();
this.stats = new FileStoreStats(builder.getStatsProvider(), this, 0);
- this.tarFiles = TarFiles.builder()
+
+ TarFiles.Builder tarFilesBuilder = TarFiles.builder()
.withDirectory(directory)
.withMemoryMapping(memoryMapping)
.withTarRecovery(recovery)
.withIOMonitor(ioMonitor)
.withFileStoreMonitor(stats)
.withMaxFileSize(builder.getMaxFileSize() * MB)
- .build();
+ .withPersistence(builder.getPersistence());
+
+ this.tarFiles = tarFilesBuilder.build();
long size = this.tarFiles.size();
this.stats.init(size);
+ this.fileReaper = this.tarFiles.createFileReaper();
+
this.snfeListener = builder.getSnfeListener();
fileStoreScheduler.scheduleAtFixedRate(format("TarMK flush [%s]", directory), 5, SECONDS,
@@ -448,9 +439,8 @@ public class FileStore extends AbstractF
}
Closer closer = Closer.create();
- closer.register(lockFile);
- closer.register(lock::release);
- closer.register(tarFiles);
+ closer.register(repositoryLock::unlock);
+ closer.register(tarFiles) ;
closer.register(revisions);
closeAndLogOnFail(closer);
@@ -923,7 +913,7 @@ public class FileStore extends AbstractF
* @throws IOException
*/
@Nonnull
- synchronized List<File> cleanup() throws IOException {
+ synchronized List<String> cleanup() throws IOException {
return cleanup(CompactionResult.skipped(
lastCompactionType,
getGcGeneration(),
@@ -938,7 +928,7 @@ public class FileStore extends AbstractF
* @throws IOException
*/
@Nonnull
- private List<File> cleanup(@Nonnull CompactionResult compactionResult)
+ private List<String> cleanup(@Nonnull CompactionResult compactionResult)
throws IOException {
PrintableStopwatch watch = PrintableStopwatch.createStarted();
@@ -950,7 +940,7 @@ public class FileStore extends AbstractF
// to clear stale weak references in the SegmentTracker
System.gc();
- CleanupResult cleanupResult = tarFiles.cleanup(newCleanupContext(compactionResult.reclaimer()));
+ TarFiles.CleanupResult cleanupResult = tarFiles.cleanup(newCleanupContext(compactionResult.reclaimer()));
if (cleanupResult.isInterrupted()) {
gcListener.info("cleanup interrupted");
}
@@ -972,7 +962,7 @@ public class FileStore extends AbstractF
return cleanupResult.getRemovableFiles();
}
- private String toFileNames(@Nonnull List<File> files) {
+ private String toFileNames(@Nonnull List<String> files) {
if (files.isEmpty()) {
return "none";
} else {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreBuilder.java Tue Feb 13 11:17:42 2018
@@ -40,6 +40,7 @@ import com.google.common.base.Predicate;
import org.apache.jackrabbit.oak.segment.CacheWeights.NodeCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.StringCacheWeigher;
import org.apache.jackrabbit.oak.segment.CacheWeights.TemplateCacheWeigher;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
import org.apache.jackrabbit.oak.segment.RecordCache;
import org.apache.jackrabbit.oak.segment.SegmentNotFoundExceptionListener;
@@ -47,6 +48,7 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.compaction.SegmentGCOptions;
import org.apache.jackrabbit.oak.segment.file.tar.IOMonitor;
import org.apache.jackrabbit.oak.segment.file.tar.IOMonitorAdapter;
+import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.apache.jackrabbit.oak.spi.gc.DelegatingGCMonitor;
import org.apache.jackrabbit.oak.spi.gc.GCMonitor;
@@ -88,6 +90,8 @@ public class FileStoreBuilder {
private boolean memoryMapping = MEMORY_MAPPING_DEFAULT;
+ private SegmentNodeStorePersistence persistence;
+
@Nonnull
private StatisticsProvider statsProvider = StatisticsProvider.NOOP;
@@ -139,6 +143,7 @@ public class FileStoreBuilder {
private FileStoreBuilder(@Nonnull File directory) {
this.directory = checkNotNull(directory);
this.gcListener.registerGCMonitor(new LoggingGCMonitor(LOG));
+ this.persistence = new TarPersistence(directory);
}
/**
@@ -314,6 +319,11 @@ public class FileStoreBuilder {
return this;
}
+ public FileStoreBuilder withCustomPersistence(SegmentNodeStorePersistence persistence) throws IOException {
+ this.persistence = persistence;
+ return this;
+ }
+
/**
* Create a new {@link FileStore} instance with the settings specified in this
* builder. If none of the {@code with} methods have been called before calling
@@ -336,7 +346,7 @@ public class FileStoreBuilder {
checkState(!built, "Cannot re-use builder");
built = true;
directory.mkdirs();
- TarRevisions revisions = new TarRevisions(directory);
+ TarRevisions revisions = new TarRevisions(persistence);
LOG.info("Creating file store {}", this);
FileStore store;
try {
@@ -376,7 +386,7 @@ public class FileStoreBuilder {
checkState(directory.exists() && directory.isDirectory(),
"%s does not exist or is not a directory", directory);
built = true;
- ReadOnlyRevisions revisions = new ReadOnlyRevisions(directory);
+ ReadOnlyRevisions revisions = new ReadOnlyRevisions(persistence);
LOG.info("Creating file store {}", this);
ReadOnlyFileStore store;
try {
@@ -443,6 +453,10 @@ public class FileStoreBuilder {
return snfeListener;
}
+ SegmentNodeStorePersistence getPersistence() {
+ return persistence;
+ }
+
/**
* @return creates or returns the {@code WriterCacheManager} this builder passes or
* passed to the store on {@link #build()}.
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/FileStoreUtil.java Tue Feb 13 11:17:42 2018
@@ -17,11 +17,11 @@
package org.apache.jackrabbit.oak.segment.file;
-import java.io.File;
import java.io.IOException;
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.SegmentIdProvider;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.SegmentStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -45,9 +45,9 @@ class FileStoreUtil {
* found.
* @throws IOException If an I/O error occurs.
*/
- static RecordId findPersistedRecordId(SegmentStore store, SegmentIdProvider idProvider, File journal)
+ static RecordId findPersistedRecordId(SegmentStore store, SegmentIdProvider idProvider, SegmentNodeStorePersistence.JournalFile journalFile)
throws IOException {
- try (JournalReader journalReader = new JournalReader(journal)) {
+ try (JournalReader journalReader = new JournalReader(journalFile)) {
while (journalReader.hasNext()) {
JournalEntry entry = journalReader.next();
try {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/GCJournal.java Tue Feb 13 11:17:42 2018
@@ -19,20 +19,10 @@
package org.apache.jackrabbit.oak.segment.file;
-import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkNotNull;
-import static java.nio.file.Files.newBufferedWriter;
-import static java.nio.file.Files.readAllLines;
-import static java.nio.file.StandardOpenOption.APPEND;
-import static java.nio.file.StandardOpenOption.CREATE;
-import static java.nio.file.StandardOpenOption.DSYNC;
-import static java.nio.file.StandardOpenOption.WRITE;
import static org.apache.jackrabbit.oak.segment.file.tar.GCGeneration.newGCGeneration;
-import java.io.BufferedWriter;
-import java.io.File;
import java.io.IOException;
-import java.nio.file.Path;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -42,13 +32,15 @@ import javax.annotation.Nonnull;
import com.google.common.base.Joiner;
import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.GCJournalFile;
import org.apache.jackrabbit.oak.segment.file.tar.GCGeneration;
+import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* Persists the repository size and the reclaimed size following a cleanup
- * operation in the {@link #GC_JOURNAL gc journal} file with the format:
+ * operation in the {@link TarPersistence#GC_JOURNAL gc journal} file with the format:
* 'repoSize, reclaimedSize, timestamp, gc generation, gc full generation (since Oak 1.8),
* number of nodes compacted, root id (since Oak 1.8)'.
*/
@@ -56,15 +48,12 @@ public class GCJournal {
private static final Logger LOG = LoggerFactory.getLogger(GCJournal.class);
- public static final String GC_JOURNAL = "gc.log";
-
- @Nonnull
- private final File directory;
+ private final GCJournalFile journalFile;
private GCJournalEntry latest;
- public GCJournal(@Nonnull File directory) {
- this.directory = checkNotNull(directory);
+ public GCJournal(@Nonnull GCJournalFile journalFile) {
+ this.journalFile = journalFile;
}
/**
@@ -90,13 +79,8 @@ public class GCJournal {
}
latest = new GCJournalEntry(repoSize, reclaimedSize,
System.currentTimeMillis(), gcGeneration, nodes, checkNotNull(root));
- Path path = new File(directory, GC_JOURNAL).toPath();
try {
- try (BufferedWriter w = newBufferedWriter(path, UTF_8, WRITE,
- APPEND, CREATE, DSYNC)) {
- w.write(latest.toString());
- w.newLine();
- }
+ journalFile.writeLine(latest.toString());
} catch (IOException e) {
LOG.error("Error writing gc journal", e);
}
@@ -130,13 +114,10 @@ public class GCJournal {
}
private List<String> readLines() {
- File file = new File(directory, GC_JOURNAL);
- if (file.exists()) {
- try {
- return readAllLines(file.toPath(), UTF_8);
- } catch (IOException e) {
- LOG.error("Error reading gc journal", e);
- }
+ try {
+ return journalFile.readLines();
+ } catch (IOException e) {
+ LOG.error("Error reading gc journal", e);
}
return new ArrayList<String>();
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/JournalReader.java Tue Feb 13 11:17:42 2018
@@ -19,14 +19,11 @@
package org.apache.jackrabbit.oak.segment.file;
-import static java.nio.charset.Charset.defaultCharset;
-
import java.io.Closeable;
-import java.io.File;
import java.io.IOException;
import java.util.List;
-import org.apache.commons.io.input.ReversedLinesFileReader;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -40,10 +37,10 @@ import com.google.common.collect.Abstrac
public final class JournalReader extends AbstractIterator<JournalEntry> implements Closeable {
private static final Logger LOG = LoggerFactory.getLogger(JournalReader.class);
- private final ReversedLinesFileReader journal;
+ private final SegmentNodeStorePersistence.JournalFileReader reader;
- public JournalReader(File journalFile) throws IOException {
- journal = new ReversedLinesFileReader(journalFile, defaultCharset());
+ public JournalReader(SegmentNodeStorePersistence.JournalFile journal) throws IOException {
+ this.reader = journal.openJournalReader();
}
/**
@@ -54,7 +51,7 @@ public final class JournalReader extends
protected JournalEntry computeNext() {
try {
String line = null;
- while ((line = journal.readLine()) != null) {
+ while ((line = reader.readLine()) != null) {
if (line.indexOf(' ') != -1) {
List<String> splits = Splitter.on(' ').splitToList(line);
String revision = splits.get(0);
@@ -83,6 +80,6 @@ public final class JournalReader extends
@Override
public void close() throws IOException {
- journal.close();
+ reader.close();
}
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalGCJournalFile.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.newBufferedWriter;
+import static java.nio.file.Files.readAllLines;
+import static java.nio.file.StandardOpenOption.APPEND;
+import static java.nio.file.StandardOpenOption.CREATE;
+import static java.nio.file.StandardOpenOption.DSYNC;
+import static java.nio.file.StandardOpenOption.WRITE;
+
+public class LocalGCJournalFile implements SegmentNodeStorePersistence.GCJournalFile {
+
+ private final File file;
+
+ public LocalGCJournalFile(File parent, String name) {
+ this(new File(parent, name));
+ }
+
+ public LocalGCJournalFile(File file) {
+ this.file = file;
+ }
+
+ @Override
+ public void writeLine(String line) throws IOException {
+ try (BufferedWriter w = newBufferedWriter(file.toPath(), UTF_8, WRITE, APPEND, CREATE, DSYNC)) {
+ w.write(line);
+ w.newLine();
+ }
+ }
+
+ @Override
+ public List<String> readLines() throws IOException {
+ if (file.exists()) {
+ return readAllLines(file.toPath(), UTF_8);
+ }
+ return new ArrayList<String>();
+ }
+}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/LocalManifestFile.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file;
+
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Properties;
+
+public class LocalManifestFile implements ManifestFile {
+
+ private final File file;
+
+ public LocalManifestFile(File parent, String name) {
+ this(new File(parent, name));
+ }
+
+ public LocalManifestFile(File file) {
+ this.file = file;
+ }
+
+ @Override
+ public boolean exists() {
+ return file.exists();
+ }
+
+ @Override
+ public Properties load() throws IOException {
+ Properties properties = new Properties();
+ try (FileReader r = new FileReader(file)) {
+ properties.load(r);
+ }
+ return properties;
+ }
+
+ @Override
+ public void save(Properties properties) throws IOException {
+ properties.store(new FileWriter(file), null);
+ }
+
+}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/Manifest.java Tue Feb 13 11:17:42 2018
@@ -17,9 +17,9 @@
package org.apache.jackrabbit.oak.segment.file;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FileWriter;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile;
+
import java.io.IOException;
import java.util.Properties;
@@ -34,12 +34,8 @@ class Manifest {
* @return A manifest file.
* @throws IOException If any error occurs when loading the manifest.
*/
- static Manifest load(File file) throws IOException {
- Properties properties = new Properties();
- try (FileReader r = new FileReader(file)) {
- properties.load(r);
- }
- return new Manifest(properties);
+ static Manifest load(ManifestFile file) throws IOException {
+ return new Manifest(file.load());
}
/**
@@ -85,8 +81,8 @@ class Manifest {
* @param file The file to save the manifest to.
* @throws IOException if an error occurs while saving the manifest.
*/
- void save(File file) throws IOException {
- properties.store(new FileWriter(file), null);
+ void save(ManifestFile file) throws IOException {
+ file.save(properties);
}
private int getIntegerProperty(String name, int otherwise) {
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ManifestChecker.java Tue Feb 13 11:17:42 2018
@@ -17,21 +17,22 @@
package org.apache.jackrabbit.oak.segment.file;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence.ManifestFile;
+
import static com.google.common.base.Preconditions.checkArgument;
-import java.io.File;
import java.io.IOException;
public class ManifestChecker {
- public static ManifestChecker newManifestChecker(File path, boolean shouldExist, int minStoreVersion, int maxStoreVersion) {
- checkArgument(path != null, "path");
+ public static ManifestChecker newManifestChecker(ManifestFile file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) {
+ checkArgument(file != null, "file");
checkArgument(minStoreVersion > 0, "minStoreVersion");
checkArgument(maxStoreVersion > 0, "maxStoreVersion");
- return new ManifestChecker(path, shouldExist, minStoreVersion, maxStoreVersion);
+ return new ManifestChecker(file, shouldExist, minStoreVersion, maxStoreVersion);
}
- private final File file;
+ private final ManifestFile file;
private final boolean shouldExist;
@@ -39,7 +40,7 @@ public class ManifestChecker {
private final int maxStoreVersion;
- private ManifestChecker(File file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) {
+ private ManifestChecker(ManifestFile file, boolean shouldExist, int minStoreVersion, int maxStoreVersion) {
this.file = file;
this.shouldExist = shouldExist;
this.minStoreVersion = minStoreVersion;
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyFileStore.java Tue Feb 13 11:17:42 2018
@@ -63,7 +63,7 @@ public class ReadOnlyFileStore extends A
ReadOnlyFileStore(FileStoreBuilder builder) throws InvalidFileStoreVersionException, IOException {
super(builder);
- newManifestChecker(directory, builder.getStrictVersionCheck()).checkManifest();
+ newManifestChecker(builder.getPersistence(), builder.getStrictVersionCheck()).checkManifest();
tarFiles = TarFiles.builder()
.withDirectory(directory)
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/ReadOnlyRevisions.java Tue Feb 13 11:17:42 2018
@@ -23,9 +23,7 @@ import static com.google.common.base.Pre
import static org.apache.jackrabbit.oak.segment.file.FileStoreUtil.findPersistedRecordId;
import java.io.Closeable;
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.util.concurrent.atomic.AtomicReference;
import javax.annotation.Nonnull;
@@ -34,31 +32,19 @@ import com.google.common.base.Function;
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Revisions;
import org.apache.jackrabbit.oak.segment.SegmentIdProvider;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.SegmentStore;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
public class ReadOnlyRevisions implements Revisions, Closeable {
- private static final Logger LOG = LoggerFactory
- .getLogger(ReadOnlyRevisions.class);
-
- public static final String JOURNAL_FILE_NAME = "journal.log";
-
@Nonnull
private final AtomicReference<RecordId> head;
@Nonnull
- private final File directory;
+ private final SegmentNodeStorePersistence.JournalFile journalFile;
- @Nonnull
- private final RandomAccessFile journalFile;
-
- public ReadOnlyRevisions(@Nonnull File directory) throws IOException {
- this.directory = checkNotNull(directory);
- this.journalFile = new RandomAccessFile(new File(directory,
- JOURNAL_FILE_NAME), "r");
- this.journalFile.seek(journalFile.length());
+ public ReadOnlyRevisions(@Nonnull SegmentNodeStorePersistence persistence) {
+ this.journalFile = checkNotNull(persistence).getJournalFile();
this.head = new AtomicReference<>(null);
}
@@ -74,7 +60,7 @@ public class ReadOnlyRevisions implement
if (head.get() != null) {
return;
}
- RecordId persistedId = findPersistedRecordId(store, idProvider, new File(directory, JOURNAL_FILE_NAME));
+ RecordId persistedId = findPersistedRecordId(store, idProvider, journalFile);
if (persistedId == null) {
throw new IllegalStateException("Cannot start readonly store from empty journal");
}
@@ -112,14 +98,8 @@ public class ReadOnlyRevisions implement
throw new UnsupportedOperationException("ReadOnly Revisions");
}
- /**
- * Close the underlying journal file.
- *
- * @throws IOException
- */
@Override
public void close() throws IOException {
- journalFile.close();
+ // do nothing
}
-
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java?rev=1824115&r1=1824114&r2=1824115&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/TarRevisions.java Tue Feb 13 11:17:42 2018
@@ -19,16 +19,13 @@
package org.apache.jackrabbit.oak.segment.file;
-import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.base.Preconditions.checkState;
import static java.lang.Long.MAX_VALUE;
import static java.util.concurrent.TimeUnit.DAYS;
import static org.apache.jackrabbit.oak.segment.file.FileStoreUtil.findPersistedRecordId;
import java.io.Closeable;
-import java.io.File;
import java.io.IOException;
-import java.io.RandomAccessFile;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.concurrent.locks.Lock;
@@ -44,13 +41,15 @@ import com.google.common.base.Supplier;
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.Revisions;
import org.apache.jackrabbit.oak.segment.SegmentIdProvider;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
import org.apache.jackrabbit.oak.segment.SegmentStore;
+import org.apache.jackrabbit.oak.segment.file.tar.TarPersistence;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
* This implementation of {@code Revisions} is backed by a
- * {@link #JOURNAL_FILE_NAME journal} file where the current head is persisted
+ * {@link TarPersistence#JOURNAL_FILE_NAME journal} file where the current head is persisted
* by calling {@link #tryFlush(Flusher)}.
* <p>
* The {@link #setHead(Function, Option...)} method supports a timeout
@@ -62,8 +61,6 @@ import org.slf4j.LoggerFactory;
public class TarRevisions implements Revisions, Closeable {
private static final Logger LOG = LoggerFactory.getLogger(TarRevisions.class);
- public static final String JOURNAL_FILE_NAME = "journal.log";
-
/**
* The lock protecting {@link #journalFile}.
*/
@@ -72,14 +69,15 @@ public class TarRevisions implements Rev
@Nonnull
private final AtomicReference<RecordId> head;
- @Nonnull
- private final File directory;
+ private final SegmentNodeStorePersistence persistence;
+
+ private final SegmentNodeStorePersistence.JournalFile journalFile;
/**
- * The journal file. It is protected by {@link #journalFileLock}. It becomes
+ * The journal file writer. It is protected by {@link #journalFileLock}. It becomes
* {@code null} after it's closed.
*/
- private volatile RandomAccessFile journalFile;
+ private volatile SegmentNodeStorePersistence.JournalFileWriter journalFileWriter;
/**
* The persisted head of the root journal, used to determine whether the
@@ -138,16 +136,15 @@ public class TarRevisions implements Rev
/**
* Create a new instance placing the journal log file into the passed
* {@code directory}.
- * @param directory directory of the journal file
+ * @param persistence object representing the segment persistence
* @throws IOException
*/
- public TarRevisions(@Nonnull File directory) throws IOException {
- this.directory = checkNotNull(directory);
- this.journalFile = new RandomAccessFile(new File(directory,
- JOURNAL_FILE_NAME), "rw");
- this.journalFile.seek(journalFile.length());
+ public TarRevisions(SegmentNodeStorePersistence persistence) throws IOException {
+ this.journalFile = persistence.getJournalFile();
+ this.journalFileWriter = journalFile.openJournalWriter();
this.head = new AtomicReference<>(null);
this.persistedHead = new AtomicReference<>(null);
+ this.persistence = persistence;
}
/**
@@ -164,7 +161,7 @@ public class TarRevisions implements Rev
if (head.get() != null) {
return;
}
- RecordId persistedId = findPersistedRecordId(store, idProvider, new File(directory, JOURNAL_FILE_NAME));
+ RecordId persistedId = findPersistedRecordId(store, idProvider, journalFile);
if (persistedId == null) {
head.set(writeInitialNode.get());
} else {
@@ -226,7 +223,7 @@ public class TarRevisions implements Rev
}
private void doFlush(Flusher flusher) throws IOException {
- if (journalFile == null) {
+ if (journalFileWriter == null) {
LOG.debug("No journal file available, skipping flush");
return;
}
@@ -238,8 +235,7 @@ public class TarRevisions implements Rev
}
flusher.flush();
LOG.debug("TarMK journal update {} -> {}", before, after);
- journalFile.writeBytes(after.toString10() + " root " + System.currentTimeMillis() + "\n");
- journalFile.getChannel().force(false);
+ journalFileWriter.writeLine(after.toString10() + " root " + System.currentTimeMillis());
persistedHead.set(after);
}
@@ -354,11 +350,11 @@ public class TarRevisions implements Rev
public void close() throws IOException {
journalFileLock.lock();
try {
- if (journalFile == null) {
+ if (journalFileWriter == null) {
return;
}
- journalFile.close();
- journalFile = null;
+ journalFileWriter.close();
+ journalFileWriter = null;
} finally {
journalFileLock.unlock();
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/GraphLoader.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,98 @@
+package org.apache.jackrabbit.oak.segment.file.tar;
+
+import org.apache.jackrabbit.oak.segment.util.ReaderAtEnd;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.zip.CRC32;
+
+import static com.google.common.collect.Lists.newArrayListWithCapacity;
+import static com.google.common.collect.Maps.newHashMapWithExpectedSize;
+import static org.apache.jackrabbit.oak.segment.file.tar.TarConstants.GRAPH_MAGIC;
+
+public final class GraphLoader {
+
+ private static final Logger log = LoggerFactory.getLogger(GraphLoader.class);
+
+ private static final int FOOTER_SIZE = 16;
+
+ private GraphLoader() {
+ }
+
+ /**
+ * Loads the optional pre-compiled graph entry from the given tar file.
+ *
+ * @return the graph or {@code null} if one was not found
+ * @throws IOException if the tar file could not be read
+ */
+ public static ByteBuffer loadGraph(ReaderAtEnd readerAtEnd) throws IOException {
+ ByteBuffer meta = readerAtEnd.readAtEnd(FOOTER_SIZE, FOOTER_SIZE);
+
+ int crc32 = meta.getInt();
+ int count = meta.getInt();
+ int bytes = meta.getInt();
+ int magic = meta.getInt();
+
+ if (magic != GRAPH_MAGIC) {
+ log.warn("Invalid graph magic number");
+ return null;
+ }
+
+ if (count < 0) {
+ log.warn("Invalid number of entries");
+ return null;
+ }
+
+ if (bytes < 4 + count * 34) {
+ log.warn("Invalid entry size");
+ return null;
+ }
+
+ ByteBuffer graph = readerAtEnd.readAtEnd(bytes, bytes);
+
+ byte[] b = new byte[bytes - FOOTER_SIZE];
+
+ graph.mark();
+ graph.get(b);
+ graph.reset();
+
+ CRC32 checksum = new CRC32();
+ checksum.update(b);
+
+ if (crc32 != (int) checksum.getValue()) {
+ log.warn("Invalid graph checksum in tar file");
+ return null;
+ }
+
+ return graph;
+ }
+
+ public static Map<UUID, List<UUID>> parseGraph(ByteBuffer buffer) {
+ int nEntries = buffer.getInt(buffer.limit() - 12);
+
+ Map<UUID, List<UUID>> graph = newHashMapWithExpectedSize(nEntries);
+
+ for (int i = 0; i < nEntries; i++) {
+ long msb = buffer.getLong();
+ long lsb = buffer.getLong();
+ int nVertices = buffer.getInt();
+
+ List<UUID> vertices = newArrayListWithCapacity(nVertices);
+
+ for (int j = 0; j < nVertices; j++) {
+ long vMsb = buffer.getLong();
+ long vLsb = buffer.getLong();
+ vertices.add(new UUID(vMsb, vLsb));
+ }
+
+ graph.put(new UUID(msb, lsb), vertices);
+ }
+
+ return graph;
+ }
+}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java?rev=1824115&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/file/tar/LocalJournalFile.java Tue Feb 13 11:17:42 2018
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.segment.file.tar;
+
+import org.apache.commons.io.input.ReversedLinesFileReader;
+import org.apache.jackrabbit.oak.segment.SegmentNodeStorePersistence;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.RandomAccessFile;
+
+import static java.nio.charset.Charset.defaultCharset;
+
+public class LocalJournalFile implements SegmentNodeStorePersistence.JournalFile {
+
+ private final File journalFile;
+
+ public LocalJournalFile(File directory, String journalFile) {
+ this.journalFile = new File(directory, journalFile);
+ }
+
+ public LocalJournalFile(File journalFile) {
+ this.journalFile = journalFile;
+ }
+
+ @Override
+ public SegmentNodeStorePersistence.JournalFileReader openJournalReader() throws IOException {
+ return new LocalJournalFileReader(journalFile);
+ }
+
+ @Override
+ public SegmentNodeStorePersistence.JournalFileWriter openJournalWriter() throws IOException {
+ return new LocalJournalFileWriter(journalFile);
+ }
+
+ @Override
+ public String getName() {
+ return journalFile.getName();
+ }
+
+ @Override
+ public boolean exists() {
+ return journalFile.exists();
+ }
+
+ private static class LocalJournalFileReader implements SegmentNodeStorePersistence.JournalFileReader {
+
+ private final ReversedLinesFileReader journal;
+
+ public LocalJournalFileReader(File file) throws IOException {
+ journal = new ReversedLinesFileReader(file, defaultCharset());
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ return journal.readLine();
+ }
+
+ @Override
+ public void close() throws IOException {
+ journal.close();
+ }
+ }
+
+ private static class LocalJournalFileWriter implements SegmentNodeStorePersistence.JournalFileWriter {
+
+ private final RandomAccessFile journalFile;
+
+ public LocalJournalFileWriter(File file) throws IOException {
+ journalFile = new RandomAccessFile(file, "rw");
+ journalFile.seek(journalFile.length());
+ }
+
+ @Override
+ public void truncate() throws IOException {
+ journalFile.setLength(0);
+ }
+
+ @Override
+ public void writeLine(String line) throws IOException {
+ journalFile.writeBytes(line + "\n");
+ journalFile.getChannel().force(false);
+ }
+
+ @Override
+ public void close() throws IOException {
+ journalFile.close();
+ }
+ }
+}