You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by to...@apache.org on 2019/09/06 21:14:23 UTC
svn commit: r1866535 - in /jackrabbit/oak/trunk/oak-segment-tar/src:
main/java/org/apache/jackrabbit/oak/segment/split/
test/java/org/apache/jackrabbit/oak/segment/split/
Author: tomekr
Date: Fri Sep 6 21:14:23 2019
New Revision: 1866535
URL: http://svn.apache.org/viewvc?rev=1866535&view=rev
Log:
OAK-8605: Read the journal lazily in the SplitPersistence
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFile.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileReader.java
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/
jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileTest.java
Modified:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFile.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFile.java?rev=1866535&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFile.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFile.java Fri Sep 6 21:14:23 2019
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.split;
+
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter;
+
+import java.io.IOException;
+import java.util.Optional;
+
+public class SplitJournalFile implements JournalFile {
+
+ private final JournalFile roJournalFile;
+
+ private final JournalFile rwJournalFile;
+
+ private final Optional<String> lastRoJournalEntry;
+
+ public SplitJournalFile(JournalFile roJournalFile, JournalFile rwJournalFile, Optional<String> lastRoJournalEntry) {
+ this.roJournalFile = roJournalFile;
+ this.rwJournalFile = rwJournalFile;
+ this.lastRoJournalEntry = lastRoJournalEntry;
+ }
+
+ @Override
+ public JournalFileReader openJournalReader() throws IOException {
+ return new SplitJournalFileReader(roJournalFile.openJournalReader(), rwJournalFile.openJournalReader(), lastRoJournalEntry);
+ }
+
+ @Override
+ public JournalFileWriter openJournalWriter() throws IOException {
+ return rwJournalFile.openJournalWriter();
+ }
+
+ @Override
+ public String getName() {
+ return rwJournalFile.getName();
+ }
+
+ @Override
+ public boolean exists() {
+ return roJournalFile.exists() || rwJournalFile.exists();
+ }
+}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileReader.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileReader.java?rev=1866535&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileReader.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileReader.java Fri Sep 6 21:14:23 2019
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.split;
+
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader;
+
+import java.io.IOException;
+import java.util.Optional;
+
+public class SplitJournalFileReader implements JournalFileReader {
+
+ private final JournalFileReader roJournalReader;
+
+ private final JournalFileReader rwJournalReader;
+
+ private final Optional<String> lastRoJournalEntry;
+
+ private boolean rwJournalReaderHasFinished;
+
+ private boolean roJournalReaderHasFinished;
+
+ public SplitJournalFileReader(JournalFileReader roJournalReader, JournalFileReader rwJournalReader, Optional<String> lastRoJournalEntry) {
+ this.roJournalReader = roJournalReader;
+ this.rwJournalReader = rwJournalReader;
+ this.lastRoJournalEntry = lastRoJournalEntry;
+ }
+
+ @Override
+ public String readLine() throws IOException {
+ if (!rwJournalReaderHasFinished) {
+ String line = rwJournalReader.readLine();
+ if (line != null) {
+ return line;
+ }
+ rwJournalReaderHasFinished = true;
+ if (lastRoJournalEntry.isPresent()) {
+ rewindToLine(roJournalReader, lastRoJournalEntry.get());
+ return lastRoJournalEntry.get();
+ } else {
+ roJournalReaderHasFinished = true;
+ }
+ }
+ if (!roJournalReaderHasFinished) {
+ String line = roJournalReader.readLine();
+ if (line != null) {
+ return line;
+ }
+ roJournalReaderHasFinished = true;
+ }
+ return null;
+ }
+
+ private void rewindToLine(JournalFileReader reader, String stopLine) throws IOException {
+ while (true) {
+ String line = reader.readLine();
+ if (line == null || line.equals(stopLine)) {
+ break;
+ }
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ roJournalReader.close();
+ rwJournalReader.close();
+ }
+}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java?rev=1866535&r1=1866534&r2=1866535&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/split/SplitPersistence.java Fri Sep 6 21:14:23 2019
@@ -20,14 +20,12 @@ import org.apache.jackrabbit.oak.segment
import org.apache.jackrabbit.oak.segment.spi.persistence.GCJournalFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader;
-import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter;
import org.apache.jackrabbit.oak.segment.spi.persistence.ManifestFile;
import org.apache.jackrabbit.oak.segment.spi.persistence.RepositoryLock;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentArchiveManager;
import org.apache.jackrabbit.oak.segment.spi.persistence.SegmentNodeStorePersistence;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
@@ -41,6 +39,8 @@ public class SplitPersistence implements
private final Optional<String> lastRoArchive;
+ private final Optional<String> lastRoJournalEntry;
+
public SplitPersistence(SegmentNodeStorePersistence roPersistence, SegmentNodeStorePersistence rwPersistence) throws IOException {
this.roPersistence = roPersistence;
this.rwPersistence = rwPersistence;
@@ -51,11 +51,18 @@ public class SplitPersistence implements
}
Properties properties = manifest.load();
lastRoArchive = Optional.ofNullable(properties.getProperty("split.lastRoArchive"));
+ lastRoJournalEntry = Optional.ofNullable(properties.getProperty("split.lastRoJournalEntry"));
}
private void initialize() throws IOException {
Properties properties = roPersistence.getManifestFile().load();
properties.setProperty("split.initialized", "true");
+ try (JournalFileReader journalFileReader = roPersistence.getJournalFile().openJournalReader()) {
+ String journalLine;
+ if ((journalLine = journalFileReader.readLine()) != null) {
+ properties.setProperty("split.lastRoJournalEntry", journalLine);
+ }
+ }
Optional<String> lastArchive = getLastArchive();
lastArchive.ifPresent(a -> properties.setProperty("split.lastRoArchive", a));
rwPersistence.getManifestFile().save(properties);
@@ -64,22 +71,6 @@ public class SplitPersistence implements
for (String line : roPersistence.getGCJournalFile().readLines()) {
gcJournalFile.writeLine(line);
}
-
- List<String> journalLines = new ArrayList<>();
- try (JournalFileReader journalFileReader = roPersistence.getJournalFile().openJournalReader()) {
- String journalLine;
- while ((journalLine = journalFileReader.readLine()) != null) {
- journalLines.add(journalLine);
- }
- }
-
- Collections.reverse(journalLines);
-
- try (JournalFileWriter journalFileWriter = rwPersistence.getJournalFile().openJournalWriter()) {
- for (String line : journalLines) {
- journalFileWriter.writeLine(line);
- }
- }
}
private Optional<String> getLastArchive() throws IOException {
@@ -112,7 +103,7 @@ public class SplitPersistence implements
@Override
public JournalFile getJournalFile() {
- return rwPersistence.getJournalFile();
+ return new SplitJournalFile(roPersistence.getJournalFile(), rwPersistence.getJournalFile(), lastRoJournalEntry);
}
@Override
Added: jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileTest.java?rev=1866535&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileTest.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/test/java/org/apache/jackrabbit/oak/segment/split/SplitJournalFileTest.java Fri Sep 6 21:14:23 2019
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.jackrabbit.oak.segment.split;
+
+import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileReader;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFileWriter;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.IOException;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+public class SplitJournalFileTest {
+
+ @Rule
+ public TemporaryFolder temporaryFolder = new TemporaryFolder();
+
+ private JournalFile roFile;
+
+ private JournalFile rwFile;
+
+ @Before
+ public void prepareJournals() throws IOException {
+ roFile = new LocalJournalFile(temporaryFolder.newFile());
+ rwFile = new LocalJournalFile(temporaryFolder.newFile());
+
+ try (JournalFileWriter writer = roFile.openJournalWriter()) {
+ writer.writeLine("line 1");
+ writer.writeLine("line 2");
+ writer.writeLine("line 3");
+ writer.writeLine("line 3a");
+ writer.writeLine("line 3b");
+ writer.writeLine("line 3c");
+ }
+
+ try (JournalFileWriter writer = rwFile.openJournalWriter()) {
+ writer.writeLine("line 4");
+ writer.writeLine("line 5");
+ writer.writeLine("line 6");
+ }
+ }
+
+ @Test
+ public void testIgnoreNewRoLines() throws IOException {
+ SplitJournalFile splitJournalFile = new SplitJournalFile(roFile, rwFile, Optional.of("line 3"));
+ try (JournalFileReader reader = splitJournalFile.openJournalReader()) {
+ for (int i = 6; i >= 1; i--) {
+ assertEquals("line " + i, reader.readLine());
+ }
+ assertNull(reader.readLine());
+ assertNull(reader.readLine());
+ }
+ }
+
+ @Test
+ public void testIgnoreWholeRoJournal() throws IOException {
+ SplitJournalFile splitJournalFile = new SplitJournalFile(roFile, rwFile, Optional.empty());
+ try (JournalFileReader reader = splitJournalFile.openJournalReader()) {
+ assertEquals("line 6", reader.readLine());
+ assertEquals("line 5", reader.readLine());
+ assertEquals("line 4", reader.readLine());
+ assertNull(reader.readLine());
+ assertNull(reader.readLine());
+ }
+ }
+}