You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2018/11/21 14:42:49 UTC
svn commit: r1847110 - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/
Author: frm
Date: Wed Nov 21 14:42:49 2018
New Revision: 1847110
URL: http://svn.apache.org/viewvc?rev=1847110&view=rev
Log:
OAK-7866 - Introduce the recover-journal command
Added:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java (with props)
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/SearchNodes.java
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java?rev=1847110&r1=1847109&r2=1847110&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/AvailableModes.java Wed Nov 21 14:42:49 2018
@@ -61,5 +61,6 @@ public final class AvailableModes {
.put(DataStoreCommand.NAME, new DataStoreCommand())
.put("segment-copy", new SegmentCopyCommand())
.put("search-nodes", new SearchNodesCommand())
+ .put("recover-journal", new RecoverJournalCommand())
.build());
}
Added: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java?rev=1847110&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java (added)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java Wed Nov 21 14:42:49 2018
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.run;
+
+import static java.util.Arrays.asList;
+
+import java.io.File;
+
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
+import org.apache.jackrabbit.oak.run.commons.Command;
+import org.apache.jackrabbit.oak.segment.tool.RecoverJournal;
+
+class RecoverJournalCommand implements Command {
+
+ @Override
+ public void execute(String... args) throws Exception {
+ OptionParser options = new OptionParser();
+ OptionSpec<?> help = options.acceptsAll(asList("h", "help"), "Prints help and exits");
+ OptionSpec<File> dir = options.nonOptions()
+ .describedAs("path")
+ .ofType(File.class);
+ OptionSet parsed = options.parse(args);
+
+ if (parsed.has(help)) {
+ options.printHelpOn(System.out);
+ System.exit(0);
+ }
+
+ if (parsed.valuesOf(dir).size() == 0) {
+ System.err.println("Segment Store path not specified");
+ System.exit(1);
+ }
+
+ if (parsed.valuesOf(dir).size() > 1) {
+ System.err.println("Too many Segment Store paths specified");
+ System.exit(1);
+ }
+
+ int code = RecoverJournal.builder()
+ .withPath(dir.value(parsed))
+ .withOut(System.out)
+ .withErr(System.err)
+ .build()
+ .run();
+ System.exit(code);
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RecoverJournalCommand.java
------------------------------------------------------------------------------
svn:eol-style = native
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java?rev=1847110&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java Wed Nov 21 14:42:49 2018
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment.tool;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Preconditions.checkState;
+import static org.apache.jackrabbit.oak.segment.tool.Utils.openReadOnlyFileStore;
+import static org.apache.jackrabbit.oak.segment.tool.Utils.parseSegmentInfoTimestamp;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.nio.file.Files;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.RecordType;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
+import org.apache.jackrabbit.oak.segment.SegmentNotFoundException;
+import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
+
+public class RecoverJournal {
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ public static class Builder {
+
+ private File path;
+
+ private PrintStream out = System.out;
+
+ private PrintStream err = System.err;
+
+ private Builder() {
+ // Prevent external instantiation.
+ }
+
+ public Builder withPath(File path) {
+ this.path = checkNotNull(path, "path");
+ return this;
+ }
+
+ public Builder withOut(PrintStream out) {
+ this.out = checkNotNull(out, "out");
+ return this;
+ }
+
+ public Builder withErr(PrintStream err) {
+ this.err = checkNotNull(err, "err");
+ return this;
+ }
+
+ public RecoverJournal build() {
+ checkState(path != null, "path not specified");
+ return new RecoverJournal(this);
+ }
+
+ }
+
+ private final File path;
+
+ private final PrintStream out;
+
+ private final PrintStream err;
+
+ private final Set<String> notFoundSegments = new HashSet<>();
+
+ private RecoverJournal(Builder builder) {
+ this.path = builder.path;
+ this.out = builder.out;
+ this.err = builder.err;
+ }
+
+ public int run() {
+ List<Entry> entries = new ArrayList<>();
+
+ try (ReadOnlyFileStore store = openReadOnlyFileStore(path)) {
+ recoverEntries(store, entries);
+ } catch (Exception e) {
+ out.println("Unable to recover the journal entries, aborting");
+ e.printStackTrace(err);
+ return 1;
+ }
+
+ File journalBackup = journalBackupName();
+
+ if (journalBackup == null) {
+ err.println("Too many journal backups, please cleanup");
+ return 1;
+ }
+
+ File journal = new File(path, "journal.log");
+
+ try {
+ Files.move(journal.toPath(), journalBackup.toPath());
+ } catch (IOException e) {
+ err.println("Unable to backup old journal, aborting");
+ e.printStackTrace(err);
+ return 1;
+ }
+
+ out.printf("Old journal backed up at %s\n", journalBackup.getName());
+
+ boolean rollback;
+
+ try (PrintWriter w = new PrintWriter(new BufferedWriter(new FileWriter(journal)))) {
+ for (Entry e : entries) {
+ w.printf("%s root %d\n", e.recordId.toString10(), e.timestamp);
+ }
+ rollback = false;
+ } catch (IOException e) {
+ err.println("Unable to write the recovered journal, rolling back");
+ e.printStackTrace(err);
+ rollback = true;
+ }
+
+ if (rollback) {
+ try {
+ Files.deleteIfExists(journal.toPath());
+ } catch (IOException e) {
+ err.println("Unable to delete the recovered journal, aborting");
+ e.printStackTrace(err);
+ return 1;
+ }
+
+ try {
+ Files.move(journalBackup.toPath(), journal.toPath());
+ } catch (IOException e) {
+ err.println("Unable to roll back the old journal, aborting");
+ e.printStackTrace(err);
+ return 1;
+ }
+
+ out.println("Old journal rolled back");
+
+ return 1;
+ }
+
+ out.println("Journal recovered");
+
+ return 0;
+ }
+
+ private File journalBackupName() {
+ for (int attempt = 0; attempt < 1000; attempt++) {
+ File backup = new File(path, String.format("journal.log.bak.%03d", attempt));
+ if (backup.exists()) {
+ continue;
+ }
+ return backup;
+ }
+ return null;
+ }
+
+ private static class Entry {
+
+ long timestamp;
+
+ RecordId recordId;
+
+ Entry(long timestamp, RecordId recordId) {
+ this.timestamp = timestamp;
+ this.recordId = recordId;
+ }
+
+ }
+
+ private void recoverEntries(ReadOnlyFileStore fileStore, List<Entry> entries) {
+ for (SegmentId segmentId : fileStore.getSegmentIds()) {
+ try {
+ recoverEntries(fileStore, segmentId, entries);
+ } catch (SegmentNotFoundException e) {
+ handle(e);
+ }
+ }
+
+ entries.sort((left, right) -> {
+ // Two entries with different timestamp will be sorted in ascending
+ // order of timestamp.
+
+ int timestampComparison = Long.compare(left.timestamp, right.timestamp);
+ if (timestampComparison != 0) {
+ return timestampComparison;
+ }
+
+ // Comparing segment IDs with the same timestamp is totally
+ // arbitrary. The relative order of two segments with the same
+ // timestamp is unimportant.
+
+ SegmentId leftSegmentId = left.recordId.getSegmentId();
+ SegmentId rightSegmentId = right.recordId.getSegmentId();
+ int segmentIdComparison = leftSegmentId.compareTo(rightSegmentId);
+ if (segmentIdComparison != 0) {
+ return segmentIdComparison;
+ }
+
+ // Records from the same segments are sorted in decreasing order
+ // of their record number. This builds on the assumption that a
+ // record with a higher record number was added after a record
+ // with a lower one, and therefor is more recent.
+
+ int leftRecordNumber = left.recordId.getRecordNumber();
+ int rightRecordNumber = right.recordId.getRecordNumber();
+ return Integer.compare(rightRecordNumber, leftRecordNumber);
+ });
+ }
+
+ private void recoverEntries(ReadOnlyFileStore fileStore, SegmentId segmentId, List<Entry> entries) {
+ if (segmentId.isBulkSegmentId()) {
+ return;
+ }
+
+ Long timestamp = parseSegmentInfoTimestamp(segmentId);
+
+ if (timestamp == null) {
+ err.printf("No timestamp found in segment %s\n", segmentId);
+ return;
+ }
+
+ segmentId.getSegment().forEachRecord((number, type, offset) -> {
+ if (type != RecordType.NODE) {
+ return;
+ }
+ try {
+ recoverEntries(fileStore, timestamp, new RecordId(segmentId, number), entries);
+ } catch (SegmentNotFoundException e) {
+ handle(e);
+ }
+ });
+ }
+
+ private void recoverEntries(ReadOnlyFileStore fileStore, long timestamp, RecordId recordId, List<Entry> entries) {
+ SegmentNodeState nodeState = fileStore.getReader().readNode(recordId);
+
+ if (nodeState.hasChildNode("checkpoints") && nodeState.hasChildNode("root")) {
+ entries.add(new Entry(timestamp, recordId));
+ }
+ }
+
+ private void handle(SegmentNotFoundException e) {
+ if (notFoundSegments.add(e.getSegmentId())) {
+ e.printStackTrace(err);
+ }
+ }
+
+}
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/RecoverJournal.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/SearchNodes.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/SearchNodes.java?rev=1847110&r1=1847109&r2=1847110&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/SearchNodes.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/SearchNodes.java Wed Nov 21 14:42:49 2018
@@ -21,6 +21,7 @@ package org.apache.jackrabbit.oak.segmen
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
+import static org.apache.jackrabbit.oak.segment.tool.Utils.parseSegmentInfoTimestamp;
import java.io.File;
import java.io.PrintStream;
@@ -31,8 +32,6 @@ import java.util.Set;
import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.api.Type;
-import org.apache.jackrabbit.oak.commons.json.JsonObject;
-import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
import org.apache.jackrabbit.oak.segment.RecordId;
import org.apache.jackrabbit.oak.segment.RecordType;
import org.apache.jackrabbit.oak.segment.SegmentId;
@@ -170,37 +169,6 @@ public class SearchNodes {
return FileStoreBuilder.fileStoreBuilder(path).buildReadOnly();
}
- private Long parseSegmentInfoTimestamp(SegmentId segmentId) {
- String segmentInfo = segmentId.getSegment().getSegmentInfo();
-
- if (segmentInfo == null) {
- err.printf("Segment info not found in %s\n", segmentId);
- return null;
- }
-
- JsopTokenizer t = new JsopTokenizer(segmentInfo, 0);
- t.read('{');
- JsonObject object = JsonObject.create(t);
-
- String timestampString = object.getProperties().get("t");
-
- if (timestampString == null) {
- err.printf("No timestamp found in segment info in %s\n", segmentId);
- return null;
- }
-
- long timestamp;
-
- try {
- timestamp = Long.parseLong(timestampString);
- } catch (NumberFormatException e) {
- err.printf("Invalid timestamp %s found in segment info in %s\n", timestampString, segmentId);
- return null;
- }
-
- return timestamp;
- }
-
private void processSegment(ReadOnlyFileStore fileStore, SegmentId segmentId) {
if (segmentId.isBulkSegmentId()) {
return;
@@ -209,6 +177,7 @@ public class SearchNodes {
Long timestamp = parseSegmentInfoTimestamp(segmentId);
if (timestamp == null) {
+ err.printf("No timestamp found in segment %s\n", segmentId);
return;
}
Modified: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java?rev=1847110&r1=1847109&r2=1847110&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java (original)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/tool/Utils.java Wed Nov 21 14:42:49 2018
@@ -28,13 +28,16 @@ import java.util.List;
import com.google.common.base.Function;
import com.google.common.collect.Iterators;
-import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
+import org.apache.jackrabbit.oak.commons.json.JsonObject;
+import org.apache.jackrabbit.oak.commons.json.JsopTokenizer;
+import org.apache.jackrabbit.oak.segment.SegmentId;
import org.apache.jackrabbit.oak.segment.file.InvalidFileStoreVersionException;
import org.apache.jackrabbit.oak.segment.file.JournalEntry;
import org.apache.jackrabbit.oak.segment.file.JournalReader;
import org.apache.jackrabbit.oak.segment.file.ReadOnlyFileStore;
import org.apache.jackrabbit.oak.segment.file.tar.LocalJournalFile;
import org.apache.jackrabbit.oak.segment.file.tooling.BasicReadOnlyBlobStore;
+import org.apache.jackrabbit.oak.segment.spi.persistence.JournalFile;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
import org.jetbrains.annotations.NotNull;
@@ -114,4 +117,32 @@ final class Utils {
return false;
}
+ static Long parseSegmentInfoTimestamp(SegmentId segmentId) {
+ String segmentInfo = segmentId.getSegment().getSegmentInfo();
+
+ if (segmentInfo == null) {
+ return null;
+ }
+
+ JsopTokenizer t = new JsopTokenizer(segmentInfo, 0);
+ t.read('{');
+ JsonObject object = JsonObject.create(t);
+
+ String timestampString = object.getProperties().get("t");
+
+ if (timestampString == null) {
+ return null;
+ }
+
+ long timestamp;
+
+ try {
+ timestamp = Long.parseLong(timestampString);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+
+ return timestamp;
+ }
+
}