You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by fr...@apache.org on 2016/05/23 08:58:03 UTC
svn commit: r1745129 - in /jackrabbit/oak/trunk:
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/
Author: frm
Date: Mon May 23 08:58:03 2016
New Revision: 1745129
URL: http://svn.apache.org/viewvc?rev=1745129&view=rev
Log:
OAK-4330 - Add a flag to choose between segment store implementations in the "debug" command
Added:
jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java (with props)
Modified:
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DebugCommand.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DebugCommand.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DebugCommand.java?rev=1745129&r1=1745128&r2=1745129&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DebugCommand.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/DebugCommand.java Mon May 23 08:58:03 2016
@@ -17,248 +17,30 @@
package org.apache.jackrabbit.oak.run;
-import static com.google.common.collect.Sets.newHashSet;
-import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
-import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openReadOnlyFileStore;
-import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Queue;
-import java.util.Set;
-import java.util.UUID;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.Logger;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Queues;
-import org.apache.jackrabbit.oak.commons.PathUtils;
-import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
-import org.apache.jackrabbit.oak.explorer.NodeStoreTree;
-import org.apache.jackrabbit.oak.json.JsopDiff;
-import org.apache.jackrabbit.oak.plugins.segment.PCMAnalyser;
-import org.apache.jackrabbit.oak.plugins.segment.RecordId;
-import org.apache.jackrabbit.oak.plugins.segment.RecordUsageAnalyser;
-import org.apache.jackrabbit.oak.plugins.segment.Segment;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
-import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
-import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
-import org.apache.jackrabbit.oak.spi.state.NodeState;
+import joptsimple.OptionParser;
+import joptsimple.OptionSet;
+import joptsimple.OptionSpec;
class DebugCommand implements Command {
- private static void debugFileStore(FileStore store) {
- Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
- int dataCount = 0;
- long dataSize = 0;
- int bulkCount = 0;
- long bulkSize = 0;
-
- ((Logger) getLogger(SegmentTracker.class)).setLevel(Level.OFF);
- RecordUsageAnalyser analyser = new RecordUsageAnalyser();
-
- for (SegmentId id : store.getSegmentIds()) {
- if (id.isDataSegmentId()) {
- Segment segment = id.getSegment();
- dataCount++;
- dataSize += segment.size();
- idmap.put(id, segment.getReferencedIds());
- analyseSegment(segment, analyser);
- } else if (id.isBulkSegmentId()) {
- bulkCount++;
- bulkSize += id.getSegment().size();
- idmap.put(id, Collections.<SegmentId>emptyList());
- }
- }
- System.out.println("Total size:");
- System.out.format(
- "%s in %6d data segments%n",
- byteCountToDisplaySize(dataSize), dataCount);
- System.out.format(
- "%s in %6d bulk segments%n",
- byteCountToDisplaySize(bulkSize), bulkCount);
- System.out.println(analyser.toString());
-
- Set<SegmentId> garbage = newHashSet(idmap.keySet());
- Queue<SegmentId> queue = Queues.newArrayDeque();
- queue.add(store.getHead().getRecordId().getSegmentId());
- while (!queue.isEmpty()) {
- SegmentId id = queue.remove();
- if (garbage.remove(id)) {
- queue.addAll(idmap.get(id));
- }
- }
- dataCount = 0;
- dataSize = 0;
- bulkCount = 0;
- bulkSize = 0;
- for (SegmentId id : garbage) {
- if (id.isDataSegmentId()) {
- dataCount++;
- dataSize += id.getSegment().size();
- } else if (id.isBulkSegmentId()) {
- bulkCount++;
- bulkSize += id.getSegment().size();
- }
- }
- System.out.format("%nAvailable for garbage collection:%n");
- System.out.format("%s in %6d data segments%n",
- byteCountToDisplaySize(dataSize), dataCount);
- System.out.format("%s in %6d bulk segments%n",
- byteCountToDisplaySize(bulkSize), bulkCount);
- System.out.format("%n%s", new PCMAnalyser(store).toString());
- }
-
- private static void analyseSegment(Segment segment, RecordUsageAnalyser analyser) {
- for (int k = 0; k < segment.getRootCount(); k++) {
- if (segment.getRootType(k) == NODE) {
- RecordId nodeId = new RecordId(segment.getSegmentId(), segment.getRootOffset(k));
- try {
- analyser.analyseNode(nodeId);
- } catch (Exception e) {
- System.err.format("Error while processing node at %s", nodeId);
- e.printStackTrace();
- }
- }
- }
- }
-
- private static void debugTarFile(FileStore store, String[] args) {
- File root = new File(args[0]);
- for (int i = 1; i < args.length; i++) {
- String f = args[i];
- if (!f.endsWith(".tar")) {
- System.out.println("skipping " + f);
- continue;
- }
- File tar = new File(root, f);
- if (!tar.exists()) {
- System.out.println("file doesn't exist, skipping " + f);
- continue;
- }
- System.out.println("Debug file " + tar + "(" + tar.length() + ")");
- Set<UUID> uuids = new HashSet<UUID>();
- boolean hasrefs = false;
- for (Map.Entry<String, Set<UUID>> e : store.getTarReaderIndex()
- .entrySet()) {
- if (e.getKey().endsWith(f)) {
- hasrefs = true;
- uuids = e.getValue();
- }
- }
- if (hasrefs) {
- System.out.println("SegmentNodeState references to " + f);
- List<String> paths = new ArrayList<String>();
- NodeStoreTree.filterNodeStates(uuids, paths, store.getHead(),
- "/");
- for (String p : paths) {
- System.out.println(" " + p);
- }
- } else {
- System.out.println("No references to " + f);
- }
-
- try {
- Map<UUID, List<UUID>> graph = store.getTarGraph(f);
- System.out.println();
- System.out.println("Tar graph:");
- for (Map.Entry<UUID, List<UUID>> entry : graph.entrySet()) {
- System.out.println("" + entry.getKey() + '=' + entry.getValue());
- }
- } catch (IOException e) {
- System.out.println("Error getting tar graph:");
- }
-
- }
- }
-
- private static void debugSegment(FileStore store, String[] args) {
- Pattern pattern = Pattern
- .compile("([0-9a-f-]+)|(([0-9a-f-]+:[0-9a-f]+)(-([0-9a-f-]+:[0-9a-f]+))?)?(/.*)?");
- for (int i = 1; i < args.length; i++) {
- Matcher matcher = pattern.matcher(args[i]);
- if (!matcher.matches()) {
- System.err.println("Unknown argument: " + args[i]);
- } else if (matcher.group(1) != null) {
- UUID uuid = UUID.fromString(matcher.group(1));
- SegmentId id = store.getTracker().getSegmentId(
- uuid.getMostSignificantBits(),
- uuid.getLeastSignificantBits());
- System.out.println(id.getSegment());
- } else {
- RecordId id1 = store.getHead().getRecordId();
- RecordId id2 = null;
- if (matcher.group(2) != null) {
- id1 = RecordId.fromString(store.getTracker(),
- matcher.group(3));
- if (matcher.group(4) != null) {
- id2 = RecordId.fromString(store.getTracker(),
- matcher.group(5));
- }
- }
- String path = "/";
- if (matcher.group(6) != null) {
- path = matcher.group(6);
- }
-
- if (id2 == null) {
- NodeState node = new SegmentNodeState(id1);
- System.out.println("/ (" + id1 + ") -> " + node);
- for (String name : PathUtils.elements(path)) {
- node = node.getChildNode(name);
- RecordId nid = null;
- if (node instanceof SegmentNodeState) {
- nid = ((SegmentNodeState) node).getRecordId();
- }
- System.out.println(" " + name + " (" + nid + ") -> "
- + node);
- }
- } else {
- NodeState node1 = new SegmentNodeState(id1);
- NodeState node2 = new SegmentNodeState(id2);
- for (String name : PathUtils.elements(path)) {
- node1 = node1.getChildNode(name);
- node2 = node2.getChildNode(name);
- }
- System.out.println(JsopBuilder.prettyPrint(JsopDiff
- .diffToJsop(node1, node2)));
- }
- }
- }
- }
-
@Override
public void execute(String... args) throws Exception {
- if (args.length == 0) {
+ OptionParser parser = new OptionParser();
+ OptionSpec segmentTar = parser.accepts("segment-tar", "Use oak-segment-tar instead of oak-segment");
+ OptionSpec<String> nonOptions = parser.nonOptions().ofType(String.class);
+ OptionSet options = parser.parse(args);
+
+ if (options.valuesOf(nonOptions).isEmpty()) {
System.err.println("usage: debug <path> [id...]");
System.exit(1);
+ }
+
+ String[] nonOptionsArray = options.valuesOf(nonOptions).toArray(new String[0]);
+
+ if (options.has(segmentTar)) {
+ SegmentTarUtils.debug(nonOptionsArray);
} else {
- // TODO: enable debug information for other node store implementations
- File file = new File(args[0]);
- System.out.println("Debug " + file);
- FileStore store = openReadOnlyFileStore(file);
- try {
- if (args.length == 1) {
- debugFileStore(store);
- } else {
- if (args[1].endsWith(".tar")) {
- debugTarFile(store, args);
- } else {
- debugSegment(store, args);
- }
- }
- } finally {
- store.close();
- }
+ SegmentUtils.debug(nonOptionsArray);
}
}
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java?rev=1745129&r1=1745128&r2=1745129&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentTarUtils.java Mon May 23 08:58:03 2016
@@ -17,19 +17,58 @@
package org.apache.jackrabbit.oak.run;
+import static com.google.common.collect.Sets.newHashSet;
+import static com.google.common.collect.Sets.newTreeSet;
+import static com.google.common.escape.Escapers.builder;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.isValidFileStoreOrFail;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore;
+import static org.apache.jackrabbit.oak.segment.RecordType.NODE;
+import static org.apache.jackrabbit.oak.segment.SegmentNodeStateHelper.getTemplateId;
+import static org.slf4j.LoggerFactory.getLogger;
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import javax.jcr.PropertyType;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.api.Blob;
+import org.apache.jackrabbit.oak.api.PropertyState;
+import org.apache.jackrabbit.oak.api.Type;
import org.apache.jackrabbit.oak.backup.FileStoreBackup;
import org.apache.jackrabbit.oak.backup.FileStoreRestore;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.json.JsopDiff;
+import org.apache.jackrabbit.oak.segment.RecordId;
+import org.apache.jackrabbit.oak.segment.RecordUsageAnalyser;
+import org.apache.jackrabbit.oak.segment.Segment;
+import org.apache.jackrabbit.oak.segment.SegmentBlob;
+import org.apache.jackrabbit.oak.segment.SegmentId;
+import org.apache.jackrabbit.oak.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.segment.SegmentPropertyState;
+import org.apache.jackrabbit.oak.segment.SegmentTracker;
import org.apache.jackrabbit.oak.segment.file.FileStore;
import org.apache.jackrabbit.oak.spi.blob.BlobStore;
+import org.apache.jackrabbit.oak.spi.state.ChildNodeEntry;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
class SegmentTarUtils {
@@ -38,6 +77,8 @@ class SegmentTarUtils {
private static final int TAR_SEGMENT_CACHE_SIZE = Integer.getInteger("cache", 256);
+ private final static int MAX_CHAR_DISPLAY = Integer.getInteger("max.char.display", 60);
+
private SegmentTarUtils() {
// Prevent instantiation
}
@@ -65,6 +106,274 @@ class SegmentTarUtils {
FileStoreRestore.restore(source, target);
}
+ static void debug(String... args) throws IOException {
+ File file = new File(args[0]);
+ System.out.println("Debug " + file);
+ FileStore store = openReadOnlyFileStore(file);
+ try {
+ if (args.length == 1) {
+ debugFileStore(store);
+ } else {
+ if (args[1].endsWith(".tar")) {
+ debugTarFile(store, args);
+ } else {
+ debugSegment(store, args);
+ }
+ }
+ } finally {
+ store.close();
+ }
+ }
+
+ private static void debugFileStore(FileStore store) {
+ Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
+ int dataCount = 0;
+ long dataSize = 0;
+ int bulkCount = 0;
+ long bulkSize = 0;
+
+ ((Logger) getLogger(SegmentTracker.class)).setLevel(Level.OFF);
+ RecordUsageAnalyser analyser = new RecordUsageAnalyser();
+
+ for (SegmentId id : store.getSegmentIds()) {
+ if (id.isDataSegmentId()) {
+ Segment segment = id.getSegment();
+ dataCount++;
+ dataSize += segment.size();
+ idmap.put(id, segment.getReferencedIds());
+ analyseSegment(segment, analyser);
+ } else if (id.isBulkSegmentId()) {
+ bulkCount++;
+ bulkSize += id.getSegment().size();
+ idmap.put(id, Collections.<SegmentId>emptyList());
+ }
+ }
+ System.out.println("Total size:");
+ System.out.format(
+ "%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
+ System.out.format(
+ "%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ System.out.println(analyser.toString());
+
+ Set<SegmentId> garbage = newHashSet(idmap.keySet());
+ Queue<SegmentId> queue = Queues.newArrayDeque();
+ queue.add(store.getHead().getRecordId().getSegmentId());
+ while (!queue.isEmpty()) {
+ SegmentId id = queue.remove();
+ if (garbage.remove(id)) {
+ queue.addAll(idmap.get(id));
+ }
+ }
+ dataCount = 0;
+ dataSize = 0;
+ bulkCount = 0;
+ bulkSize = 0;
+ for (SegmentId id : garbage) {
+ if (id.isDataSegmentId()) {
+ dataCount++;
+ dataSize += id.getSegment().size();
+ } else if (id.isBulkSegmentId()) {
+ bulkCount++;
+ bulkSize += id.getSegment().size();
+ }
+ }
+ System.out.format("%nAvailable for garbage collection:%n");
+ System.out.format("%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
+ System.out.format("%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ }
+
+ private static void analyseSegment(Segment segment, RecordUsageAnalyser analyser) {
+ for (int k = 0; k < segment.getRootCount(); k++) {
+ if (segment.getRootType(k) == NODE) {
+ RecordId nodeId = new RecordId(segment.getSegmentId(), segment.getRootOffset(k));
+ try {
+ analyser.analyseNode(nodeId);
+ } catch (Exception e) {
+ System.err.format("Error while processing node at %s", nodeId);
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private static void debugTarFile(FileStore store, String[] args) {
+ File root = new File(args[0]);
+ for (int i = 1; i < args.length; i++) {
+ String f = args[i];
+ if (!f.endsWith(".tar")) {
+ System.out.println("skipping " + f);
+ continue;
+ }
+ File tar = new File(root, f);
+ if (!tar.exists()) {
+ System.out.println("file doesn't exist, skipping " + f);
+ continue;
+ }
+ System.out.println("Debug file " + tar + "(" + tar.length() + ")");
+ Set<UUID> uuids = new HashSet<UUID>();
+ boolean hasrefs = false;
+ for (Map.Entry<String, Set<UUID>> e : store.getTarReaderIndex()
+ .entrySet()) {
+ if (e.getKey().endsWith(f)) {
+ hasrefs = true;
+ uuids = e.getValue();
+ }
+ }
+ if (hasrefs) {
+ System.out.println("SegmentNodeState references to " + f);
+ List<String> paths = new ArrayList<String>();
+ filterNodeStates(uuids, paths, store.getHead(),
+ "/");
+ for (String p : paths) {
+ System.out.println(" " + p);
+ }
+ } else {
+ System.out.println("No references to " + f);
+ }
+
+ try {
+ Map<UUID, List<UUID>> graph = store.getTarGraph(f);
+ System.out.println();
+ System.out.println("Tar graph:");
+ for (Map.Entry<UUID, List<UUID>> entry : graph.entrySet()) {
+ System.out.println("" + entry.getKey() + '=' + entry.getValue());
+ }
+ } catch (IOException e) {
+ System.out.println("Error getting tar graph:");
+ }
+
+ }
+ }
+
+ private static void filterNodeStates(Set<UUID> uuids, List<String> paths, SegmentNodeState state, String path) {
+ Set<String> localPaths = newTreeSet();
+ for (PropertyState ps : state.getProperties()) {
+ if (ps instanceof SegmentPropertyState) {
+ SegmentPropertyState sps = (SegmentPropertyState) ps;
+ RecordId recordId = sps.getRecordId();
+ UUID id = recordId.getSegmentId().asUUID();
+ if (uuids.contains(id)) {
+ if (ps.getType().tag() == PropertyType.STRING) {
+ String val = "";
+ if (ps.count() > 0) {
+ // only shows the first value, do we need more?
+ val = displayString(ps.getValue(Type.STRING, 0));
+ }
+ localPaths.add(path + ps.getName() + " = " + val
+ + " [SegmentPropertyState<" + ps.getType()
+ + ">@" + recordId + "]");
+ } else {
+ localPaths.add(path + ps + " [SegmentPropertyState<"
+ + ps.getType() + ">@" + recordId + "]");
+ }
+
+ }
+ if (ps.getType().tag() == PropertyType.BINARY) {
+ // look for extra segment references
+ for (int i = 0; i < ps.count(); i++) {
+ Blob b = ps.getValue(Type.BINARY, i);
+ for (SegmentId sbid : SegmentBlob.getBulkSegmentIds(b)) {
+ UUID bid = sbid.asUUID();
+ if (!bid.equals(id) && uuids.contains(bid)) {
+ localPaths.add(path + ps
+ + " [SegmentPropertyState<"
+ + ps.getType() + ">@" + recordId + "]");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ RecordId stateId = state.getRecordId();
+ if (uuids.contains(stateId.getSegmentId().asUUID())) {
+ localPaths.add(path + " [SegmentNodeState@" + stateId + "]");
+ }
+
+ RecordId templateId = getTemplateId(state);
+ if (uuids.contains(templateId.getSegmentId().asUUID())) {
+ localPaths.add(path + "[Template@" + templateId + "]");
+ }
+ paths.addAll(localPaths);
+ for (ChildNodeEntry ce : state.getChildNodeEntries()) {
+ NodeState c = ce.getNodeState();
+ if (c instanceof SegmentNodeState) {
+ filterNodeStates(uuids, paths, (SegmentNodeState) c,
+ path + ce.getName() + "/");
+ }
+ }
+ }
+
+ private static String displayString(String value) {
+ if (MAX_CHAR_DISPLAY > 0 && value.length() > MAX_CHAR_DISPLAY) {
+ value = value.substring(0, MAX_CHAR_DISPLAY) + "... ("
+ + value.length() + " chars)";
+ }
+ String escaped = builder().setSafeRange(' ', '~')
+ .addEscape('"', "\\\"").addEscape('\\', "\\\\").build()
+ .escape(value);
+ return '"' + escaped + '"';
+ }
+
+ private static void debugSegment(FileStore store, String[] args) {
+ Pattern pattern = Pattern
+ .compile("([0-9a-f-]+)|(([0-9a-f-]+:[0-9a-f]+)(-([0-9a-f-]+:[0-9a-f]+))?)?(/.*)?");
+ for (int i = 1; i < args.length; i++) {
+ Matcher matcher = pattern.matcher(args[i]);
+ if (!matcher.matches()) {
+ System.err.println("Unknown argument: " + args[i]);
+ } else if (matcher.group(1) != null) {
+ UUID uuid = UUID.fromString(matcher.group(1));
+ SegmentId id = store.getTracker().getSegmentId(
+ uuid.getMostSignificantBits(),
+ uuid.getLeastSignificantBits());
+ System.out.println(id.getSegment());
+ } else {
+ RecordId id1 = store.getHead().getRecordId();
+ RecordId id2 = null;
+ if (matcher.group(2) != null) {
+ id1 = RecordId.fromString(store.getTracker(),
+ matcher.group(3));
+ if (matcher.group(4) != null) {
+ id2 = RecordId.fromString(store.getTracker(),
+ matcher.group(5));
+ }
+ }
+ String path = "/";
+ if (matcher.group(6) != null) {
+ path = matcher.group(6);
+ }
+
+ if (id2 == null) {
+ NodeState node = new SegmentNodeState(id1);
+ System.out.println("/ (" + id1 + ") -> " + node);
+ for (String name : PathUtils.elements(path)) {
+ node = node.getChildNode(name);
+ RecordId nid = null;
+ if (node instanceof SegmentNodeState) {
+ nid = ((SegmentNodeState) node).getRecordId();
+ }
+ System.out.println(" " + name + " (" + nid + ") -> "
+ + node);
+ }
+ } else {
+ NodeState node1 = new SegmentNodeState(id1);
+ NodeState node2 = new SegmentNodeState(id2);
+ for (String name : PathUtils.elements(path)) {
+ node1 = node1.getChildNode(name);
+ node2 = node2.getChildNode(name);
+ }
+ System.out.println(JsopBuilder.prettyPrint(JsopDiff
+ .diffToJsop(node1, node2)));
+ }
+ }
+ }
+ }
+
private static FileStore openReadOnlyFileStore(File path, BlobStore blobStore) throws IOException {
return FileStore.builder(isValidFileStoreOrFail(path))
.withCacheSize(TAR_SEGMENT_CACHE_SIZE)
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java?rev=1745129&r1=1745128&r2=1745129&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/SegmentUtils.java Mon May 23 08:58:03 2016
@@ -17,18 +17,48 @@
package org.apache.jackrabbit.oak.run;
+import static com.google.common.collect.Sets.newHashSet;
+import static org.apache.commons.io.FileUtils.byteCountToDisplaySize;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.newBasicReadOnlyBlobStore;
import static org.apache.jackrabbit.oak.plugins.segment.FileStoreHelper.openReadOnlyFileStore;
+import static org.apache.jackrabbit.oak.plugins.segment.RecordType.NODE;
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
+import static org.slf4j.LoggerFactory.getLogger;
import java.io.File;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Queues;
import com.google.common.io.Closer;
+import org.apache.jackrabbit.oak.commons.PathUtils;
+import org.apache.jackrabbit.oak.commons.json.JsopBuilder;
+import org.apache.jackrabbit.oak.explorer.NodeStoreTree;
+import org.apache.jackrabbit.oak.json.JsopDiff;
import org.apache.jackrabbit.oak.plugins.backup.FileStoreBackup;
import org.apache.jackrabbit.oak.plugins.backup.FileStoreRestore;
+import org.apache.jackrabbit.oak.plugins.segment.PCMAnalyser;
+import org.apache.jackrabbit.oak.plugins.segment.RecordId;
+import org.apache.jackrabbit.oak.plugins.segment.RecordUsageAnalyser;
+import org.apache.jackrabbit.oak.plugins.segment.Segment;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentId;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeState;
import org.apache.jackrabbit.oak.plugins.segment.SegmentNodeStore;
+import org.apache.jackrabbit.oak.plugins.segment.SegmentTracker;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
+import org.apache.jackrabbit.oak.spi.state.NodeState;
import org.apache.jackrabbit.oak.spi.state.NodeStore;
class SegmentUtils {
@@ -60,4 +90,203 @@ class SegmentUtils {
FileStoreRestore.restore(source, target);
}
+ static void debug(String... args) throws IOException {
+ File file = new File(args[0]);
+ System.out.println("Debug " + file);
+ FileStore store = openReadOnlyFileStore(file);
+ try {
+ if (args.length == 1) {
+ debugFileStore(store);
+ } else {
+ if (args[1].endsWith(".tar")) {
+ debugTarFile(store, args);
+ } else {
+ debugSegment(store, args);
+ }
+ }
+ } finally {
+ store.close();
+ }
+ }
+
+ private static void debugFileStore(FileStore store) {
+ Map<SegmentId, List<SegmentId>> idmap = Maps.newHashMap();
+ int dataCount = 0;
+ long dataSize = 0;
+ int bulkCount = 0;
+ long bulkSize = 0;
+
+ ((Logger) getLogger(SegmentTracker.class)).setLevel(Level.OFF);
+ RecordUsageAnalyser analyser = new RecordUsageAnalyser();
+
+ for (SegmentId id : store.getSegmentIds()) {
+ if (id.isDataSegmentId()) {
+ Segment segment = id.getSegment();
+ dataCount++;
+ dataSize += segment.size();
+ idmap.put(id, segment.getReferencedIds());
+ analyseSegment(segment, analyser);
+ } else if (id.isBulkSegmentId()) {
+ bulkCount++;
+ bulkSize += id.getSegment().size();
+ idmap.put(id, Collections.<SegmentId>emptyList());
+ }
+ }
+ System.out.println("Total size:");
+ System.out.format(
+ "%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
+ System.out.format(
+ "%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ System.out.println(analyser.toString());
+
+ Set<SegmentId> garbage = newHashSet(idmap.keySet());
+ Queue<SegmentId> queue = Queues.newArrayDeque();
+ queue.add(store.getHead().getRecordId().getSegmentId());
+ while (!queue.isEmpty()) {
+ SegmentId id = queue.remove();
+ if (garbage.remove(id)) {
+ queue.addAll(idmap.get(id));
+ }
+ }
+ dataCount = 0;
+ dataSize = 0;
+ bulkCount = 0;
+ bulkSize = 0;
+ for (SegmentId id : garbage) {
+ if (id.isDataSegmentId()) {
+ dataCount++;
+ dataSize += id.getSegment().size();
+ } else if (id.isBulkSegmentId()) {
+ bulkCount++;
+ bulkSize += id.getSegment().size();
+ }
+ }
+ System.out.format("%nAvailable for garbage collection:%n");
+ System.out.format("%s in %6d data segments%n",
+ byteCountToDisplaySize(dataSize), dataCount);
+ System.out.format("%s in %6d bulk segments%n",
+ byteCountToDisplaySize(bulkSize), bulkCount);
+ System.out.format("%n%s", new PCMAnalyser(store).toString());
+ }
+
+ private static void analyseSegment(Segment segment, RecordUsageAnalyser analyser) {
+ for (int k = 0; k < segment.getRootCount(); k++) {
+ if (segment.getRootType(k) == NODE) {
+ RecordId nodeId = new RecordId(segment.getSegmentId(), segment.getRootOffset(k));
+ try {
+ analyser.analyseNode(nodeId);
+ } catch (Exception e) {
+ System.err.format("Error while processing node at %s", nodeId);
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+
+ private static void debugTarFile(FileStore store, String[] args) {
+ File root = new File(args[0]);
+ for (int i = 1; i < args.length; i++) {
+ String f = args[i];
+ if (!f.endsWith(".tar")) {
+ System.out.println("skipping " + f);
+ continue;
+ }
+ File tar = new File(root, f);
+ if (!tar.exists()) {
+ System.out.println("file doesn't exist, skipping " + f);
+ continue;
+ }
+ System.out.println("Debug file " + tar + "(" + tar.length() + ")");
+ Set<UUID> uuids = new HashSet<UUID>();
+ boolean hasrefs = false;
+ for (Map.Entry<String, Set<UUID>> e : store.getTarReaderIndex()
+ .entrySet()) {
+ if (e.getKey().endsWith(f)) {
+ hasrefs = true;
+ uuids = e.getValue();
+ }
+ }
+ if (hasrefs) {
+ System.out.println("SegmentNodeState references to " + f);
+ List<String> paths = new ArrayList<String>();
+ NodeStoreTree.filterNodeStates(uuids, paths, store.getHead(),
+ "/");
+ for (String p : paths) {
+ System.out.println(" " + p);
+ }
+ } else {
+ System.out.println("No references to " + f);
+ }
+
+ try {
+ Map<UUID, List<UUID>> graph = store.getTarGraph(f);
+ System.out.println();
+ System.out.println("Tar graph:");
+ for (Map.Entry<UUID, List<UUID>> entry : graph.entrySet()) {
+ System.out.println("" + entry.getKey() + '=' + entry.getValue());
+ }
+ } catch (IOException e) {
+ System.out.println("Error getting tar graph:");
+ }
+
+ }
+ }
+
+ private static void debugSegment(FileStore store, String[] args) {
+ Pattern pattern = Pattern
+ .compile("([0-9a-f-]+)|(([0-9a-f-]+:[0-9a-f]+)(-([0-9a-f-]+:[0-9a-f]+))?)?(/.*)?");
+ for (int i = 1; i < args.length; i++) {
+ Matcher matcher = pattern.matcher(args[i]);
+ if (!matcher.matches()) {
+ System.err.println("Unknown argument: " + args[i]);
+ } else if (matcher.group(1) != null) {
+ UUID uuid = UUID.fromString(matcher.group(1));
+ SegmentId id = store.getTracker().getSegmentId(
+ uuid.getMostSignificantBits(),
+ uuid.getLeastSignificantBits());
+ System.out.println(id.getSegment());
+ } else {
+ RecordId id1 = store.getHead().getRecordId();
+ RecordId id2 = null;
+ if (matcher.group(2) != null) {
+ id1 = RecordId.fromString(store.getTracker(),
+ matcher.group(3));
+ if (matcher.group(4) != null) {
+ id2 = RecordId.fromString(store.getTracker(),
+ matcher.group(5));
+ }
+ }
+ String path = "/";
+ if (matcher.group(6) != null) {
+ path = matcher.group(6);
+ }
+
+ if (id2 == null) {
+ NodeState node = new SegmentNodeState(id1);
+ System.out.println("/ (" + id1 + ") -> " + node);
+ for (String name : PathUtils.elements(path)) {
+ node = node.getChildNode(name);
+ RecordId nid = null;
+ if (node instanceof SegmentNodeState) {
+ nid = ((SegmentNodeState) node).getRecordId();
+ }
+ System.out.println(" " + name + " (" + nid + ") -> "
+ + node);
+ }
+ } else {
+ NodeState node1 = new SegmentNodeState(id1);
+ NodeState node2 = new SegmentNodeState(id2);
+ for (String name : PathUtils.elements(path)) {
+ node1 = node1.getChildNode(name);
+ node2 = node2.getChildNode(name);
+ }
+ System.out.println(JsopBuilder.prettyPrint(JsopDiff
+ .diffToJsop(node1, node2)));
+ }
+ }
+ }
+ }
+
}
Added: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java?rev=1745129&view=auto
==============================================================================
--- jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java (added)
+++ jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java Mon May 23 08:58:03 2016
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.jackrabbit.oak.segment;
+
+public class SegmentNodeStateHelper {
+
+ private SegmentNodeStateHelper() {
+ // Prevent instantiation
+ }
+
+ public static RecordId getTemplateId(SegmentNodeState s) {
+ return s.getTemplateId();
+ }
+
+}
\ No newline at end of file
Propchange: jackrabbit/oak/trunk/oak-segment-tar/src/main/java/org/apache/jackrabbit/oak/segment/SegmentNodeStateHelper.java
------------------------------------------------------------------------------
svn:eol-style = native