You are viewing a plain text version of this content. The canonical link for it is here.
Posted to oak-commits@jackrabbit.apache.org by md...@apache.org on 2016/01/19 16:47:39 UTC
svn commit: r1725552 - in /jackrabbit/oak/trunk:
oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/
oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/ oak-run/
oak-run/src/main/java/org/apache/jackrabbit/oak/run/
Author: mduerig
Date: Tue Jan 19 15:47:39 2016
New Revision: 1725552
URL: http://svn.apache.org/viewvc?rev=1725552&view=rev
Log:
OAK-3898: Add filter capabilities to the segment graph run mode
RegExp filter on the segment info can be specified via --pattern
Modified:
jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
jackrabbit/oak/trunk/oak-run/README.md
jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
Modified: jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java?rev=1725552&r1=1725551&r2=1725552&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraph.java Tue Jan 19 15:47:39 2016
@@ -24,6 +24,7 @@ import static com.google.common.base.Pre
import static com.google.common.collect.Maps.newHashMap;
import static com.google.common.collect.Sets.newHashSet;
import static java.util.Collections.singletonMap;
+import static java.util.regex.Pattern.compile;
import static org.apache.jackrabbit.oak.commons.IOUtils.closeQuietly;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentId.isDataSegmentId;
@@ -39,6 +40,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.UUID;
+import java.util.regex.Pattern;
import javax.annotation.CheckForNull;
import javax.annotation.Nonnull;
@@ -46,6 +48,8 @@ import javax.annotation.Nullable;
import com.google.common.base.Function;
import com.google.common.base.Functions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import com.google.common.collect.HashMultiset;
import com.google.common.collect.Multiset;
import org.apache.jackrabbit.oak.api.Type;
@@ -143,19 +147,24 @@ public final class SegmentGraph {
* @param fileStore file store to graph
* @param out stream to write the graph to
* @param epoch epoch (in milliseconds)
+ * @param pattern regular expression specifying inclusion of nodes or {@code null}
+ * for all nodes.
* @throws Exception
*/
public static void writeSegmentGraph(
@Nonnull ReadOnlyStore fileStore,
@Nonnull OutputStream out,
- @Nonnull Date epoch) throws Exception {
-
+ @Nonnull Date epoch,
+ @CheckForNull String pattern) throws Exception {
checkNotNull(epoch);
PrintWriter writer = new PrintWriter(checkNotNull(out));
try {
SegmentNodeState root = checkNotNull(fileStore).getHead();
- Graph<UUID> segmentGraph = parseSegmentGraph(fileStore);
+ Predicate<UUID> filter = pattern == null
+ ? Predicates.<UUID>alwaysTrue()
+ : createRegExpFilter(pattern, fileStore.getTracker());
+ Graph<UUID> segmentGraph = parseSegmentGraph(fileStore, filter);
Graph<UUID> headGraph = parseHeadGraph(root.getRecordId());
writer.write("nodedef>name VARCHAR, label VARCHAR, type VARCHAR, wid VARCHAR, gc INT, t INT, head BOOLEAN\n");
@@ -180,18 +189,52 @@ public final class SegmentGraph {
}
/**
+ * Create a regular expression based inclusion filter for segment.
+ *
+ * @param pattern regular expression specifying inclusion of nodes.
+ * @param tracker the segment tracker of the store acting upon.
+ * @return
+ */
+ public static Predicate<UUID> createRegExpFilter(
+ @Nonnull String pattern,
+ @Nonnull final SegmentTracker tracker) {
+ final Pattern regExp = compile(checkNotNull(pattern));
+ checkNotNull(tracker);
+
+ return new Predicate<UUID>() {
+ @Override
+ public boolean apply(UUID segment) {
+ try {
+ String info = getSegmentInfo(segment, tracker);
+ if (info == null) {
+ info = "NULL";
+ }
+ return regExp.matcher(info).matches();
+ } catch (Exception e) {
+ System.err.println("Error accessing segment " + segment + ": " + e);
+ return false;
+ }
+ }
+ };
+ }
+
+ /**
* Parse the segment graph of a file store.
*
* @param fileStore file store to parse
+ * @param filter inclusion criteria for vertices and edges. An edge is only included if
+ * both its source and target vertex are included.
* @return the segment graph rooted as the segment containing the head node
* state of {@code fileStore}.
* @throws IOException
*/
@Nonnull
- public static Graph<UUID> parseSegmentGraph(@Nonnull ReadOnlyStore fileStore) throws IOException {
+ public static Graph<UUID> parseSegmentGraph(
+ @Nonnull ReadOnlyStore fileStore,
+ @Nonnull Predicate<UUID> filter) throws IOException {
SegmentNodeState root = checkNotNull(fileStore).getHead();
HashSet<UUID> roots = newHashSet(root.getRecordId().asUUID());
- return parseSegmentGraph(fileStore, roots, Functions.<UUID>identity());
+ return parseSegmentGraph(fileStore, roots, filter, Functions.<UUID>identity());
}
/**
@@ -244,10 +287,10 @@ public final class SegmentGraph {
throws IOException {
SegmentNodeState root = checkNotNull(fileStore).getHead();
HashSet<UUID> roots = newHashSet(root.getRecordId().asUUID());
- return parseSegmentGraph(fileStore, roots, new Function<UUID, String>() {
+ return parseSegmentGraph(fileStore, roots, Predicates.<UUID>alwaysTrue(), new Function<UUID, String>() {
@Override @Nullable
public String apply(UUID segmentId) {
- Map<String, String> info = getSegmentInfo(segmentId, fileStore.getTracker());
+ Map<String, String> info = getSegmentInfoMap(segmentId, fileStore.getTracker());
if (info != null) {
String error = info.get("error");
if (error != null) {
@@ -266,32 +309,44 @@ public final class SegmentGraph {
/**
* Parse the segment graph of a file store starting with a given set of root segments.
- * The full segment graph is mapped through the passed {@code homomorphism} to the
+ * The full segment graph is mapped through the passed {@code map} to the
* graph returned by this function.
*
* @param fileStore file store to parse
* @param roots the initial set of segments
- * @param homomorphism map from the segment graph into the returned graph
+ * @param map map defining an homomorphism from the segment graph into the returned graph
+ * @param filter inclusion criteria for vertices and edges. An edge is only included if
+ * both its source and target vertex are included.
* @return the segment graph of {@code fileStore} rooted at {@code roots} and mapped
- * by {@code homomorphism}
+ * by {@code map}
* @throws IOException
*/
@Nonnull
public static <T> Graph<T> parseSegmentGraph(
- @Nonnull ReadOnlyStore fileStore,
+ @Nonnull final ReadOnlyStore fileStore,
@Nonnull Set<UUID> roots,
- @Nonnull final Function<UUID, T> homomorphism) throws IOException {
+ @Nonnull final Predicate<UUID> filter,
+ @Nonnull final Function<UUID, T> map) throws IOException {
final Graph<T> graph = new Graph<T>();
- checkNotNull(homomorphism);
+ checkNotNull(filter);
+ checkNotNull(map);
checkNotNull(fileStore).traverseSegmentGraph(checkNotNull(roots),
new SegmentGraphVisitor() {
@Override
public void accept(@Nonnull UUID from, @CheckForNull UUID to) {
- graph.addVertex(homomorphism.apply(from));
- if (to != null) {
- graph.addVertex((homomorphism.apply(to)));
- graph.addEdge(homomorphism.apply(from), homomorphism.apply(to));
+ T fromT = null;
+ T toT = null;
+ if (filter.apply(from)) {
+ fromT = map.apply(from);
+ graph.addVertex(fromT);
+ }
+ if (to != null && filter.apply(to)) {
+ toT = map.apply(to);
+ graph.addVertex(toT);
+ }
+ if (fromT != null && toT != null) {
+ graph.addEdge(fromT, toT);
}
}
});
@@ -395,7 +450,7 @@ public final class SegmentGraph {
}
private static void writeNode(UUID node, PrintWriter writer, boolean inHead, Date epoch, SegmentTracker tracker) {
- Map<String, String> sInfo = getSegmentInfo(node, tracker);
+ Map<String, String> sInfo = getSegmentInfoMap(node, tracker);
if (sInfo == null) {
writer.write(node + ",b,bulk,b,-1,-1," + inHead + "\n");
} else {
@@ -437,21 +492,25 @@ public final class SegmentGraph {
return Long.valueOf(string);
}
- private static Map<String, String> getSegmentInfo(UUID node, SegmentTracker tracker) {
- if (isDataSegmentId(node.getLeastSignificantBits())) {
- SegmentId id = tracker.getSegmentId(node.getMostSignificantBits(), node.getLeastSignificantBits());
- try {
- String info = id.getSegment().getSegmentInfo();
- if (info != null) {
- JsopTokenizer tokenizer = new JsopTokenizer(info);
- tokenizer.read('{');
- return JsonObject.create(tokenizer).getProperties();
- } else {
- return null;
- }
- } catch (SegmentNotFoundException e) {
- return singletonMap("error", toString(e));
+ private static Map<String, String> getSegmentInfoMap(UUID segment, SegmentTracker tracker) {
+ try {
+ String info = getSegmentInfo(segment, tracker);
+ if (info != null) {
+ JsopTokenizer tokenizer = new JsopTokenizer(info);
+ tokenizer.read('{');
+ return JsonObject.create(tokenizer).getProperties();
+ } else {
+ return null;
}
+ } catch (SegmentNotFoundException e) {
+ return singletonMap("error", toString(e));
+ }
+ }
+
+ private static String getSegmentInfo(UUID segment, SegmentTracker tracker) {
+ if (isDataSegmentId(segment.getLeastSignificantBits())) {
+ SegmentId id = tracker.getSegmentId(segment.getMostSignificantBits(), segment.getLeastSignificantBits());
+ return id.getSegment().getSegmentInfo();
} else {
return null;
}
Modified: jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java?rev=1725552&r1=1725551&r2=1725552&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java (original)
+++ jackrabbit/oak/trunk/oak-core/src/test/java/org/apache/jackrabbit/oak/plugins/segment/SegmentGraphTest.java Tue Jan 19 15:47:39 2016
@@ -24,6 +24,7 @@ import static com.google.common.collect.
import static com.google.common.collect.Sets.newHashSet;
import static java.io.File.createTempFile;
import static java.util.Collections.singleton;
+import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.createRegExpFilter;
import static org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.parseSegmentGraph;
import static org.junit.Assert.assertEquals;
@@ -39,11 +40,12 @@ import java.util.UUID;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
+import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Multiset;
import org.apache.commons.io.IOUtils;
import org.apache.jackrabbit.oak.plugins.segment.SegmentGraph.Graph;
-import org.apache.jackrabbit.oak.plugins.segment.file.FileStore;
import org.apache.jackrabbit.oak.plugins.segment.file.FileStore.ReadOnlyStore;
import org.junit.After;
import org.junit.Before;
@@ -69,6 +71,20 @@ public class SegmentGraphTest {
UUID.fromString("ab61b8c9-222c-4119-a73b-5f61c0bc4741"))
);
+ private final Set<UUID> filteredSegments = newHashSet(
+ UUID.fromString("fdaca71e-f71e-4f19-abf5-144e8c85f9e3"),
+ UUID.fromString("2eae0bc2-d3dd-4ba4-a765-70c38073437d"),
+ UUID.fromString("ab61b8c9-222c-4119-a73b-5f61c0bc4741")
+ );
+
+ private final Map<UUID, Set<UUID>> filteredReferences = ImmutableMap.<UUID, Set<UUID>>of(
+ UUID.fromString("fdaca71e-f71e-4f19-abf5-144e8c85f9e3"),
+ newHashSet(UUID.fromString("ab61b8c9-222c-4119-a73b-5f61c0bc4741")),
+ UUID.fromString("2eae0bc2-d3dd-4ba4-a765-70c38073437d"),
+ newHashSet(UUID.fromString("2fdaca71e-f71e-4f19-abf5-144e8c85f9e3"),
+ UUID.fromString("ab61b8c9-222c-4119-a73b-5f61c0bc4741"))
+ );
+
private final Set<String> gcGenerations = newHashSet("0", "1");
private final Map<String, Set<String>> gcReferences = ImmutableMap.of(
"0", singleton("0"),
@@ -84,8 +100,6 @@ public class SegmentGraphTest {
storeDir.mkdir();
unzip(SegmentGraphTest.class.getResourceAsStream("file-store.zip"), storeDir);
-
- FileStore store = FileStore.newFileStore(storeDir).create();
}
@After
@@ -97,7 +111,7 @@ public class SegmentGraphTest {
public void testSegmentGraph() throws IOException {
ReadOnlyStore store = new ReadOnlyStore(storeDir);
try {
- Graph<UUID> segmentGraph = parseSegmentGraph(store);
+ Graph<UUID> segmentGraph = parseSegmentGraph(store, Predicates.<UUID>alwaysTrue());
assertEquals(segments, newHashSet(segmentGraph.vertices()));
Map<UUID, Set<UUID>> map = newHashMap();
for (Entry<UUID, Multiset<UUID>> entry : segmentGraph.edges()) {
@@ -107,6 +121,23 @@ public class SegmentGraphTest {
} finally {
store.close();
}
+ }
+
+ @Test
+ public void testSegmentGraphWithFilter() throws IOException {
+ ReadOnlyStore store = new ReadOnlyStore(storeDir);
+ try {
+ Predicate<UUID> filter = createRegExpFilter(".*testWriter.*", store.getTracker());
+ Graph<UUID> segmentGraph = parseSegmentGraph(store, filter);
+ assertEquals(filteredSegments, newHashSet(segmentGraph.vertices()));
+ Map<UUID, Set<UUID>> map = newHashMap();
+ for (Entry<UUID, Multiset<UUID>> entry : segmentGraph.edges()) {
+ map.put(entry.getKey(), entry.getValue().elementSet());
+ }
+ assertEquals(filteredReferences, map);
+ } finally {
+ store.close();
+ }
}
@Test
Modified: jackrabbit/oak/trunk/oak-run/README.md
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/README.md?rev=1725552&r1=1725551&r2=1725552&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/README.md (original)
+++ jackrabbit/oak/trunk/oak-run/README.md Tue Jan 19 15:47:39 2016
@@ -122,6 +122,9 @@ a negative offset translating all timest
given)
--output <File> Output file (default: segments.gdf)
--gc Write the gc generation graph instead of the full graph
+ --pattern Regular exception specifying which
+ nodes to include (optional). Ignore
+ when --gc is specified.
History
-------
Modified: jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java
URL: http://svn.apache.org/viewvc/jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java?rev=1725552&r1=1725551&r2=1725552&view=diff
==============================================================================
--- jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java (original)
+++ jackrabbit/oak/trunk/oak-run/src/main/java/org/apache/jackrabbit/oak/run/Main.java Tue Jan 19 15:47:39 2016
@@ -818,6 +818,10 @@ public final class Main {
.withRequiredArg().ofType(Long.class);
OptionSpec<Void> gcGraphArg = parser.accepts(
"gc", "Write the gc generation graph instead of the full graph");
+ OptionSpec<String> regExpArg = parser.accepts(
+ "pattern", "Regular exception specifying which nodes to include (optional). " +
+ "Ignore when --gc is specified.")
+ .withRequiredArg().ofType(String.class);
OptionSet options = parser.parse(args);
@@ -832,6 +836,8 @@ public final class Main {
System.exit(1);
}
+ String regExp = regExpArg.value(options);
+
File outFile = outFileArg.value(options);
Date epoch;
if (options.has(epochArg)) {
@@ -854,13 +860,13 @@ public final class Main {
}
System.out.println("Setting epoch to " + epoch);
- System.out.println("Writing graph to " + outFile);
+ System.out.println("Writing graph to " + outFile.getAbsolutePath());
FileOutputStream out = new FileOutputStream(outFile);
if (options.has(gcGraphArg)) {
writeGCGraph(fileStore, out);
} else {
- writeSegmentGraph(fileStore, out, epoch);
+ writeSegmentGraph(fileStore, out, epoch, regExp);
}
}