You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/02/06 20:54:39 UTC
[08/30] incubator-tinkerpop git commit: TinkerGraph now supports
GraphComputer GraphFilter. Sort of an elegant solution that makes use of
tagging elements that are legal or not. As of right now,
the full test suite passes (integration too). GraphFilter w
TinkerGraph now supports GraphComputer GraphFilter. Sort of an elegant solution that makes use of tagging elements that are legal or not. As of right now, the full test suite passes (integration too). GraphFilter works -- this is going to be huge for speeding up OLAP times.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/eee16c93
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/eee16c93
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/eee16c93
Branch: refs/heads/master
Commit: eee16c9354602a49e7dfb7738f2ce4d9fe36152c
Parents: d0ac652
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Feb 2 12:17:59 2016 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Feb 2 12:17:59 2016 -0700
----------------------------------------------------------------------
.../process/computer/GraphComputerTest.java | 2 --
.../hadoop/structure/io/GraphFilterAware.java | 6 ----
.../process/computer/TinkerGraphComputer.java | 10 ++++--
.../computer/TinkerGraphComputerView.java | 35 ++++++++++++++++++--
.../tinkergraph/structure/TinkerGraph.java | 15 ++++++---
.../tinkergraph/structure/TinkerHelper.java | 5 +--
.../tinkergraph/structure/TinkerVertex.java | 13 ++++++--
7 files changed, 64 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
index c77468b..16559c1 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/GraphComputerTest.java
@@ -34,7 +34,6 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
-import org.junit.Ignore;
import org.junit.Test;
import java.util.ArrayList;
@@ -1394,7 +1393,6 @@ public class GraphComputerTest extends AbstractGremlinProcessTest {
/////////////////////////////////////////////
@Test
- @Ignore
@LoadGraphWith(GRATEFUL)
public void shouldSupportWorkerCount() throws Exception {
int maxWorkers = graph.compute(graphComputerClass.get()).features().getMaxWorkers();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/GraphFilterAware.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/GraphFilterAware.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/GraphFilterAware.java
index fa59a4e..5a52c8c 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/GraphFilterAware.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/io/GraphFilterAware.java
@@ -23,12 +23,6 @@ import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.util.VertexProgramHelper;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
-import org.apache.tinkerpop.gremlin.structure.Direction;
-import org.apache.tinkerpop.gremlin.structure.Edge;
-import org.apache.tinkerpop.gremlin.structure.Vertex;
-
-import java.util.Iterator;
/**
* An input graph class is {@code GraphFilterAware} if it can filter out vertices and edges as its loading the graph from the
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
index f354f9f..c6dfdea 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputer.java
@@ -20,6 +20,7 @@ package org.apache.tinkerpop.gremlin.tinkergraph.process.computer;
import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.ComputerGraph;
@@ -58,6 +59,7 @@ public final class TinkerGraphComputer implements GraphComputer {
private boolean executed = false;
private final Set<MapReduce> mapReducers = new HashSet<>();
private int workers = Runtime.getRuntime().availableProcessors();
+ private final GraphFilter graphFilter = new GraphFilter();
public TinkerGraphComputer(final TinkerGraph graph) {
this.graph = graph;
@@ -95,12 +97,14 @@ public final class TinkerGraphComputer implements GraphComputer {
@Override
public GraphComputer vertices(final Traversal<Vertex, Vertex> vertexFilter) {
- throw new UnsupportedOperationException();
+ this.graphFilter.setVertexFilter(vertexFilter);
+ return this;
}
@Override
public GraphComputer edges(final Traversal<Vertex, Edge> edgeFilter) {
- throw new UnsupportedOperationException();
+ this.graphFilter.setEdgeFilter(edgeFilter);
+ return this;
}
@Override
@@ -134,7 +138,7 @@ public final class TinkerGraphComputer implements GraphComputer {
final long time = System.currentTimeMillis();
try (final TinkerWorkerPool workers = new TinkerWorkerPool(this.workers)) {
if (null != this.vertexProgram) {
- TinkerHelper.createGraphComputerView(this.graph, this.vertexProgram.getElementComputeKeys());
+ TinkerHelper.createGraphComputerView(this.graph, this.graphFilter, this.vertexProgram.getElementComputeKeys());
// execute the vertex program
this.vertexProgram.setup(this.memory);
this.memory.completeSubRound();
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
index 3c5fdc6..524d675 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/process/computer/TinkerGraphComputerView.java
@@ -19,6 +19,8 @@
package org.apache.tinkerpop.gremlin.tinkergraph.process.computer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Element;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -35,6 +37,8 @@ import org.apache.tinkerpop.gremlin.tinkergraph.structure.TinkerVertexProperty;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -50,11 +54,29 @@ public final class TinkerGraphComputerView {
private final TinkerGraph graph;
protected final Set<String> computeKeys;
private Map<Element, Map<String, List<VertexProperty<?>>>> computeProperties;
+ private final Set<Object> legalVertices = new HashSet<>();
+ private final Map<Object, Set<Object>> legalEdges = new HashMap<>();
+ private final GraphFilter graphFilter;
- public TinkerGraphComputerView(final TinkerGraph graph, final Set<String> computeKeys) {
+ public TinkerGraphComputerView(final TinkerGraph graph, final GraphFilter graphFilter, final Set<String> computeKeys) {
this.graph = graph;
this.computeKeys = computeKeys;
this.computeProperties = new ConcurrentHashMap<>();
+ this.graphFilter = graphFilter;
+ if (this.graphFilter.hasFilter()) {
+ graph.vertices().forEachRemaining(vertex -> {
+ boolean legalVertex = false;
+ if (this.graphFilter.hasVertexFilter() && TraversalUtil.applyAll(vertex, this.graphFilter.getVertexFilter()).hasNext()) {
+ this.legalVertices.add(vertex.id());
+ legalVertex = true;
+ }
+ if ((legalVertex || !this.graphFilter.hasVertexFilter()) && this.graphFilter.hasEdgeFilter()) {
+ final Set<Object> edges = new HashSet<>();
+ this.legalEdges.put(vertex.id(), edges);
+ TraversalUtil.applyAll(vertex, this.graphFilter.getEdgeFilter()).forEachRemaining(edge -> edges.add(edge.id()));
+ }
+ });
+ }
}
public <V> Property<V> addProperty(final TinkerVertex vertex, final String key, final V value) {
@@ -93,9 +115,18 @@ public final class TinkerGraphComputerView {
}
}
+ public boolean legalVertex(final Vertex vertex) {
+ return !this.graphFilter.hasVertexFilter() || this.legalVertices.contains(vertex.id());
+ }
+
+ public boolean legalEdge(final Vertex vertex, final Edge edge) {
+ return !this.graphFilter.hasEdgeFilter() || this.legalEdges.get(vertex.id()).contains(edge.id());
+ }
+
//////////////////////
- public Graph processResultGraphPersist(final GraphComputer.ResultGraph resultGraph, final GraphComputer.Persist persist) {
+ public Graph processResultGraphPersist(final GraphComputer.ResultGraph resultGraph,
+ final GraphComputer.Persist persist) {
if (GraphComputer.Persist.NOTHING == persist) {
if (GraphComputer.ResultGraph.ORIGINAL == resultGraph)
return this.graph;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 35e268e..73a4205 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -36,9 +36,9 @@ import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComputer;
import org.apache.tinkerpop.gremlin.tinkergraph.process.computer.TinkerGraphComputerView;
import org.apache.tinkerpop.gremlin.tinkergraph.process.traversal.strategy.optimization.TinkerGraphStepStrategy;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
import java.io.File;
-import java.io.IOException;
import java.util.Collections;
import java.util.Iterator;
import java.util.Map;
@@ -318,12 +318,12 @@ public final class TinkerGraph implements Graph {
}
-
private <T extends Element> Iterator<T> createElementIterator(final Class<T> clazz, final Map<Object, T> elements,
final IdManager idManager,
final Object... ids) {
+ final Iterator<T> iterator;
if (0 == ids.length) {
- return elements.values().iterator();
+ iterator = elements.values().iterator();
} else {
// base the conversion function on the first item in the id list as the expectation is that these
// id values will be a uniform list
@@ -336,14 +336,19 @@ public final class TinkerGraph implements Graph {
// other implementation. the assumption is that id conversion is not required for detached
// stuff - doesn't seem likely someone would detach a Titan vertex then try to expect that
// vertex to be findable in OrientDB
- return Stream.of(ids).map(id -> elements.get(((T) id).id())).filter(Objects::nonNull).iterator();
+ iterator = Stream.of(ids).map(id -> elements.get(((T) id).id())).filter(Objects::nonNull).iterator();
} else {
final Class<?> firstClass = ids[0].getClass();
if (!Stream.of(ids).map(Object::getClass).allMatch(firstClass::equals))
throw Graph.Exceptions.idArgsMustBeEitherIdOrElement();
- return Stream.of(ids).map(id -> idManager.convert(id)).map(elements::get).filter(Objects::nonNull).iterator();
+ iterator = Stream.of(ids).map(id -> idManager.convert(id)).map(elements::get).filter(Objects::nonNull).iterator();
}
}
+ return TinkerHelper.inComputerMode(this) ?
+ (Iterator<T>) (clazz.equals(Vertex.class) ?
+ IteratorUtils.filter((Iterator<Vertex>) iterator, t -> this.graphComputerView.legalVertex(t)) :
+ IteratorUtils.filter((Iterator<Edge>) iterator, t -> this.graphComputerView.legalEdge(t.outVertex(), t))) :
+ iterator;
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerHelper.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerHelper.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerHelper.java
index 7e6ace0..705fa49 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerHelper.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerHelper.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.tinkergraph.structure;
+import org.apache.tinkerpop.gremlin.process.computer.GraphFilter;
import org.apache.tinkerpop.gremlin.structure.Direction;
import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
@@ -100,8 +101,8 @@ public final class TinkerHelper {
return null != graph.graphComputerView;
}
- public static TinkerGraphComputerView createGraphComputerView(final TinkerGraph graph, final Set<String> computeKeys) {
- return graph.graphComputerView = new TinkerGraphComputerView(graph, computeKeys);
+ public static TinkerGraphComputerView createGraphComputerView(final TinkerGraph graph, final GraphFilter graphFilter, final Set<String> computeKeys) {
+ return graph.graphComputerView = new TinkerGraphComputerView(graph, graphFilter, computeKeys);
}
public static TinkerGraphComputerView getGraphComputerView(final TinkerGraph graph) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/eee16c93/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerVertex.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerVertex.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerVertex.java
index 877d338..be08137 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerVertex.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerVertex.java
@@ -143,12 +143,21 @@ public final class TinkerVertex extends TinkerElement implements Vertex {
@Override
public Iterator<Edge> edges(final Direction direction, final String... edgeLabels) {
- return (Iterator) TinkerHelper.getEdges(this, direction, edgeLabels);
+ final Iterator<Edge> edgeIterator = (Iterator) TinkerHelper.getEdges(this, direction, edgeLabels);
+ return TinkerHelper.inComputerMode(this.graph) ?
+ IteratorUtils.filter(edgeIterator, edge -> this.graph.graphComputerView.legalEdge(this, edge)) :
+ edgeIterator;
}
@Override
public Iterator<Vertex> vertices(final Direction direction, final String... edgeLabels) {
- return (Iterator) TinkerHelper.getVertices(this, direction, edgeLabels);
+ return TinkerHelper.inComputerMode(this.graph) ?
+ direction.equals(Direction.BOTH) ?
+ IteratorUtils.concat(
+ IteratorUtils.map(this.edges(Direction.OUT, edgeLabels), Edge::inVertex),
+ IteratorUtils.map(this.edges(Direction.IN, edgeLabels), Edge::outVertex)) :
+ IteratorUtils.map(this.edges(direction, edgeLabels), edge -> edge.vertices(direction.opposite()).next()) :
+ (Iterator) TinkerHelper.getVertices(this, direction, edgeLabels);
}
@Override