You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2015/11/04 20:11:42 UTC
[1/7] incubator-tinkerpop git commit: Optimized BLVP
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 2edefa604 -> c4fcae6a9
Optimized BLVP
BLVP now uses EventStrategy to monitor what the actual BulkLoader implementation does (e.g. whether it creates a new vertex or just returns an existing one).
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/8b8222da
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/8b8222da
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/8b8222da
Branch: refs/heads/master
Commit: 8b8222da7fed57fa1e7e0232ab788944716404f7
Parents: 19b7ae0
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Nov 3 15:14:12 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Nov 3 15:14:12 2015 +0100
----------------------------------------------------------------------
.../computer/bulkloading/BulkLoader.java | 31 +++++
.../bulkloading/BulkLoaderVertexProgram.java | 116 ++++++++++++++++---
.../bulkloading/IncrementalBulkLoader.java | 14 ++-
.../structure/TinkerGraphPlayTest.java | 30 ++---
4 files changed, 158 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8b8222da/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
index dd5eaf4..2640d2e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoader.java
@@ -41,6 +41,22 @@ public interface BulkLoader {
public Vertex getOrCreateVertex(final Vertex vertex, final Graph graph, final GraphTraversalSource g);
/**
+ * Creates a clone of the given edge between the given in- and out-vertices.
+ *
+ * @param edge The edge to be cloned.
+ * @param outVertex The out-vertex in the given graph..
+ * @param inVertex The in-vertex in the given graph.
+ * @param graph The graph that holds the cloned edge after this method was called.
+ * @param g A standard traversal source for the given graph.
+ * @return The cloned edge.
+ */
+ public default Edge createEdge(final Edge edge, final Vertex outVertex, final Vertex inVertex, final Graph graph, final GraphTraversalSource g) {
+ final Edge result = outVertex.addEdge(edge.label(), inVertex);
+ edge.properties().forEachRemaining(property -> result.property(property.key(), property.value()));
+ return result;
+ }
+
+ /**
* Gets or creates a clone of the given edge between the given in- and out-vertices.
*
* @param edge The edge to be cloned.
@@ -53,6 +69,21 @@ public interface BulkLoader {
public Edge getOrCreateEdge(final Edge edge, final Vertex outVertex, final Vertex inVertex, final Graph graph, final GraphTraversalSource g);
/**
+ * Creates a clone of the given property for the given vertex.
+ *
+ * @param property The property to be cloned.
+ * @param vertex The vertex in the given graph..
+ * @param graph The graph that holds the given vertex.
+ * @param g A standard traversal source for the given graph.
+ * @return The cloned property.
+ */
+ public default VertexProperty createVertexProperty(final VertexProperty<?> property, final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
+ final VertexProperty result = vertex.property(property.key(), property.value());
+ property.properties().forEachRemaining(metaProperty -> result.property(metaProperty.key(), metaProperty.value()));
+ return result;
+ }
+
+ /**
* Gets or creates a clone of the given property for the given vertex.
*
* @param property The property to be cloned.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8b8222da/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
index 99bb1f5..2950bfd 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
@@ -31,8 +31,12 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
+import org.apache.tinkerpop.gremlin.process.traversal.step.util.event.MutationListener;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategy;
import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.structure.Edge;
import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Property;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.VertexProperty;
import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
@@ -48,7 +52,6 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.atomic.AtomicLong;
/**
* @author Daniel Kuppitz (http://gremlin.guru)
@@ -74,12 +77,7 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
private GraphTraversalSource g;
private long intermediateBatchSize;
- private static final ThreadLocal<AtomicLong> counter = new ThreadLocal<AtomicLong>() {
- @Override
- protected AtomicLong initialValue() {
- return new AtomicLong();
- }
- };
+ private BulkLoadingListener listener;
private BulkLoaderVertexProgram() {
messageScope = MessageScope.Local.of(__::inE);
@@ -116,17 +114,19 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
* @param close Whether to close the current graph instance after calling commit() or not.
*/
private void commit(final boolean close) {
- if (!close && (intermediateBatchSize == 0L || counter.get().incrementAndGet() % intermediateBatchSize != 0))
+ if (!close && (intermediateBatchSize == 0L || listener.mutations() < intermediateBatchSize))
return;
if (null != graph) {
if (graph.features().graph().supportsTransactions()) {
- LOGGER.info("Committing transaction on Graph instance: {} [{}]", graph, counter.get().get());
+ LOGGER.info("Committing transaction on Graph instance: {} [{} mutations]", graph, listener.mutations());
try {
graph.tx().commit();
LOGGER.debug("Committed transaction on Graph instance: {}", graph);
+ listener.resetCounter();
} catch (Exception e) {
LOGGER.error("Failed to commit transaction on Graph instance: {}", graph);
graph.tx().rollback();
+ listener.resetCounter();
throw e;
}
}
@@ -144,7 +144,6 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
@Override
public void setup(final Memory memory) {
- counter.get().set(0L);
}
@Override
@@ -172,7 +171,8 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
graph = GraphFactory.open(configuration.subset(WRITE_GRAPH_CFG_KEY));
LOGGER.info("Opened Graph instance: {}", graph);
try {
- g = graph.traversal();
+ listener = new BulkLoadingListener();
+ g = GraphTraversalSource.build().with(EventStrategy.build().addListener(listener).create()).create(graph);
} catch (Exception e) {
try {
graph.close();
@@ -205,12 +205,15 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
private void executeInternal(final Vertex sourceVertex, final Messenger<Tuple> messenger, final Memory memory) {
if (memory.isInitialIteration()) {
+ this.listener.resetStats();
// get or create the vertex
final Vertex targetVertex = bulkLoader.getOrCreateVertex(sourceVertex, graph, g);
// write all the properties of the vertex to the newly created vertex
final Iterator<VertexProperty<Object>> vpi = sourceVertex.properties();
- while (vpi.hasNext()) {
- bulkLoader.getOrCreateVertexProperty(vpi.next(), targetVertex, graph, g);
+ if (this.listener.isNewVertex()) {
+ vpi.forEachRemaining(vp -> bulkLoader.createVertexProperty(vp, targetVertex, graph, g));
+ } else {
+ vpi.forEachRemaining(vp -> bulkLoader.getOrCreateVertexProperty(vp, targetVertex, graph, g));
}
this.commit(false);
if (!bulkLoader.useUserSuppliedIds()) {
@@ -221,9 +224,14 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
} else if (memory.getIteration() == 1) {
if (bulkLoader.useUserSuppliedIds()) {
final Vertex outV = bulkLoader.getVertex(sourceVertex, graph, g);
+ final boolean incremental = outV.edges(Direction.OUT).hasNext();
sourceVertex.edges(Direction.OUT).forEachRemaining(edge -> {
final Vertex inV = bulkLoader.getVertex(edge.inVertex(), graph, g);
- bulkLoader.getOrCreateEdge(edge, outV, inV, graph, g);
+ if (incremental) {
+ bulkLoader.getOrCreateEdge(edge, outV, inV, graph, g);
+ } else {
+ bulkLoader.createEdge(edge, outV, inV, graph, g);
+ }
this.commit(false);
});
} else {
@@ -407,4 +415,84 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
}
};
}
+
+ static class BulkLoadingListener implements MutationListener {
+
+ private long counter;
+ private boolean isNewVertex;
+
+ public BulkLoadingListener() {
+ this.counter = 0L;
+ this.isNewVertex = false;
+ ;
+ }
+
+ public boolean isNewVertex() {
+ return this.isNewVertex;
+ }
+
+ public long mutations() {
+ return this.counter;
+ }
+
+ public void resetStats() {
+ this.isNewVertex = false;
+ }
+
+ public void resetCounter() {
+ this.counter = 0L;
+ }
+
+ @Override
+ public void vertexAdded(final Vertex vertex) {
+ this.isNewVertex = true;
+ this.counter++;
+ }
+
+ @Override
+ public void vertexRemoved(final Vertex vertex) {
+ this.counter++;
+ }
+
+ @Override
+ public void vertexPropertyChanged(final Vertex element, final Property oldValue, final Object setValue,
+ final Object... vertexPropertyKeyValues) {
+ this.counter++;
+ }
+
+ @Override
+ public void vertexPropertyRemoved(final VertexProperty vertexProperty) {
+ this.counter++;
+ }
+
+ @Override
+ public void edgeAdded(final Edge edge) {
+ this.counter++;
+ }
+
+ @Override
+ public void edgeRemoved(final Edge edge) {
+ this.counter++;
+ }
+
+ @Override
+ public void edgePropertyChanged(final Edge element, final Property oldValue, final Object setValue) {
+ this.counter++;
+ }
+
+ @Override
+ public void edgePropertyRemoved(final Edge element, final Property property) {
+ this.counter++;
+ }
+
+ @Override
+ public void vertexPropertyPropertyChanged(final VertexProperty element, final Property oldValue, final Object setValue) {
+ this.counter++;
+ }
+
+ @Override
+ public void vertexPropertyPropertyRemoved(final VertexProperty element, final Property property) {
+ this.counter++;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8b8222da/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
index f03cd18..8219515 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/IncrementalBulkLoader.java
@@ -51,8 +51,10 @@ public class IncrementalBulkLoader implements BulkLoader {
return iterator.hasNext()
? iterator.next()
: useUserSuppliedIds()
- ? graph.addVertex(T.id, vertex.id(), T.label, vertex.label())
- : graph.addVertex(T.label, vertex.label(), getVertexIdProperty(), vertex.id());
+ ? g.addV(vertex.label()).property(T.id, vertex.id()).next()
+ : g.addV(vertex.label()).property(getVertexIdProperty(), vertex.id()).next();
+ //? graph.addVertex(T.id, vertex.id(), T.label, vertex.label())
+ //: graph.addVertex(T.label, vertex.label(), getVertexIdProperty(), vertex.id());
}
/**
@@ -71,8 +73,7 @@ public class IncrementalBulkLoader implements BulkLoader {
}
});
} else {
- e = outVertex.addEdge(edge.label(), inVertex);
- edge.properties().forEachRemaining(property -> e.property(property.key(), property.value()));
+ e = createEdge(edge, outVertex, inVertex, graph, g);
}
return e;
}
@@ -84,7 +85,10 @@ public class IncrementalBulkLoader implements BulkLoader {
public VertexProperty getOrCreateVertexProperty(final VertexProperty<?> property, final Vertex vertex, final Graph graph, final GraphTraversalSource g) {
final VertexProperty<?> vp;
final VertexProperty<?> existing = vertex.property(property.key());
- if (!existing.isPresent() || !existing.value().equals(property.value())) {
+ if (!existing.isPresent()) {
+ return createVertexProperty(property, vertex, graph, g);
+ }
+ if (!existing.value().equals(property.value())) {
vp = vertex.property(property.key(), property.value());
} else {
vp = existing;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8b8222da/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
index 06994e2..e7c54ce 100644
--- a/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
+++ b/tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.tinkerpop.gremlin.tinkergraph.structure;
+import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgram;
import org.apache.tinkerpop.gremlin.process.traversal.Operator;
import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
@@ -30,10 +31,12 @@ import org.apache.tinkerpop.gremlin.structure.Graph;
import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.io.graphml.GraphMLIo;
+import org.apache.tinkerpop.gremlin.structure.util.GraphFactory;
import org.apache.tinkerpop.gremlin.util.TimeUtil;
import org.junit.Ignore;
import org.junit.Test;
+import java.io.File;
import java.util.Arrays;
import java.util.List;
import java.util.function.Supplier;
@@ -192,20 +195,19 @@ public class TinkerGraphPlayTest {
@Test
@Ignore
public void testPlayDK() throws Exception {
- final Graph graph = TinkerFactory.createModern();
- final GraphTraversalSource g = graph.traversal();
- Traversal traversal = g.V().where(out().and().in()).profile().cap(TraversalMetrics.METRICS_KEY);
- //traversal.forEachRemaining(System.out::println);
- System.out.println(traversal.toString());
- traversal.asAdmin().applyStrategies();
- System.out.println(traversal.toString());
- traversal.forEachRemaining(System.out::println);
- traversal = g.V().where(and(out(), in())).profile().cap(TraversalMetrics.METRICS_KEY);
- //traversal.forEachRemaining(System.out::println);
- System.out.println(traversal.toString());
- traversal.asAdmin().applyStrategies();
- System.out.println(traversal.toString());
- //System.out.println(traversal.toString());
+
+ new File("/tmp/tinkergraph2.kryo").deleteOnExit();
+ new File("/tmp/tinkergraph3.kryo").deleteOnExit();
+
+ final Graph graph1 = TinkerFactory.createModern();
+ final Graph graph2 = GraphFactory.open("/tmp/graph2.properties");
+ TinkerFactory.generateModern((TinkerGraph) graph2);
+ graph2.close();
+
+ System.out.println("graph1 -> graph2");
+ graph1.compute().workers(1).program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph("/tmp/graph2.properties").create(graph1)).submit().get();
+ System.out.println("graph1 -> graph3");
+ graph1.compute().workers(1).program(BulkLoaderVertexProgram.build().userSuppliedIds(true).writeGraph("/tmp/graph3.properties").create(graph1)).submit().get();
}
@Test
[4/7] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP3-904
Posted by dk...@apache.org.
Merge branch 'master' into TINKERPOP3-904
Resolved Conflicts:
CHANGELOG.asciidoc
tinkergraph-gremlin/src/test/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraphPlayTest.java
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a2b94c4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a2b94c4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a2b94c4e
Branch: refs/heads/master
Commit: a2b94c4e4e25a04bdbd8b926fdc9d949ed6e42d2
Parents: 63c8cb8 558c04e
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Nov 4 13:51:18 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Nov 4 13:51:18 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../upgrade-release-3.1.x-incubating.asciidoc | 3 +-
.../gremlin/process/traversal/NumberHelper.java | 12 +++-
.../traversal/dsl/graph/GraphTraversal.java | 16 ++---
.../traversal/step/map/MaxGlobalStep.java | 14 +++--
.../traversal/step/map/MaxLocalStep.java | 5 +-
.../traversal/step/map/MeanGlobalStep.java | 61 +++++++++++---------
.../traversal/step/map/MeanLocalStep.java | 11 ++--
.../traversal/step/map/MinGlobalStep.java | 14 +++--
.../traversal/step/map/MinLocalStep.java | 5 +-
.../traversal/step/map/SumGlobalStep.java | 25 ++++----
.../traversal/step/map/SumLocalStep.java | 5 +-
.../traversal/step/map/MeanGlobalStepTest.java | 14 +++++
.../traversal/step/map/MeanLocalStepTest.java | 14 +++++
.../step/sideEffect/GroovyGroupTest.groovy | 2 +-
.../process/computer/GraphComputerTest.java | 29 +++++-----
.../process/traversal/step/map/SumTest.java | 2 +-
.../traversal/step/sideEffect/GroupTest.java | 22 ++++---
...ComputerVerificationStrategyProcessTest.java | 3 +-
.../groovy/plugin/HadoopGremlinPluginTest.java | 2 +-
.../spark/structure/io/InputRDDTest.java | 2 +-
21 files changed, 164 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a2b94c4e/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index ef0d032,5ad89e5..8d74212
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -25,7 -25,7 +25,8 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.0 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Optimized `BulkLoaderVertexProgram`. It now uses `EventStrategy` to monitor what the underlying `BulkLoader` implementation does (e.g. whether it creates a new vertex or returns an existing).
+ * Integrated `NumberHelper` in `SumStep`, `MinStep`, `MaxStep` and `MeanStep` (local and global step variants).
* Bumped to Neo4j 2.3.0.
* Added `PersistedInputRDD` and `PersistedOutputRDD` which enables `SparkGraphComputer` to store the graph RDD in the context between jobs (no HDFS serialization required).
* Renamed the `public static String` configuration variable names of TinkerGraph (deprecated old variables).
[2/7] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP3-904
Posted by dk...@apache.org.
Merge branch 'master' into TINKERPOP3-904
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/2a4b4ac6
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/2a4b4ac6
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/2a4b4ac6
Branch: refs/heads/master
Commit: 2a4b4ac67f4b1770a2ad69566fb819e42f312b07
Parents: 8b8222d 06d747f
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Nov 3 17:48:56 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Nov 3 17:48:56 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +-
docs/src/implementations.asciidoc | 36 +--
docs/src/the-traversal.asciidoc | 11 +-
.../upgrade-release-3.1.x-incubating.asciidoc | 18 +-
.../process/computer/GiraphGraphComputer.java | 11 +-
.../computer/io/GiraphVertexInputFormat.java | 70 ------
.../computer/io/GiraphVertexOutputFormat.java | 65 ------
.../process/computer/io/GiraphVertexReader.java | 67 ------
.../process/computer/io/GiraphVertexWriter.java | 57 -----
.../structure/io/GiraphVertexInputFormat.java | 70 ++++++
.../structure/io/GiraphVertexOutputFormat.java | 65 ++++++
.../giraph/structure/io/GiraphVertexReader.java | 67 ++++++
.../giraph/structure/io/GiraphVertexWriter.java | 57 +++++
.../gremlin/process/computer/GraphComputer.java | 5 +-
.../process/computer/GraphComputerTest.java | 2 +-
.../tinkerpop/gremlin/hadoop/Constants.java | 1 -
.../structure/hdfs/HadoopElementIterator.java | 4 +-
.../hadoop/structure/io/InputOutputHelper.java | 22 ++
.../hadoop/structure/util/HadoopHelper.java | 50 -----
.../process/computer/SparkGraphComputer.java | 75 ++++---
.../process/computer/io/InputFormatRDD.java | 47 ----
.../spark/process/computer/io/InputRDD.java | 41 ----
.../process/computer/io/OutputFormatRDD.java | 49 -----
.../spark/process/computer/io/OutputRDD.java | 31 ---
.../spark/structure/io/InputFormatRDD.java | 47 ++++
.../spark/structure/io/InputOutputHelper.java | 81 +++++++
.../gremlin/spark/structure/io/InputRDD.java | 41 ++++
.../spark/structure/io/OutputFormatRDD.java | 49 +++++
.../gremlin/spark/structure/io/OutputRDD.java | 31 +++
.../spark/structure/io/PersistedInputRDD.java | 60 +++++
.../spark/structure/io/PersistedOutputRDD.java | 41 ++++
.../process/computer/io/ExampleInputRDD.java | 47 ----
.../process/computer/io/ExampleOutputRDD.java | 45 ----
.../process/computer/io/InputOutputRDDTest.java | 59 -----
.../spark/process/computer/io/InputRDDTest.java | 54 -----
.../process/computer/io/OutputRDDTest.java | 62 ------
.../spark/structure/io/ExampleInputRDD.java | 51 +++++
.../spark/structure/io/ExampleOutputRDD.java | 50 +++++
.../spark/structure/io/InputOutputRDDTest.java | 59 +++++
.../spark/structure/io/InputRDDTest.java | 54 +++++
.../spark/structure/io/OutputRDDTest.java | 62 ++++++
.../io/PersistedInputOutputRDDTest.java | 217 +++++++++++++++++++
.../process/computer/TinkerGraphComputer.java | 5 -
43 files changed, 1228 insertions(+), 811 deletions(-)
----------------------------------------------------------------------
[5/7] incubator-tinkerpop git commit: removed empty code line
Posted by dk...@apache.org.
removed empty code line
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/f2b90833
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/f2b90833
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/f2b90833
Branch: refs/heads/master
Commit: f2b90833398a20b67f18fc106300986a6b229027
Parents: a2b94c4
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Nov 4 14:58:13 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Nov 4 14:58:13 2015 +0100
----------------------------------------------------------------------
.../process/computer/bulkloading/BulkLoaderVertexProgram.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/f2b90833/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
index 2950bfd..cb36cf4 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkloading/BulkLoaderVertexProgram.java
@@ -424,7 +424,6 @@ public class BulkLoaderVertexProgram implements VertexProgram<Tuple> {
public BulkLoadingListener() {
this.counter = 0L;
this.isNewVertex = false;
- ;
}
public boolean isNewVertex() {
[3/7] incubator-tinkerpop git commit: updated CHANGELOG
Posted by dk...@apache.org.
updated CHANGELOG
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/63c8cb87
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/63c8cb87
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/63c8cb87
Branch: refs/heads/master
Commit: 63c8cb87572ab2b6b3b3e386c5326edca017421e
Parents: 2a4b4ac
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Tue Nov 3 17:52:31 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Tue Nov 3 17:52:31 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/63c8cb87/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index e79a0a0..ef0d032 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -25,6 +25,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.0 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Optimized `BulkLoaderVertexProgram`. It now uses `EventStrategy` to monitor what the underlying `BulkLoader` implementation does (e.g. whether it creates a new vertex or returns an existing).
* Bumped to Neo4j 2.3.0.
* Added `PersistedInputRDD` and `PersistedOutputRDD` which enables `SparkGraphComputer` to store the graph RDD in the context between jobs (no HDFS serialization required).
* Renamed the `public static String` configuration variable names of TinkerGraph (deprecated old variables).
[6/7] incubator-tinkerpop git commit: added a note for BulkLoader
implementers
Posted by dk...@apache.org.
added a note for BulkLoader implementers
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/77fac190
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/77fac190
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/77fac190
Branch: refs/heads/master
Commit: 77fac190dfedd107a38086e201334298e0bd3829
Parents: f2b9083
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Nov 4 20:07:27 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Nov 4 20:07:27 2015 +0100
----------------------------------------------------------------------
docs/src/the-graphcomputer.asciidoc | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/77fac190/docs/src/the-graphcomputer.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/the-graphcomputer.asciidoc b/docs/src/the-graphcomputer.asciidoc
index 4fd0e8c..af65f07 100644
--- a/docs/src/the-graphcomputer.asciidoc
+++ b/docs/src/the-graphcomputer.asciidoc
@@ -368,6 +368,10 @@ will work for the most use-cases, but has one limitation though: It doesn't supp
`IncrementalBulkLoader` will handle every property as a single-valued property. A custom `BulkLoader` implementation
has to be used if the default behavior is not sufficient.
+NOTE: A custom `BulkLoader` implementation for incremental loading should use `GraphTraversal` methods to create/update
+elements (e.g. `g.addV()` instead of `graph.addVertex()`). This way the `BulkLoaderVertexProgram` is able to efficiently
+track changes in the underlying graph and can apply several optimization techniques.
+
[[traversalvertexprogram]]
TraversalVertexProgram
~~~~~~~~~~~~~~~~~~~~~~
[7/7] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP3-904
Posted by dk...@apache.org.
Merge branch 'master' into TINKERPOP3-904
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c4fcae6a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c4fcae6a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c4fcae6a
Branch: refs/heads/master
Commit: c4fcae6a942fa67b457d14ef6ac10d98b732164b
Parents: 77fac19 2edefa6
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Nov 4 20:08:20 2015 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Nov 4 20:08:20 2015 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 3 +
docs/src/implementations.asciidoc | 9 ++
.../process/traversal/step/map/MatchStep.java | 46 ++++++---
.../traversal/step/map/MatchStepTest.java | 82 ++++++++++++++-
.../process/computer/SparkGraphComputer.java | 32 ++++++
.../process/computer/LocalPropertyTest.java | 100 +++++++++++++++++++
.../spark/structure/io/InputOutputRDDTest.java | 3 +-
.../spark/structure/io/InputRDDTest.java | 3 +-
.../spark/structure/io/OutputRDDTest.java | 5 +-
.../io/PersistedInputOutputRDDTest.java | 13 ++-
10 files changed, 270 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c4fcae6a/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 8d74212,085bed7..9f0e18e
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -25,8 -25,10 +25,11 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.0 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Optimized `BulkLoaderVertexProgram`. It now uses `EventStrategy` to monitor what the underlying `BulkLoader` implementation does (e.g. whether it creates a new vertex or returns an existing).
* Integrated `NumberHelper` in `SumStep`, `MinStep`, `MaxStep` and `MeanStep` (local and global step variants).
+ * `CountMatchAlgorithm`, in OLAP, now biases traversal selection towards those traversals that start at the current traverser location to reduce message passing.
+ * Fixed a file stream bug in Hadoop OLTP that showed up if the streamed file was more than 2G of data.
+ * Added the ability to set thread local properties in `SparkGraphComputer` when using a persistent context.
* Bumped to Neo4j 2.3.0.
* Added `PersistedInputRDD` and `PersistedOutputRDD` which enables `SparkGraphComputer` to store the graph RDD in the context between jobs (no HDFS serialization required).
* Renamed the `public static String` configuration variable names of TinkerGraph (deprecated old variables).