You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by dk...@apache.org on 2016/01/13 13:28:42 UTC
[1/6] incubator-tinkerpop git commit: implemented
BulkDumperVertexProgram + integration tests
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 96f34b12a -> 869c0d174
implemented BulkDumperVertexProgram + integration tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/bdfd4902
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/bdfd4902
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/bdfd4902
Branch: refs/heads/master
Commit: bdfd49023b2dbaab531035b618f9a2d461228172
Parents: 46c7189
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Fri Jan 8 20:17:45 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Fri Jan 8 20:17:45 2016 +0100
----------------------------------------------------------------------
.../bulkdumping/BulkDumperVertexProgram.java | 98 +++++++++++++++++
.../process/GroovyProcessComputerSuite.java | 2 +
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../BulkDumperVertexProgramTest.java | 104 +++++++++++++++++++
4 files changed, 206 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bdfd4902/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgram.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgram.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgram.java
new file mode 100644
index 0000000..fcbdc6c
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgram.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.bulkdumping;
+
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.Memory;
+import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
+import org.apache.tinkerpop.gremlin.process.computer.Messenger;
+import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
+import org.apache.tinkerpop.gremlin.process.computer.util.AbstractVertexProgramBuilder;
+import org.apache.tinkerpop.gremlin.structure.Graph;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
+import org.javatuples.Tuple;
+
+import java.util.Collections;
+import java.util.Set;
+
+/**
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class BulkDumperVertexProgram implements VertexProgram<Tuple> {
+
+ private BulkDumperVertexProgram() {
+ }
+
+ @Override
+ public void setup(final Memory memory) {
+ }
+
+ @Override
+ public void execute(final Vertex sourceVertex, final Messenger<Tuple> messenger, final Memory memory) {
+ }
+
+ @Override
+ public boolean terminate(final Memory memory) {
+ return true;
+ }
+
+ @Override
+ public Set<MessageScope> getMessageScopes(final Memory memory) {
+ return Collections.emptySet();
+ }
+
+ @SuppressWarnings({"CloneDoesntDeclareCloneNotSupportedException", "CloneDoesntCallSuperClone"})
+ @Override
+ public VertexProgram<Tuple> clone() {
+ return this;
+ }
+
+ @Override
+ public GraphComputer.ResultGraph getPreferredResultGraph() {
+ return GraphComputer.ResultGraph.NEW;
+ }
+
+ @Override
+ public GraphComputer.Persist getPreferredPersist() {
+ return GraphComputer.Persist.EDGES;
+ }
+
+ @Override
+ public String toString() {
+ return StringFactory.vertexProgramString(this);
+ }
+
+ public static Builder build() {
+ return new Builder();
+ }
+
+ public static class Builder extends AbstractVertexProgramBuilder<Builder> {
+
+ private Builder() {
+ super(BulkDumperVertexProgram.class);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public BulkDumperVertexProgram create(final Graph graph) {
+ return (BulkDumperVertexProgram) VertexProgram.createVertexProgram(graph, configuration);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bdfd4902/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
index 56b6b34..88be667 100644
--- a/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
+++ b/gremlin-groovy-test/src/main/java/org/apache/tinkerpop/gremlin/process/GroovyProcessComputerSuite.java
@@ -23,6 +23,7 @@ import org.apache.tinkerpop.gremlin.GraphManager;
import org.apache.tinkerpop.gremlin.groovy.loaders.SugarLoader;
import org.apache.tinkerpop.gremlin.groovy.util.SugarTestHelper;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.bulkdumping.BulkDumperVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgramTest;
@@ -171,6 +172,7 @@ public class GroovyProcessComputerSuite extends ProcessComputerSuite {
PageRankVertexProgramTest.class,
PeerPressureVertexProgramTest.class,
BulkLoaderVertexProgramTest.class,
+ BulkDumperVertexProgramTest.class,
};
public GroovyProcessComputerSuite(final Class<?> klass, final RunnerBuilder builder) throws InitializationError {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bdfd4902/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
index 76919be..3338acf 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/ProcessComputerSuite.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.process;
import org.apache.tinkerpop.gremlin.AbstractGremlinSuite;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputerTest;
+import org.apache.tinkerpop.gremlin.process.computer.bulkdumping.BulkDumperVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgramTest;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgramTest;
@@ -175,6 +176,7 @@ public class ProcessComputerSuite extends AbstractGremlinSuite {
PageRankVertexProgramTest.class,
PeerPressureVertexProgramTest.class,
BulkLoaderVertexProgramTest.class,
+ BulkDumperVertexProgramTest.class,
// strategy
ComputerVerificationStrategyProcessTest.ComputerTraversals.class,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/bdfd4902/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgramTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgramTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgramTest.java
new file mode 100644
index 0000000..f33e2cf
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/computer/bulkdumping/BulkDumperVertexProgramTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.computer.bulkdumping;
+
+import org.apache.tinkerpop.gremlin.LoadGraphWith;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.computer.ComputerResult;
+import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.structure.Direction;
+import org.apache.tinkerpop.gremlin.util.iterator.IteratorUtils;
+import org.junit.Test;
+
+import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author Daniel Kuppitz (http://gremlin.guru)
+ */
+public class BulkDumperVertexProgramTest extends AbstractGremlinProcessTest {
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void shouldDumpWholeGraph() throws Exception {
+ if (g.getGraphComputer().get().features().supportsResultGraphPersistCombination(GraphComputer.ResultGraph.NEW, GraphComputer.Persist.EDGES)) {
+ final ComputerResult result = graph.compute(g.getGraphComputer().get().getClass()).program(BulkDumperVertexProgram.build().create(graph)).submit().get();
+ result.graph().traversal().V().forEachRemaining(v -> {
+ assertEquals(2, v.keys().size());
+ assertTrue(v.keys().contains("name"));
+ assertTrue(v.keys().contains("age") || v.keys().contains("lang"));
+ assertEquals(1, IteratorUtils.count(v.values("name")));
+ assertEquals(1, IteratorUtils.count(v.values("age", "lang")));
+ final String name = v.value("name");
+ if (name.equals("marko")) {
+ assertEquals(1, v.id());
+ assertEquals("person", v.label());
+ assertEquals(Integer.valueOf(29), v.value("age"));
+ assertEquals(3, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT, "knows")));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT, "created")));
+ assertEquals(0, IteratorUtils.count(v.edges(Direction.IN)));
+ } else if (name.equals("vadas")) {
+ assertEquals(2, v.id());
+ assertEquals("person", v.label());
+ assertEquals(Integer.valueOf(27), v.value("age"));
+ assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "knows")));
+ } else if (name.equals("lop")) {
+ assertEquals(3, v.id());
+ assertEquals("software", v.label());
+ assertEquals("java", v.value("lang"));
+ assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(3, IteratorUtils.count(v.edges(Direction.IN)));
+ assertEquals(3, IteratorUtils.count(v.edges(Direction.IN, "created")));
+ } else if (name.equals("josh")) {
+ assertEquals(4, v.id());
+ assertEquals("person", v.label());
+ assertEquals(Integer.valueOf(32), v.value("age"));
+ assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(2, IteratorUtils.count(v.edges(Direction.OUT, "created")));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "knows")));
+ } else if (name.equals("ripple")) {
+ assertEquals(5, v.id());
+ assertEquals("software", v.label());
+ assertEquals("java", v.value("lang"));
+ assertEquals(0, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.IN, "created")));
+ } else if (name.equals("peter")) {
+ assertEquals(6, v.id());
+ assertEquals("person", v.label());
+ assertEquals(Integer.valueOf(35), v.value("age"));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT)));
+ assertEquals(1, IteratorUtils.count(v.edges(Direction.OUT, "created")));
+ assertEquals(0, IteratorUtils.count(v.edges(Direction.IN)));
+ } else
+ throw new IllegalStateException("The following vertex should not exist in the graph: " + name);
+ });
+ assertEquals(3.5, (Double) result.graph().traversal().E().values("weight").sum().next(), 0.01);
+ assertEquals(1.5, (Double) result.graph().traversal().E().hasLabel("knows").values("weight").sum().next(), 0.01);
+ assertEquals(2.0, (Double) result.graph().traversal().E().hasLabel("created").values("weight").sum().next(), 0.01);
+ assertEquals(result.memory().getIteration(), 0);
+ assertEquals(result.memory().asMap().size(), 0);
+ }
+ }
+}
\ No newline at end of file
[5/6] incubator-tinkerpop git commit: Merge branch 'master' into
TINKERPOP-320
Posted by dk...@apache.org.
Merge branch 'master' into TINKERPOP-320
Resolved Conflicts:
CHANGELOG.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/96388ee8
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/96388ee8
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/96388ee8
Branch: refs/heads/master
Commit: 96388ee83537bc7e2d5a15d3323e15b81b6a50d7
Parents: d7ae923 1e989b7
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Mon Jan 11 17:48:00 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Mon Jan 11 17:48:00 2016 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
.../traversal/util/DependantMutableMetrics.java | 34 +++++++++++---------
.../process/traversal/util/MutableMetrics.java | 3 +-
.../customizer/SimpleSandboxExtension.groovy | 5 +--
.../server/GremlinServerIntegrateTest.java | 30 +++++++++++++++++
.../hadoop/structure/io/ObjectWritable.java | 7 +++-
6 files changed, 61 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/96388ee8/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 27003b6,fdbfb90..85280af
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -26,7 -26,7 +26,8 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `BulkDumperVertex` that allows to dump a whole graph in any of the supported IO formats (GraphSON, Gryo, Script).
+ * Fixed a bug around duration calculations of `cap()`-step during profiling.
* It is possible to completely avoid using HDFS with Spark if `PersistedInputRDD` and `PersistedOutpuRDD` are leveraged.
* `InputRDD` and `OutputRDD` can now process both graphs and memory (i.e. sideEffects).
* Removed Groovy specific meta-programming overloads for handling Hadoop `FileSystem` (instead, its all accessible via `FileSystemStorage`).
[2/6] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master' into TINKERPOP-320
Posted by dk...@apache.org.
Merge remote-tracking branch 'origin/master' into TINKERPOP-320
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/43eaf8dd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/43eaf8dd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/43eaf8dd
Branch: refs/heads/master
Commit: 43eaf8dda431f5deedf0f2725d9c87f0bf58aa47
Parents: bdfd490 114609d
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Fri Jan 8 20:19:39 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Fri Jan 8 20:19:39 2016 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 7 +
docs/src/reference/implementations.asciidoc | 104 ++++----
.../upgrade/release-3.1.x-incubating.asciidoc | 26 ++
.../process/computer/GiraphGraphComputer.java | 43 ++--
.../GiraphHadoopGremlinIntegrateTest.java | 33 +++
.../computer/GiraphHadoopGraphProvider.java | 6 +-
.../GiraphHadoopGremlinPluginIntegrateTest.java | 33 ---
.../peerpressure/ClusterCountMapReduce.java | 7 +-
.../tinkerpop/gremlin/structure/io/Storage.java | 139 +++++++++++
.../gremlin/structure/util/StringFactory.java | 18 +-
.../tinkerpop/gremlin/driver/Cluster.java | 3 +
.../tinkerpop/gremlin/driver/Settings.java | 2 +
.../process/computer/GraphComputerTest.java | 54 ++--
.../conf/hadoop-grateful-gryo.properties | 6 +-
.../hadoop/groovy/plugin/HadoopLoader.groovy | 138 -----------
.../tinkerpop/gremlin/hadoop/Constants.java | 20 ++
.../groovy/plugin/HadoopGremlinPlugin.java | 11 +-
.../process/computer/util/MapReduceHelper.java | 8 +-
.../hadoop/structure/HadoopConfiguration.java | 5 +
.../gremlin/hadoop/structure/HadoopGraph.java | 4 +-
.../hadoop/structure/hdfs/HDFSTools.java | 113 ---------
.../structure/hdfs/HadoopEdgeIterator.java | 83 -------
.../structure/hdfs/HadoopElementIterator.java | 74 ------
.../structure/hdfs/HadoopVertexIterator.java | 82 -------
.../hadoop/structure/hdfs/HiddenFileFilter.java | 44 ----
.../hadoop/structure/hdfs/TextIterator.java | 91 -------
.../hadoop/structure/io/FileSystemStorage.java | 246 +++++++++++++++++++
.../hadoop/structure/io/HadoopEdgeIterator.java | 79 ++++++
.../structure/io/HadoopElementIterator.java | 75 ++++++
.../structure/io/HadoopVertexIterator.java | 78 ++++++
.../hadoop/structure/io/HiddenFileFilter.java | 44 ++++
.../hadoop/structure/io/InputOutputHelper.java | 2 +-
.../structure/io/ObjectWritableIterator.java | 12 +-
.../hadoop/structure/io/TextIterator.java | 91 +++++++
.../structure/io/VertexWritableIterator.java | 10 +-
.../gremlin/hadoop/HadoopGremlinSuite.java | 36 +++
.../groovy/plugin/HadoopGremlinPluginCheck.java | 71 +-----
.../hadoop/groovy/plugin/HadoopPluginSuite.java | 34 ---
.../structure/io/AbstractStorageCheck.java | 145 +++++++++++
.../structure/io/FileSystemStorageCheck.java | 88 +++++++
pom.xml | 2 +-
spark-gremlin/pom.xml | 2 +-
.../spark/groovy/plugin/SparkLoader.groovy | 68 -----
.../spark/groovy/plugin/SparkGremlinPlugin.java | 5 +-
.../spark/process/computer/SparkExecutor.java | 28 ---
.../process/computer/SparkGraphComputer.java | 18 +-
.../gremlin/spark/structure/Spark.java | 2 +
.../spark/structure/io/InputFormatRDD.java | 15 ++
.../spark/structure/io/InputOutputHelper.java | 3 -
.../gremlin/spark/structure/io/InputRDD.java | 17 +-
.../spark/structure/io/OutputFormatRDD.java | 30 ++-
.../gremlin/spark/structure/io/OutputRDD.java | 21 ++
.../spark/structure/io/PersistedInputRDD.java | 14 +-
.../spark/structure/io/PersistedOutputRDD.java | 25 +-
.../spark/structure/io/SparkContextStorage.java | 164 +++++++++++++
.../gremlin/spark/AbstractSparkTest.java | 30 +++
.../gremlin/spark/SparkGremlinSuite.java | 35 +++
.../gremlin/spark/SparkGremlinTest.java | 33 +++
.../gremlin/spark/SparkHadoopGremlinTest.java | 33 +++
.../process/computer/LocalPropertyTest.java | 2 +-
.../computer/SparkHadoopGraphProvider.java | 12 +-
.../groovy/plugin/SparkGremlinPluginTest.java | 126 ----------
.../plugin/SparkHadoopGremlinPluginTest.java | 33 ---
.../gremlin/spark/structure/SparkTest.java | 10 +-
.../spark/structure/io/ExampleInputRDD.java | 5 +
.../spark/structure/io/ExampleOutputRDD.java | 6 +
.../io/PersistedInputOutputRDDTest.java | 29 +--
.../structure/io/SparkContextStorageCheck.java | 74 ++++++
.../spark/structure/io/ToyGraphInputRDD.java | 7 +
.../tinkergraph/structure/TinkerIoRegistry.java | 1 +
.../tinkergraph/structure/TinkerGraphTest.java | 129 +++++++++-
71 files changed, 1939 insertions(+), 1205 deletions(-)
----------------------------------------------------------------------
[4/6] incubator-tinkerpop git commit: updated CHANGELOG and reference
docs
Posted by dk...@apache.org.
updated CHANGELOG and reference docs
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/d7ae9236
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/d7ae9236
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/d7ae9236
Branch: refs/heads/master
Commit: d7ae9236bd2a08212f0aa6ac1547bd7b3da95d68
Parents: 53f28d4
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Mon Jan 11 17:46:23 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Mon Jan 11 17:46:23 2016 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/reference/implementations.asciidoc | 18 ++++++++++++++++++
docs/src/reference/the-graphcomputer.asciidoc | 9 +++++++++
3 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d7ae9236/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 3185290..27003b6 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,7 @@ image::https://raw.githubusercontent.com/apache/incubator-tinkerpop/master/docs/
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `BulkDumperVertex` that allows to dump a whole graph in any of the supported IO formats (GraphSON, Gryo, Script).
* It is possible to completely avoid using HDFS with Spark if `PersistedInputRDD` and `PersistedOutpuRDD` are leveraged.
* `InputRDD` and `OutputRDD` can now process both graphs and memory (i.e. sideEffects).
* Removed Groovy specific meta-programming overloads for handling Hadoop `FileSystem` (instead, its all accessible via `FileSystemStorage`).
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d7ae9236/docs/src/reference/implementations.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/implementations.asciidoc b/docs/src/reference/implementations.asciidoc
index add8555..b55e36b 100644
--- a/docs/src/reference/implementations.asciidoc
+++ b/docs/src/reference/implementations.asciidoc
@@ -1227,6 +1227,24 @@ references to that Spark Context. The exception to this rule are those propertie
Finally, there is a `spark` object that can be used to manage persisted RDDs (see <<interacting-with-spark, Interacting with Spark>>).
+[[bulkdumpervertexprogramusingspark]]
+Exporting with BulkDumperVertexProgram
+++++++++++++++++++++++++++++++++++++++
+
+The <<bulkdumpervertexprogram, BulkDumperVertexProgram>> exports a whole graph in any of the supported Hadoop GraphOutputFormats (`GraphSONOutputFormat`,
+`GryoOutputFormat` or `ScriptOutputFormat`). The example below takes a Hadoop graph as the input (in `GryoInputFormat`) and exports it as a GraphSON file
+(`GraphSONOutputFormat`).
+
+[gremlin-groovy]
+----
+hdfs.copyFromLocal('data/tinkerpop-modern.kryo', 'tinkerpop-modern.kryo')
+graph = GraphFactory.open('conf/hadoop/hadoop-gryo.properties')
+graph.configuration().setProperty('gremlin.hadoop.graphOutputFormat', 'org.apache.tinkerpop.gremlin.hadoop.structure.io.graphson.GraphSONOutputFormat')
+graph.compute(SparkGraphComputer).program(BulkDumperVertexProgram.build().create()).submit().get()
+hdfs.ls('output')
+hdfs.head('output/~g')
+----
+
Loading with BulkLoaderVertexProgram
++++++++++++++++++++++++++++++++++++
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/d7ae9236/docs/src/reference/the-graphcomputer.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/reference/the-graphcomputer.asciidoc b/docs/src/reference/the-graphcomputer.asciidoc
index d2a429a..f9116fb 100644
--- a/docs/src/reference/the-graphcomputer.asciidoc
+++ b/docs/src/reference/the-graphcomputer.asciidoc
@@ -319,6 +319,15 @@ same cluster. The algorithm proceeds in the following manner.
.. If there is a tie, then the cluster with the lowest `toString()` comparison is selected.
. Steps 3 and 4 repeat until either a max number of iterations has occurred or no vertex has adjusted its cluster anymore.
+[[bulkdumpervertexprogram]]
+BulkDumperVertexProgram
+~~~~~~~~~~~~~~~~~~~~~~~
+
+The `BulkDumperVertexProgram` can be used to export a whole graph in any of the provided Hadoop GraphOutputFormats (e.g.
+`GraphSONOutputFormat`, `GryoOutputFormat` or `ScriptOutputFormat`). The input can be any Hadoop GraphInputFormat
+(e.g. `GraphSONInputFormat`, `GryoInputFormat` or `ScriptInputFormat`). An <<bulkdumpervertexprogramusingspark,example>>
+is provided in the SparkGraphComputer section.
+
[[bulkloadervertexprogram]]
BulkLoaderVertexProgram
~~~~~~~~~~~~~~~~~~~~~~~
[3/6] incubator-tinkerpop git commit: added import for
BulkDumperVertexProgram to console standard imports
Posted by dk...@apache.org.
added import for BulkDumperVertexProgram to console standard imports
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/53f28d4b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/53f28d4b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/53f28d4b
Branch: refs/heads/master
Commit: 53f28d4b8cf2dc89e492e82a7836e70dcfa9e559
Parents: 43eaf8d
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Sat Jan 9 15:03:36 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Sat Jan 9 15:03:36 2016 +0100
----------------------------------------------------------------------
.../tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/53f28d4b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java
----------------------------------------------------------------------
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java
index 5df8cfe..f5bcde8 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/AbstractImportCustomizerProvider.java
@@ -24,6 +24,7 @@ import org.apache.commons.configuration.Configuration;
import org.apache.tinkerpop.gremlin.groovy.function.GFunction;
import org.apache.tinkerpop.gremlin.groovy.loaders.GremlinLoader;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
+import org.apache.tinkerpop.gremlin.process.computer.bulkdumping.BulkDumperVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.bulkloading.BulkLoaderVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.clustering.peerpressure.PeerPressureVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.ranking.pagerank.PageRankVertexProgram;
@@ -133,6 +134,7 @@ public abstract class AbstractImportCustomizerProvider implements ImportCustomiz
imports.add(PageRankVertexProgram.class.getPackage().getName() + DOT_STAR);
imports.add(TraversalVertexProgram.class.getPackage().getName() + DOT_STAR);
imports.add(BulkLoaderVertexProgram.class.getPackage().getName() + DOT_STAR);
+ imports.add(BulkDumperVertexProgram.class.getPackage().getName() + DOT_STAR);
// groovy extras
imports.add(Grape.class.getCanonicalName());
[6/6] incubator-tinkerpop git commit: Merge branch 'TINKERPOP-320'
Posted by dk...@apache.org.
Merge branch 'TINKERPOP-320'
Resolved Conflicts:
CHANGELOG.asciidoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/869c0d17
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/869c0d17
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/869c0d17
Branch: refs/heads/master
Commit: 869c0d174ef45ed711ab5329d4c382bbe671f77e
Parents: 96f34b1 96388ee
Author: Daniel Kuppitz <da...@hotmail.com>
Authored: Wed Jan 13 13:27:58 2016 +0100
Committer: Daniel Kuppitz <da...@hotmail.com>
Committed: Wed Jan 13 13:27:58 2016 +0100
----------------------------------------------------------------------
CHANGELOG.asciidoc | 1 +
docs/src/reference/implementations.asciidoc | 18 ++++
docs/src/reference/the-graphcomputer.asciidoc | 9 ++
.../bulkdumping/BulkDumperVertexProgram.java | 98 +++++++++++++++++
.../process/GroovyProcessComputerSuite.java | 2 +
.../AbstractImportCustomizerProvider.java | 2 +
.../gremlin/process/ProcessComputerSuite.java | 2 +
.../BulkDumperVertexProgramTest.java | 104 +++++++++++++++++++
8 files changed, 236 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/869c0d17/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --cc CHANGELOG.asciidoc
index 1049c97,85280af..581f46e
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@@ -26,8 -26,7 +26,9 @@@ image::https://raw.githubusercontent.co
TinkerPop 3.1.1 (NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* The Spark persistence `StorageLevel` can now be set for both job graphs and `PersistedOutputRDD` data.
+* Added to the list of "invalid binding keys" allowed by Gremlin Server to cover the private fields of `T` which get exposed in the `ScriptEngine` on static imports.
+ * Added `BulkDumperVertex` that allows to dump a whole graph in any of the supported IO formats (GraphSON, Gryo, Script).
* Fixed a bug around duration calculations of `cap()`-step during profiling.
* It is possible to completely avoid using HDFS with Spark if `PersistedInputRDD` and `PersistedOutpuRDD` are leveraged.
* `InputRDD` and `OutputRDD` can now process both graphs and memory (i.e. sideEffects).
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/869c0d17/docs/src/reference/implementations.asciidoc
----------------------------------------------------------------------