You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2015/04/10 15:34:57 UTC
[1/9] incubator-tinkerpop git commit: added
ReferenceVertexPropertyTest.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/TINKERPOP3-581 b67712a9d -> 78d27103f
added ReferenceVertexPropertyTest.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/302f0129
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/302f0129
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/302f0129
Branch: refs/heads/TINKERPOP3-581
Commit: 302f0129442583ac5a663f3ad898d9b39e457eec
Parents: 5575ab6
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 15:40:32 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 15:40:32 2015 -0600
----------------------------------------------------------------------
.../util/reference/ReferenceVertexProperty.java | 9 +-
.../structure/StructureStandardSuite.java | 2 +
.../reference/ReferenceVertexPropertyTest.java | 101 +++++++++++++++++++
3 files changed, 110 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/302f0129/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexProperty.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexProperty.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexProperty.java
index 4ef4913..073529c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexProperty.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexProperty.java
@@ -38,16 +38,21 @@ import java.util.NoSuchElementException;
*/
public class ReferenceVertexProperty<V> extends ReferenceElement<VertexProperty> implements VertexProperty<V> {
+ private ReferenceVertex vertex;
+
private ReferenceVertexProperty() {
}
public ReferenceVertexProperty(final VertexProperty vertexProperty) {
super(vertexProperty);
+ this.vertex = ReferenceFactory.detach(vertexProperty.element());
}
@Override
public VertexProperty<V> attach(final Vertex hostVertex) {
+ if (!hostVertex.equals(this.vertex))
+ throw Attachable.Exceptions.canNotAttachVertexPropertyToHostVertex(this, hostVertex);
final Iterator<VertexProperty<V>> vertexPropertyIterator = IteratorUtils.filter(hostVertex.<V>properties(), vp -> vp.id().equals(this.id));
if (!vertexPropertyIterator.hasNext())
throw Attachable.Exceptions.canNotAttachVertexPropertyToHostVertex(this, hostVertex);
@@ -56,7 +61,7 @@ public class ReferenceVertexProperty<V> extends ReferenceElement<VertexProperty>
@Override
public VertexProperty<V> attach(final Graph hostGraph) {
- throw new UnsupportedOperationException();
+ return this.attach(this.vertex.attach(hostGraph));
}
@Override
@@ -81,7 +86,7 @@ public class ReferenceVertexProperty<V> extends ReferenceElement<VertexProperty>
@Override
public Vertex element() {
- return null;
+ return this.vertex;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/302f0129/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/StructureStandardSuite.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/StructureStandardSuite.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/StructureStandardSuite.java
index 1b65447..3211f6b 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/StructureStandardSuite.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/StructureStandardSuite.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedPropertyTest
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexPropertyTest;
import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexTest;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdgeTest;
+import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexPropertyTest;
import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertexTest;
import org.apache.tinkerpop.gremlin.structure.util.star.StarGraphTest;
import org.junit.runners.model.InitializationError;
@@ -81,6 +82,7 @@ public class StructureStandardSuite extends AbstractGremlinSuite {
VariablesTest.class,
PropertyTest.class,
ReferenceEdgeTest.class,
+ ReferenceVertexPropertyTest.class,
ReferenceVertexTest.class,
SerializationTest.class,
StarGraphTest.class,
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/302f0129/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexPropertyTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexPropertyTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexPropertyTest.java
new file mode 100644
index 0000000..500a3ba
--- /dev/null
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/structure/util/reference/ReferenceVertexPropertyTest.java
@@ -0,0 +1,101 @@
+/*
+ *
+ * * Licensed to the Apache Software Foundation (ASF) under one
+ * * or more contributor license agreements. See the NOTICE file
+ * * distributed with this work for additional information
+ * * regarding copyright ownership. The ASF licenses this file
+ * * to you under the Apache License, Version 2.0 (the
+ * * "License"); you may not use this file except in compliance
+ * * with the License. You may obtain a copy of the License at
+ * *
+ * * http://www.apache.org/licenses/LICENSE-2.0
+ * *
+ * * Unless required by applicable law or agreed to in writing,
+ * * software distributed under the License is distributed on an
+ * * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * * KIND, either express or implied. See the License for the
+ * * specific language governing permissions and limitations
+ * * under the License.
+ *
+ */
+
+package org.apache.tinkerpop.gremlin.structure.util.reference;
+
+import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
+import org.apache.tinkerpop.gremlin.FeatureRequirementSet;
+import org.apache.tinkerpop.gremlin.structure.Vertex;
+import org.apache.tinkerpop.gremlin.structure.VertexProperty;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+
+/**
+ * @author Marko A. Rodriguez (http://markorodriguez.com)
+ */
+public class ReferenceVertexPropertyTest extends AbstractGremlinTest {
+
+ @Test
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldNotConstructNewWithSomethingAlreadyReferenced() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty vp = v.property("test", "this");
+ final ReferenceVertexProperty dvp = ReferenceFactory.detach(vp);
+ assertSame(dvp, ReferenceFactory.detach(dvp));
+ }
+
+ @Test(expected = IllegalStateException.class)
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldNotSupportRemove() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty vp = v.property("test", "this");
+ ReferenceFactory.detach(vp).remove();
+ }
+
+ @Test
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldBeEqualsPropertiesAsIdIsTheSame() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty vp = v.property("test", "this");
+ final ReferenceVertexProperty vp1 = ReferenceFactory.detach(vp);
+ final ReferenceVertexProperty vp2 = ReferenceFactory.detach(vp);
+ assertTrue(vp1.equals(vp2));
+ assertTrue(vp1.equals(vp));
+ assertTrue(vp.equals(vp2));
+ }
+
+ @Test
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldNotBeEqualsPropertiesAsIdIsDifferent() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty vp1 = v.property("test", "this");
+ final ReferenceVertexProperty mp1 = ReferenceFactory.detach(vp1);
+ final VertexProperty vp2 = v.property("testing", "this");
+ final ReferenceVertexProperty mp2 = ReferenceFactory.detach(vp2);
+ assertFalse(mp1.equals(mp2));
+ }
+
+ @Test
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldAttachToGraph() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty toReference = v.property("test", "this");
+ final ReferenceVertexProperty rvp = ReferenceFactory.detach(toReference);
+ final VertexProperty referenced = rvp.attach(graph);
+
+ assertEquals(toReference, referenced);
+ assertFalse(referenced instanceof ReferenceVertexProperty);
+ }
+
+ @Test
+ @FeatureRequirementSet(FeatureRequirementSet.Package.SIMPLE)
+ public void shouldAttachToVertex() {
+ final Vertex v = graph.addVertex();
+ final VertexProperty toReference = v.property("test", "this");
+ final ReferenceVertexProperty rvp = ReferenceFactory.detach(toReference);
+ final VertexProperty referenced = rvp.attach(v);
+
+ assertEquals(toReference, referenced);
+ assertEquals(toReference.getClass(), referenced.getClass());
+ assertFalse(referenced instanceof ReferenceVertexProperty);
+ }
+}
[2/9] incubator-tinkerpop git commit: minimized object creation in
StarGraph.addTo(). Minor reduction in object creation in SparkExecutor.
Posted by sp...@apache.org.
minimized object creation in StarGraph.addTo(). Minor reduction in object creation in SparkExecutor.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ec99f516
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ec99f516
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ec99f516
Branch: refs/heads/TINKERPOP3-581
Commit: ec99f516ea1fa838c240e5ea11efaf7a0e2a4644
Parents: 302f012
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 17:21:25 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 17:21:25 2015 -0600
----------------------------------------------------------------------
.../gremlin/structure/util/star/StarGraph.java | 26 ++++++--------------
.../process/computer/spark/SparkExecutor.java | 7 +++---
.../computer/spark/SparkGraphComputer.java | 4 +--
.../process/computer/spark/SparkMessenger.java | 3 ++-
4 files changed, 15 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ec99f516/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java
index 335bd16..654332a 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java
@@ -164,32 +164,20 @@ public final class StarGraph implements Graph {
public static Vertex addTo(final StarGraph graph, final DetachedVertex detachedVertex) {
if (null != graph.starVertex)
return null;
-
graph.addVertex(T.id, detachedVertex.id(), T.label, detachedVertex.label());
detachedVertex.properties().forEachRemaining(detachedVertexProperty -> {
- final List<Object> keyValues = new ArrayList<>();
- keyValues.add(T.id);
- keyValues.add(detachedVertexProperty.id());
- detachedVertexProperty.properties().forEachRemaining(detachedVertexPropertyProperty -> {
- keyValues.add(detachedVertexPropertyProperty.key());
- keyValues.add(detachedVertexPropertyProperty.value());
- });
- graph.starVertex.property(VertexProperty.Cardinality.list, detachedVertexProperty.key(), detachedVertexProperty.value(), keyValues.toArray());
+ final VertexProperty<?> starVertexProperty = graph.starVertex.property(VertexProperty.Cardinality.list, detachedVertexProperty.key(), detachedVertexProperty.value(), T.id, detachedVertexProperty.id());
+ detachedVertexProperty.properties().forEachRemaining(detachedVertexPropertyProperty -> starVertexProperty.property(detachedVertexPropertyProperty.key(), detachedVertexPropertyProperty.value()));
});
return graph.starVertex;
}
public static Edge addTo(final StarGraph graph, final DetachedEdge edge) {
- final List<Object> keyValues = new ArrayList<>();
- keyValues.add(T.id);
- keyValues.add(edge.id());
- edge.properties().forEachRemaining(property -> {
- keyValues.add(property.key());
- keyValues.add(property.value());
- });
- return !graph.starVertex.id().equals(edge.inVertex().id()) ?
- graph.starVertex.addOutEdge(edge.label(), edge.inVertex(), keyValues.toArray()) :
- graph.starVertex.addInEdge(edge.label(), edge.outVertex(), keyValues.toArray());
+ final Edge starEdge = !graph.starVertex.id().equals(edge.inVertex().id()) ?
+ graph.starVertex.addOutEdge(edge.label(), edge.inVertex(), T.id, edge.id()) :
+ graph.starVertex.addInEdge(edge.label(), edge.outVertex(), T.id, edge.id());
+ edge.properties().forEachRemaining(property -> starEdge.property(property.key(), property.value()));
+ return starEdge;
}
protected Long generateId() {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ec99f516/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
index af5a9ff..f493c23 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
@@ -79,6 +79,7 @@ public final class SparkExecutor {
final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
final Set<String> elementComputeKeys = workerVertexProgram.getElementComputeKeys(); // the compute keys as a set
final String[] elementComputeKeysArray = elementComputeKeys.size() == 0 ? null : elementComputeKeys.toArray(new String[elementComputeKeys.size()]); // the compute keys as an array
+ final SparkMessenger<M> messenger = new SparkMessenger<>();
workerVertexProgram.workerIterationStart(memory); // start the worker
return () -> IteratorUtils.map(partitionIterator, vertexViewIncoming -> {
final Vertex vertex = vertexViewIncoming._2()._1().get(); // get the vertex from the vertex writable
@@ -87,7 +88,7 @@ public final class SparkExecutor {
final List<M> incomingMessages = hasViewAndMessages ? vertexViewIncoming._2()._2().get().getIncomingMessages() : Collections.emptyList();
previousView.forEach(property -> DetachedVertexProperty.addTo(vertex, property)); // attach the view to the vertex
///
- final SparkMessenger<M> messenger = new SparkMessenger<>(vertex, incomingMessages); // create the messenger with the incoming messages
+ messenger.setVertexAndIncomingMessages(vertex, incomingMessages); // set the messenger with the incoming messages
workerVertexProgram.execute(ComputerGraph.of(vertex, elementComputeKeys), messenger, memory); // execute the vertex program on this vertex for this iteration
///
final List<DetachedVertexProperty<Object>> nextView = null == elementComputeKeysArray ? // not all vertex programs have compute keys
@@ -98,7 +99,7 @@ public final class SparkExecutor {
workerVertexProgram.workerIterationEnd(memory); // if no more vertices in the partition, end the worker's iteration
return new Tuple2<>(vertex.id(), new ViewOutgoingPayload<>(nextView, outgoingMessages));
});
- })).setName("viewOutgoingRDD");
+ }));
// "message pass" by reducing on the vertex object id of the view and message payloads
final MessageCombiner<M> messageCombiner = VertexProgram.<VertexProgram<M>>createVertexProgram(apacheConfiguration).getMessageCombiner().orElse(null);
@@ -126,7 +127,7 @@ public final class SparkExecutor {
(ViewIncomingPayload<M>) payload : // this happens if there is a vertex with incoming messages
new ViewIncomingPayload<>((ViewPayload) payload)); // this happens if there is a vertex with no incoming messages
- newViewIncomingRDD.setName("viewIncomingRDD")
+ newViewIncomingRDD
.foreachPartition(partitionIterator -> {
}); // need to complete a task so its BSP and the memory for this iteration is updated
return newViewIncomingRDD;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ec99f516/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 4b063bb..0c979a7 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -211,10 +211,10 @@ public final class SparkGraphComputer implements GraphComputer {
final HadoopConfiguration newApacheConfiguration = new HadoopConfiguration(apacheConfiguration);
mapReduce.storeState(newApacheConfiguration);
// map
- final JavaPairRDD mapRDD = SparkExecutor.executeMap((JavaPairRDD) mapReduceGraphRDD, mapReduce, newApacheConfiguration).setName("mapRDD");
+ final JavaPairRDD mapRDD = SparkExecutor.executeMap((JavaPairRDD) mapReduceGraphRDD, mapReduce, newApacheConfiguration);
// combine TODO? is this really needed
// reduce
- final JavaPairRDD reduceRDD = (mapReduce.doStage(MapReduce.Stage.REDUCE)) ? SparkExecutor.executeReduce(mapRDD, mapReduce, newApacheConfiguration).setName("reduceRDD") : null;
+ final JavaPairRDD reduceRDD = (mapReduce.doStage(MapReduce.Stage.REDUCE)) ? SparkExecutor.executeReduce(mapRDD, mapReduce, newApacheConfiguration) : null;
// write the map reduce output back to disk (memory)
SparkExecutor.saveMapReduceRDD(null == reduceRDD ? mapRDD : reduceRDD, mapReduce, finalMemory, hadoopConfiguration);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ec99f516/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
index 5d9223c..f3f8e33 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
@@ -42,9 +42,10 @@ public final class SparkMessenger<M> implements Messenger<M> {
private Iterable<M> incomingMessages;
private final List<Tuple2<Object, M>> outgoingMessages = new ArrayList<>();
- public SparkMessenger(final Vertex vertex, final Iterable<M> incomingMessages) {
+ public void setVertexAndIncomingMessages(final Vertex vertex, final Iterable<M> incomingMessages) {
this.vertex = vertex;
this.incomingMessages = incomingMessages;
+ this.outgoingMessages.clear();
}
public List<Tuple2<Object, M>> getOutgoingMessages() {
[4/9] incubator-tinkerpop git commit: registered the Gryo classes
with SparkGraphComputer at construction time.
Posted by sp...@apache.org.
registered the Gryo classes with SparkGraphComputer at construction time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5b708e45
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5b708e45
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5b708e45
Branch: refs/heads/TINKERPOP3-581
Commit: 5b708e45f4d807f0b06911b34f2b2f036ab45641
Parents: 5b7ad6f
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 17:45:33 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 17:45:33 2015 -0600
----------------------------------------------------------------------
.../apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java | 4 ++++
.../hadoop/process/computer/spark/SparkGraphComputer.java | 5 +++++
2 files changed, 9 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b708e45/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
index 17b09e5..5ab18e6 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/io/gryo/GryoMapper.java
@@ -141,6 +141,10 @@ public final class GryoMapper implements Mapper<Kryo> {
return headerReader;
}
+ public List<Class> getRegisteredClasses() {
+ return this.serializationList.stream().map(Triplet::getValue0).collect(Collectors.toList());
+ }
+
/**
* Gets the header for a Gremlin Kryo file, which is based on the version of Gremlin Kryo that is constructed
* via the builder classes.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b708e45/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 0c979a7..5eb353d 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -43,6 +43,7 @@ import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.DefaultComputerResult;
import org.apache.tinkerpop.gremlin.process.computer.util.GraphComputerHelper;
import org.apache.tinkerpop.gremlin.process.computer.util.MapMemory;
+import org.apache.tinkerpop.gremlin.structure.io.gryo.GryoMapper;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -50,6 +51,7 @@ import scala.Tuple2;
import java.io.File;
import java.util.HashSet;
+import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.CompletableFuture;
@@ -150,6 +152,9 @@ public final class SparkGraphComputer implements GraphComputer {
// wire up a spark context
final SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]");
+ final List<Class> classes = GryoMapper.build().create().getRegisteredClasses();
+ sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()]));
+
hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class)))
hadoopConfiguration.set(Constants.MAPRED_INPUT_DIR, SparkExecutor.getInputLocation(hadoopConfiguration)); // necessary for Spark and newAPIHadoopRDD
[9/9] incubator-tinkerpop git commit: In this branch,
g.V() doesn't accept a Traverser so correct syntax of performance
test.
Posted by sp...@apache.org.
In this branch, g.V() doesn't accept a Traverser so correct syntax of performance test.
Not sure if g.V() should accept Traverser. Unclear how this was working in master. It does show that more tests are necessary in standard process tests.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/78d27103
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/78d27103
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/78d27103
Branch: refs/heads/TINKERPOP3-581
Commit: 78d27103f0671ea2f3ecaa03366e5f9626971e2a
Parents: ef5f8f0
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 10 09:29:06 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 10 09:29:06 2015 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/process/TraversalPerformanceTest.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/78d27103/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/TraversalPerformanceTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/TraversalPerformanceTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/TraversalPerformanceTest.java
index 2953464..0606409 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/TraversalPerformanceTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/TraversalPerformanceTest.java
@@ -101,7 +101,8 @@ public class TraversalPerformanceTest extends AbstractGremlinTest {
@LoadGraphWith(LoadGraphWith.GraphData.GRATEFUL)
@Test
public void g_V_out_mapXout_out_valuesXnameX_toListX() throws Exception {
- g.V().out().map(v -> g.V(v).out().out().values("name").toList()).iterate();
+ // todo: this used to be g.V().out().map(v -> g.V(v).out().out().values("name").toList()).iterate(); - should g.V() accept Traverser??
+ g.V().out().map(v -> g.V(v.get()).out().out().values("name").toList()).iterate();
}
@BenchmarkOptions(benchmarkRounds = DEFAULT_BENCHMARK_ROUNDS, warmupRounds = DEFAULT_WARMUP_ROUNDS, concurrency = BenchmarkOptions.CONCURRENCY_SEQUENTIAL)
[3/9] incubator-tinkerpop git commit: minor tweak to SparkMessenger.
Posted by sp...@apache.org.
minor tweak to SparkMessenger.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/5b7ad6ff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/5b7ad6ff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/5b7ad6ff
Branch: refs/heads/TINKERPOP3-581
Commit: 5b7ad6ff694b161ef1dd7375858500f9f853ec6f
Parents: ec99f51
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 17:27:24 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 17:27:24 2015 -0600
----------------------------------------------------------------------
.../gremlin/hadoop/process/computer/spark/SparkMessenger.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/5b7ad6ff/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
index f3f8e33..01153a1 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkMessenger.java
@@ -40,12 +40,12 @@ public final class SparkMessenger<M> implements Messenger<M> {
private Vertex vertex;
private Iterable<M> incomingMessages;
- private final List<Tuple2<Object, M>> outgoingMessages = new ArrayList<>();
+ private List<Tuple2<Object, M>> outgoingMessages = new ArrayList<>();
public void setVertexAndIncomingMessages(final Vertex vertex, final Iterable<M> incomingMessages) {
this.vertex = vertex;
this.incomingMessages = incomingMessages;
- this.outgoingMessages.clear();
+ this.outgoingMessages = new ArrayList<>();
}
public List<Tuple2<Object, M>> getOutgoingMessages() {
[8/9] incubator-tinkerpop git commit: Improved error messaging in
exception of IdManager.
Posted by sp...@apache.org.
Improved error messaging in exception of IdManager.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/ef5f8f01
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/ef5f8f01
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/ef5f8f01
Branch: refs/heads/TINKERPOP3-581
Commit: ef5f8f01b44b0a4da640bf59e6a18c6a09d33c37
Parents: fe106ad
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 10 09:13:06 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 10 09:13:06 2015 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/ef5f8f01/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 339d677..9cc7501 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -495,7 +495,7 @@ public class TinkerGraph implements Graph {
else if (id instanceof String)
return Long.parseLong((String) id);
else
- throw new IllegalArgumentException("Expected an id that is convertible to Long");
+ throw new IllegalArgumentException(String.format("Expected an id that is convertible to Long but received %s", id.getClass()));
}
},
@@ -518,7 +518,7 @@ public class TinkerGraph implements Graph {
else if (id instanceof String)
return Integer.parseInt((String) id);
else
- throw new IllegalArgumentException("Expected an id that is convertible to Integer");
+ throw new IllegalArgumentException(String.format("Expected an id that is convertible to Integer but received %s", id.getClass()));
}
},
@@ -539,7 +539,7 @@ public class TinkerGraph implements Graph {
else if (id instanceof String)
return java.util.UUID.fromString((String) id);
else
- throw new IllegalArgumentException("Expected an id that is convertible to UUID");
+ throw new IllegalArgumentException(String.format("Expected an id that is convertible to UUID but received %s", id.getClass()));
}
},
[5/9] incubator-tinkerpop git commit: registered the Gryo classes
with SparkGraphComputer at construction time.
Posted by sp...@apache.org.
registered the Gryo classes with SparkGraphComputer at construction time.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/26e73444
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/26e73444
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/26e73444
Branch: refs/heads/TINKERPOP3-581
Commit: 26e734446e66c72a73aa722ca6a52af5b0b504fe
Parents: 5b708e4
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Thu Apr 9 17:56:05 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Thu Apr 9 17:56:05 2015 -0600
----------------------------------------------------------------------
hadoop-gremlin/conf/hadoop-gryo.properties | 2 ++
.../hadoop/process/computer/spark/SparkGraphComputer.java | 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26e73444/hadoop-gremlin/conf/hadoop-gryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-gryo.properties b/hadoop-gremlin/conf/hadoop-gryo.properties
index 1c2422a..428cb5c 100644
--- a/hadoop-gremlin/conf/hadoop-gryo.properties
+++ b/hadoop-gremlin/conf/hadoop-gryo.properties
@@ -47,5 +47,7 @@ giraph.maxMessagesInMemory=100000
spark.master=local[4]
spark.executor.memory=1g
spark.serializer=org.apache.spark.serializer.KryoSerializer
+# spark.kryo.registrationRequired=true
+# spark.storage.memoryFraction=0.3
# spark.eventLog.enabled=true
# spark.eventLog.dir=/tmp/spark-event-logs
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/26e73444/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
index 5eb353d..fc79c64 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkGraphComputer.java
@@ -152,8 +152,8 @@ public final class SparkGraphComputer implements GraphComputer {
// wire up a spark context
final SparkConf sparkConfiguration = new SparkConf();
sparkConfiguration.setAppName(Constants.GREMLIN_HADOOP_SPARK_JOB_PREFIX + (null == this.vertexProgram ? "No VertexProgram" : this.vertexProgram) + "[" + this.mapReducers + "]");
- final List<Class> classes = GryoMapper.build().create().getRegisteredClasses();
- sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()]));
+ //final List<Class> classes = GryoMapper.build().create().getRegisteredClasses();
+ //sparkConfiguration.registerKryoClasses(classes.toArray(new Class[classes.size()])); //TODO
hadoopConfiguration.forEach(entry -> sparkConfiguration.set(entry.getKey(), entry.getValue()));
if (FileInputFormat.class.isAssignableFrom(hadoopConfiguration.getClass(Constants.GREMLIN_HADOOP_GRAPH_INPUT_FORMAT, InputFormat.class)))
[7/9] incubator-tinkerpop git commit: Check for specific type in
IdManagers and return without additional conversion.
Posted by sp...@apache.org.
Check for specific type in IdManagers and return without additional conversion.
Seems to save a tick or two on the processing as performance tests seem to run more inline as a result.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/fe106adc
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/fe106adc
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/fe106adc
Branch: refs/heads/TINKERPOP3-581
Commit: fe106adcc69a708f8c99be9ce97f5617a84217fc
Parents: c3c1c01
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 10 08:50:47 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 10 08:50:47 2015 -0400
----------------------------------------------------------------------
.../tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/fe106adc/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
----------------------------------------------------------------------
diff --git a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
index 1d26304..339d677 100644
--- a/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
+++ b/tinkergraph-gremlin/src/main/java/org/apache/tinkerpop/gremlin/tinkergraph/structure/TinkerGraph.java
@@ -488,7 +488,9 @@ public class TinkerGraph implements Graph {
@Override
public Object convert(final Object id) {
- if (id instanceof Number)
+ if (id instanceof Long)
+ return id;
+ else if (id instanceof Number)
return ((Number) id).longValue();
else if (id instanceof String)
return Long.parseLong((String) id);
@@ -509,7 +511,9 @@ public class TinkerGraph implements Graph {
@Override
public Object convert(final Object id) {
- if (id instanceof Number)
+ if (id instanceof Integer)
+ return id;
+ else if (id instanceof Number)
return ((Number) id).intValue();
else if (id instanceof String)
return Integer.parseInt((String) id);
[6/9] incubator-tinkerpop git commit: Merge remote-tracking branch
'origin/master' into TINKERPOP3-581
Posted by sp...@apache.org.
Merge remote-tracking branch 'origin/master' into TINKERPOP3-581
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/c3c1c011
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/c3c1c011
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/c3c1c011
Branch: refs/heads/TINKERPOP3-581
Commit: c3c1c011671f7449e32e9f996999674858f4e7de
Parents: b67712a 26e7344
Author: Stephen Mallette <sp...@genoprime.com>
Authored: Fri Apr 10 06:55:33 2015 -0400
Committer: Stephen Mallette <sp...@genoprime.com>
Committed: Fri Apr 10 06:55:33 2015 -0400
----------------------------------------------------------------------
.../gremlin/structure/io/gryo/GryoMapper.java | 4 +
.../util/reference/ReferenceVertexProperty.java | 9 +-
.../gremlin/structure/util/star/StarGraph.java | 26 ++---
.../structure/StructureStandardSuite.java | 2 +
.../reference/ReferenceVertexPropertyTest.java | 101 +++++++++++++++++++
hadoop-gremlin/conf/hadoop-gryo.properties | 2 +
.../process/computer/spark/SparkExecutor.java | 7 +-
.../computer/spark/SparkGraphComputer.java | 9 +-
.../process/computer/spark/SparkMessenger.java | 5 +-
9 files changed, 137 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/c3c1c011/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/structure/util/star/StarGraph.java
----------------------------------------------------------------------