You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/05/18 20:56:23 UTC
[1/2] incubator-tinkerpop git commit: Okay -- made the Gryo pool
intialization a bit more sturdy in both Giraph and Spark. Removed new
Consumer() in ProfileTest as its just a lambda.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 6f1907806 -> 39cf09220
Okay -- made the Gryo pool intialization a bit more sturdy in both Giraph and Spark. Removed new Consumer() in ProfileTest as its just a lambda.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/39cf0922
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/39cf0922
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/39cf0922
Branch: refs/heads/master
Commit: 39cf092209bf36c3b4867dd92cfd2288ea72a458
Parents: a843ccf
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Mon May 18 12:56:27 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon May 18 12:56:34 2015 -0600
----------------------------------------------------------------------
.../traversal/step/sideEffect/ProfileTest.java | 27 ++++++++------------
.../hadoop/process/computer/HadoopCombine.java | 2 ++
.../hadoop/process/computer/HadoopMap.java | 2 ++
.../hadoop/process/computer/HadoopReduce.java | 2 ++
.../computer/giraph/GiraphWorkerContext.java | 2 ++
.../process/computer/spark/SparkExecutor.java | 7 ++++-
6 files changed, 24 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
index 12e62ba..d21b839 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
@@ -41,7 +41,6 @@ import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.TimeUnit;
-import java.util.function.Consumer;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.GRATEFUL;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
@@ -331,23 +330,17 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Override
public Traversal<Vertex, StandardTraversalMetrics> get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile() {
- return (Traversal) g.V().sideEffect(new Consumer<Traverser<Vertex>>() {
- @Override
- public void accept(final Traverser<Vertex> vertexTraverser) {
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ return (Traversal) g.V().sideEffect(v -> {
+ try {
+ Thread.sleep(10);
+ } catch (final InterruptedException e) {
+ e.printStackTrace();
}
- }).sideEffect(new Consumer<Traverser<Vertex>>() {
- @Override
- public void accept(final Traverser<Vertex> vertexTraverser) {
- try {
- Thread.sleep(5);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
+ }).sideEffect(v -> {
+ try {
+ Thread.sleep(5);
+ } catch (final InterruptedException e) {
+ e.printStackTrace();
}
}).profile();
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
index 7c0b985..bbcf524 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopCombine.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.hadoop.process.computer;
import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
@@ -46,6 +47,7 @@ public class HadoopCombine extends Reducer<ObjectWritable, ObjectWritable, Objec
@Override
public void setup(final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.COMBINE);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
index def0753..a0a6c00 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopMap.java
@@ -22,6 +22,7 @@ import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
@@ -49,6 +50,7 @@ public class HadoopMap extends Mapper<NullWritable, VertexWritable, ObjectWritab
@Override
public void setup(final Mapper<NullWritable, VertexWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.MAP);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
index 66a14e3..ba88339 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/HadoopReduce.java
@@ -21,6 +21,7 @@ package org.apache.tinkerpop.gremlin.hadoop.process.computer;
import org.apache.commons.configuration.Configuration;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
@@ -46,6 +47,7 @@ public class HadoopReduce extends Reducer<ObjectWritable, ObjectWritable, Object
@Override
public void setup(final Reducer<ObjectWritable, ObjectWritable, ObjectWritable, ObjectWritable>.Context context) {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(context.getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
this.mapReduce = MapReduce.createMapReduce(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.mapReduce.workerStart(MapReduce.Stage.REDUCE);
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
index 0547aef..cd42fa0 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/giraph/GiraphWorkerContext.java
@@ -23,6 +23,7 @@ import org.apache.giraph.conf.GiraphConstants;
import org.apache.giraph.worker.WorkerContext;
import org.apache.tinkerpop.gremlin.hadoop.Constants;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.util.ConfUtil;
import org.apache.tinkerpop.gremlin.process.computer.VertexProgram;
@@ -46,6 +47,7 @@ public final class GiraphWorkerContext extends WorkerContext {
public void preApplication() throws InstantiationException, IllegalAccessException {
final Configuration apacheConfiguration = ConfUtil.makeApacheConfiguration(this.getContext().getConfiguration());
+ HadoopPools.initialize(apacheConfiguration);
final VertexProgram vertexProgram = VertexProgram.createVertexProgram(HadoopGraph.open(apacheConfiguration), apacheConfiguration);
this.vertexProgramPool = new VertexProgramPool(vertexProgram, this.getContext().getConfiguration().getInt(GiraphConstants.NUM_COMPUTE_THREADS.getKey(), 1));
this.memory = new GiraphMemory(this, vertexProgram);
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/39cf0922/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
index afa8dc5..1b70fa6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/process/computer/spark/SparkExecutor.java
@@ -33,6 +33,7 @@ import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewIn
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewOutgoingPayload;
import org.apache.tinkerpop.gremlin.hadoop.process.computer.spark.payload.ViewPayload;
import org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph;
+import org.apache.tinkerpop.gremlin.hadoop.structure.io.HadoopPools;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritable;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.ObjectWritableIterator;
import org.apache.tinkerpop.gremlin.hadoop.structure.io.VertexWritable;
@@ -81,6 +82,7 @@ public final class SparkExecutor {
graphRDD.leftOuterJoin(viewIncomingRDD)) // every other iteration may have views and messages
// for each partition of vertices
.mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
final VertexProgram<M> workerVertexProgram = VertexProgram.<VertexProgram<M>>createVertexProgram(HadoopGraph.open(apacheConfiguration),apacheConfiguration); // each partition(Spark)/worker(TP3) has a local copy of the vertex program (a worker's task)
final Set<String> elementComputeKeys = workerVertexProgram.getElementComputeKeys(); // the compute keys as a set
final String[] elementComputeKeysArray = elementComputeKeys.size() == 0 ? EMPTY_ARRAY : elementComputeKeys.toArray(new String[elementComputeKeys.size()]); // the compute keys as an array
@@ -136,6 +138,7 @@ public final class SparkExecutor {
newViewIncomingRDD.setName("viewIncomingRDD")
.foreachPartition(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
}); // need to complete a task so its BSP and the memory for this iteration is updated
return newViewIncomingRDD;
}
@@ -147,7 +150,7 @@ public final class SparkExecutor {
public static <M> JavaPairRDD<Object, VertexWritable> prepareGraphRDDForMapReduce(final JavaPairRDD<Object, VertexWritable> graphRDD, final JavaPairRDD<Object, ViewIncomingPayload<M>> viewIncomingRDD, final String[] elementComputeKeys) {
return (null == viewIncomingRDD) ? // there was no vertex program
graphRDD.mapValues(vertexWritable -> {
- ((StarGraph.StarVertex) vertexWritable.get()).dropEdges();
+ vertexWritable.get().dropEdges();
return vertexWritable;
}) :
graphRDD.leftOuterJoin(viewIncomingRDD)
@@ -163,6 +166,7 @@ public final class SparkExecutor {
public static <K, V> JavaPairRDD<K, V> executeMap(final JavaPairRDD<Object, VertexWritable> graphRDD, final MapReduce<K, V, ?, ?, ?> mapReduce, final Configuration apacheConfiguration) {
JavaPairRDD<K, V> mapRDD = graphRDD.mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
final MapReduce<K, V, ?, ?, ?> workerMapReduce = MapReduce.<MapReduce<K, V, ?, ?, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration);
workerMapReduce.workerStart(MapReduce.Stage.MAP);
final SparkMapEmitter<K, V> mapEmitter = new SparkMapEmitter<>();
@@ -182,6 +186,7 @@ public final class SparkExecutor {
public static <K, V, OK, OV> JavaPairRDD<OK, OV> executeReduce(final JavaPairRDD<K, V> mapRDD, final MapReduce<K, V, OK, OV, ?> mapReduce, final Configuration apacheConfiguration) {
JavaPairRDD<OK, OV> reduceRDD = mapRDD.groupByKey().mapPartitionsToPair(partitionIterator -> {
+ HadoopPools.initialize(apacheConfiguration);
final MapReduce<K, V, OK, OV, ?> workerMapReduce = MapReduce.<MapReduce<K, V, OK, OV, ?>>createMapReduce(HadoopGraph.open(apacheConfiguration),apacheConfiguration);
workerMapReduce.workerStart(MapReduce.Stage.REDUCE);
final SparkReduceEmitter<OK, OV> reduceEmitter = new SparkReduceEmitter<>();
[2/2] incubator-tinkerpop git commit: Rename Profileable to
Profiling. Add a more generic ProfileTest case. Fix some ProfileTest method
names.
Posted by ok...@apache.org.
Rename Profileable to Profiling. Add a more generic ProfileTest case. Fix some ProfileTest method names.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/a843ccff
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/a843ccff
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/a843ccff
Branch: refs/heads/master
Commit: a843ccff5dfcb89c9ba4b9e5a75a4cbe26a26950
Parents: 6f19078
Author: rjbriody <bo...@datastax.com>
Authored: Mon May 18 14:10:47 2015 -0400
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Mon May 18 12:56:34 2015 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 +-
.../process/traversal/step/Profileable.java | 31 -----------
.../process/traversal/step/Profiling.java | 31 +++++++++++
.../traversal/step/sideEffect/ProfileStep.java | 6 +-
.../step/sideEffect/GroovyProfileTest.groovy | 3 +-
.../traversal/step/sideEffect/ProfileTest.java | 58 +++++++++++++++++---
.../gremlin/hadoop/structure/HadoopGraph.java | 4 +-
7 files changed, 87 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index ae54bff..3f6ba0c 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -102,7 +102,7 @@ TinkerPop 3.0.0.M9 (NOT OFFICIALLY RELEASED YET)
* `Attachable` now has a set of static exception messages in an `Exceptions` inner class.
* Added `StarGraph` which is a heap efficient representation of a vertex and its incident edges (useful for `GraphComputer` implementations).
* `TraverserSet` uses a `FastNoSuchElementException` on `remove()` for increased performance.
-* Add `Profileable` interface to enable vendors to receive a `Step's MutableMetrics`.
+* Add `Profiling` interface to enable vendors to receive a `Step's MutableMetrics`.
TinkerPop 3.0.0.M8 (Release Date: April 6, 2015)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profileable.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profileable.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profileable.java
deleted file mode 100644
index eb7a2f1..0000000
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profileable.java
+++ /dev/null
@@ -1,31 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.tinkerpop.gremlin.process.traversal.step;
-
-import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
-
-/**
- * A Step can implement this interface in order to receive a reference to the MutableMetrics object for the Step. The
- * MutableMetrics is initialized when the ProfileStrategy executes.
- *
- * @author Bob Briody (http://bobbriody.com)
- */
-public interface Profileable {
- public void setMetrics(final MutableMetrics metrics);
-}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profiling.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profiling.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profiling.java
new file mode 100644
index 0000000..3e4ff19
--- /dev/null
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/Profiling.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tinkerpop.gremlin.process.traversal.step;
+
+import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
+
+/**
+ * A Step can implement this interface in order to receive a reference to the MutableMetrics object for the Step. The
+ * MutableMetrics is initialized when the ProfileStrategy executes.
+ *
+ * @author Bob Briody (http://bobbriody.com)
+ */
+public interface Profiling {
+ public void setMetrics(final MutableMetrics metrics);
+}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileStep.java
index 3b94bbf..e42e5fa 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileStep.java
@@ -26,7 +26,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Profileable;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.AbstractStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
@@ -147,8 +147,8 @@ public final class ProfileStep<S> extends AbstractStep<S, S> implements MapReduc
prevMetrics = (DependantMutableMetrics) metrics;
}
- if (step instanceof Profileable) {
- ((Profileable) step).setMetrics(metrics);
+ if (step instanceof Profiling) {
+ ((Profiling) step).setMetrics(metrics);
}
// Initialize counters (necessary because some steps might end up being 0)
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyProfileTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyProfileTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyProfileTest.groovy
index b90657a..327c1dd 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyProfileTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/GroovyProfileTest.groovy
@@ -35,7 +35,6 @@ public abstract class GroovyProfileTest {
g.V.out.out.profile() // locked traversals
}
-
@Override
public Traversal<Vertex, StandardTraversalMetrics> get_g_V_repeat_both_profile() {
TraversalScriptHelper.compute("g.V.repeat(__.both()).times(3).profile()", g);
@@ -47,7 +46,7 @@ public abstract class GroovyProfileTest {
}
@Override
- Traversal<Vertex, StandardTraversalMetrics> get_g_V_sleep_sleep_profile() {
+ Traversal<Vertex, StandardTraversalMetrics> get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile() {
TraversalScriptHelper.compute("g.V().sideEffect{Thread.sleep(10)}.sideEffect{Thread.sleep(5)}.profile()", g)
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
index be7dbe0..12e62ba 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/sideEffect/ProfileTest.java
@@ -19,12 +19,15 @@
package org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect;
import org.apache.tinkerpop.gremlin.LoadGraphWith;
-import org.apache.tinkerpop.gremlin.process.*;
+import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
+import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
+import org.apache.tinkerpop.gremlin.process.IgnoreEngine;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__;
-import org.apache.tinkerpop.gremlin.process.traversal.step.Profileable;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Profiling;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.FlatMapStep;
import org.apache.tinkerpop.gremlin.process.traversal.util.Metrics;
import org.apache.tinkerpop.gremlin.process.traversal.util.MutableMetrics;
@@ -55,13 +58,50 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, StandardTraversalMetrics> get_g_V_repeat_both_profile();
- public abstract Traversal<Vertex, StandardTraversalMetrics> get_g_V_sleep_sleep_profile();
+ public abstract Traversal<Vertex, StandardTraversalMetrics> get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile();
public abstract Traversal<Vertex, StandardTraversalMetrics> get_g_V_whereXinXcreatedX_count_isX1XX_valuesXnameX_profile();
+ /**
+ * Many of the tests in this class are coupled to not-totally-generic vendor behavior. However, this test is intended to provide
+ * fully generic validation.
+ */
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_out_out_profile_simple() {
+ final Traversal<Vertex, StandardTraversalMetrics> traversal = get_g_V_out_out_profile();
+ printTraversalForm(traversal);
+
+ traversal.iterate();
+
+ final TraversalMetrics traversalMetrics = traversal.asAdmin().getSideEffects().<TraversalMetrics>get(TraversalMetrics.METRICS_KEY).get();
+ traversalMetrics.toString(); // ensure no exceptions are thrown
+
+ // Every other step should be a Profile step
+ List<Step> steps = traversal.asAdmin().getSteps();
+ for (int ii = 1; ii < steps.size(); ii += 2) {
+ assertEquals("Every other Step should be a ProfileStep.", ProfileStep.class, steps.get(ii).getClass());
+ }
+
+ // Validate the last Metrics only, which must be consistent across vendors.
+ Metrics metrics = traversalMetrics.getMetrics(traversalMetrics.getMetrics().size() - 1);
+ assertEquals(2, metrics.getCount(TraversalMetrics.ELEMENT_COUNT_ID).longValue());
+ assertNotEquals(0, metrics.getCount(TraversalMetrics.TRAVERSER_COUNT_ID).longValue());
+ assertTrue("Percent duration should be positive.", (Double) metrics.getAnnotation(TraversalMetrics.PERCENT_DURATION_KEY) > 0);
+ assertTrue("Times should be positive.", metrics.getDuration(TimeUnit.MICROSECONDS) > 0);
+
+ // Ensure durations sum to 100
+ double totalPercentDuration = 0;
+ for (Metrics m : traversalMetrics.getMetrics()) {
+ totalPercentDuration += (Double) m.getAnnotation(TraversalMetrics.PERCENT_DURATION_KEY);
+ }
+ assertEquals(100, totalPercentDuration, 0.000001);
+ }
+
+
@Test
@LoadGraphWith(MODERN)
- public void g_V_out_out_modern_profile() {
+ public void g_V_out_out_profile_modern() {
final Traversal<Vertex, StandardTraversalMetrics> traversal = get_g_V_out_out_profile();
printTraversalForm(traversal);
@@ -98,7 +138,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(GRATEFUL)
- public void g_V_out_out_grateful_profile() {
+ public void g_V_out_out_profile_grateful() {
final Traversal<Vertex, StandardTraversalMetrics> traversal = get_g_V_out_out_profile();
printTraversalForm(traversal);
@@ -134,8 +174,8 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
@IgnoreEngine(TraversalEngine.Type.COMPUTER)
- public void g_V_sleep_sleep_profile() {
- final Traversal<Vertex, StandardTraversalMetrics> traversal = get_g_V_sleep_sleep_profile();
+ public void g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile() {
+ final Traversal<Vertex, StandardTraversalMetrics> traversal = get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile();
printTraversalForm(traversal);
traversal.iterate();
@@ -242,7 +282,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
*/
// Setup a "mock" step to test the strategy
- static public class MockStep extends FlatMapStep<Vertex, Vertex> implements Profileable {
+ static public class MockStep extends FlatMapStep<Vertex, Vertex> implements Profiling {
public static boolean callbackCalled = false;
public MockStep(final Traversal.Admin traversal) {
@@ -290,7 +330,7 @@ public abstract class ProfileTest extends AbstractGremlinProcessTest {
}
@Override
- public Traversal<Vertex, StandardTraversalMetrics> get_g_V_sleep_sleep_profile() {
+ public Traversal<Vertex, StandardTraversalMetrics> get_g_V_sideEffectXThread_sleepX10XX_sideEffectXThread_sleepX5XX_profile() {
return (Traversal) g.V().sideEffect(new Consumer<Traverser<Vertex>>() {
@Override
public void accept(final Traverser<Vertex> vertexTraverser) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/a843ccff/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
index febd4c4..ed964b6 100644
--- a/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
+++ b/hadoop-gremlin/src/main/java/org/apache/tinkerpop/gremlin/hadoop/structure/HadoopGraph.java
@@ -116,11 +116,11 @@ import java.util.stream.Stream;
reason = "Giraph does a hard kill on failure and stops threads which stops test cases. Exception handling semantics are correct though.")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.ProfileTest$Traversals",
- method = "g_V_out_out_grateful_profile",
+ method = "g_V_out_out_profile_grateful",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
@Graph.OptOut(
test = "org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.GroovyProfileTest$Traversals",
- method = "g_V_out_out_grateful_profile",
+ method = "g_V_out_out_profile_grateful",
reason = "Hadoop-Gremlin is OLAP-oriented and for OLTP operations, linear-scan joins are required. This particular tests takes many minutes to execute.")
public class HadoopGraph implements Graph {