You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/02/20 16:52:08 UTC
incubator-tinkerpop git commit: added TraversalUtil.applyNullable()
so that we can have null children in traversals -- not IdentityTraversal for
speed purposes and ease of toString(). Fixed up naming and toString() of the
recently added Reducing steps an
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 101ace9d3 -> 08d3d7fb1
added TraversalUtil.applyNullable() so that we can have null children in traversals -- not IdentityTraversal for speed purposes and ease of toString(). Fixed up naming and toString() of the recently added Reducing steps and their MapReduce inners.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/08d3d7fb
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/08d3d7fb
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/08d3d7fb
Branch: refs/heads/master
Commit: 08d3d7fb1171e3efde60d458c34eb9806faa0086
Parents: 101ace9
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri Feb 20 08:52:05 2015 -0700
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri Feb 20 08:52:05 2015 -0700
----------------------------------------------------------------------
docs/src/implementations.asciidoc | 34 +++++++-----
.../traversal/step/map/CountGlobalStep.java | 10 ++--
.../traversal/step/map/GroupCountStep.java | 18 ++++--
.../graph/traversal/step/map/GroupStep.java | 58 ++++++++++++--------
.../graph/traversal/step/map/MaxGlobalStep.java | 20 +++----
.../traversal/step/map/MeanGlobalStep.java | 20 +++----
.../graph/traversal/step/map/MinGlobalStep.java | 20 +++----
.../graph/traversal/step/map/SumGlobalStep.java | 20 +++----
.../step/util/ReducingBarrierStep.java | 1 -
.../process/traversal/util/TraversalUtil.java | 8 +++
hadoop-gremlin/conf/hadoop-kryo.properties | 2 +-
11 files changed, 121 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/docs/src/implementations.asciidoc
----------------------------------------------------------------------
diff --git a/docs/src/implementations.asciidoc b/docs/src/implementations.asciidoc
index bb797d9..2911166 100644
--- a/docs/src/implementations.asciidoc
+++ b/docs/src/implementations.asciidoc
@@ -716,7 +716,7 @@ gremlin> g.V().count()
gremlin> g.V().out().out().values('name')
==>ripple
==>lop
-gremlin> g.V().group().by{it.value('name')[1]}.by{it.value('name')}.next()
+gremlin> g.V().group().by{it.value('name')[1]}.by('name').next()
==>a={marko=1, vadas=1}
==>e={peter=1}
==>i={ripple=1}
@@ -749,35 +749,41 @@ WARNING: Giraph uses a large number of Hadoop counters. The default for Hadoop i
WARNING: The maximum number of workers can be no larger than the number of map-slots in the Hadoop cluster minus 1. For example, if the Hadoop cluster has 4 map slots, then `giraph.maxWorkers` can not be larger than 3. One map-slot is reserved for the master compute node and all other slots can be allocated as workers to execute the VertexPrograms on the vertices of the graph.
[source,text]
-gremlin> :remote connect tinkerpop.hadoop ../../../hadoop-gremlin/conf/hadoop-kryo.properties
-==>hadoopgraph[kryoinputformat->kryooutputformat]
-gremlin> :> g.V().count()
-INFO org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[HadoopGraphStep(vertex), CountStep, SideEffectCapStep]
+gremlin> g.engine(computer)
+==>null
+gremlin> g.V().count()
+INFO org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[GraphStep(vertex), CountGlobalStep, ComputerResultStep]
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0037
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
-INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: CountMapReduce
+INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: CountGlobalMapReduce
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0038
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
==>6
-gremlin> :> g.V().out().out().values('name')
-INFO org.apache.tinkerpop.gremlin.giraph.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[HadoopGraphStep(vertex), VertexStep(OUT), VertexStep(OUT), ElementValueStep(name)]
+gremlin> g.V().out().out().values('name')
+INFO org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[GraphStep(vertex), VertexStep(OUT,vertex), VertexStep(OUT,vertex), PropertiesStep([name],value), ComputerResultStep]
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0031
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
-INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: TraversalResultMapReduce
+INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: TraverserMapReduce
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0032
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
==>ripple
==>lop
-gremlin> :> g.V().group('a').by{it.value('name')[1]}.by{it.value('name')}
-INFO org.apache.tinkerpop.gremlin.giraph.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[HadoopGraphStep(vertex), GroupByStep@a, SideEffectCapStep(a)]
+
+IMPORTANT: The examples above do not use lambdas (i.e. closures in Gremlin-Groovy). This makes the traversal serializable and thus, able to be distributed to all machines in the Hadoop cluster. If a lambda is required in a traversal, then the traversal must be sent as a `String` and compiled locally at each machine in the cluster. The following example demonstrates the `:remote` command which allows for submitting Gremlin traversals as a `String`.
+
+[source,text]
+gremlin> :remote connect tinkerpop.hadoop ../../../hadoop-gremlin/conf/hadoop-kryo.properties
+==>hadoopgraph[kryoinputformat->kryooutputformat]
+gremlin> :> g.V().group('a').by{it.value('name')[1]}.by('name')
+INFO org.apache.tinkerpop.gremlin.hadoop.process.computer.giraph.GiraphGraphComputer - HadoopGremlin(Giraph): TraversalVertexProgram[GraphStep(vertex), GroupStep(lambda,value(name)), ComputerResultStep]
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0039
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
-INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: GroupByMapReduce[a]
+INFO org.apache.tinkerpop.gremlin.hadoop.structure.HadoopGraph - HadoopGremlin: GroupStepMapReduce
INFO org.apache.hadoop.mapred.JobClient - Running job: job_201407281259_0040
INFO org.apache.hadoop.mapred.JobClient - map 0% reduce 0%
...
@@ -788,14 +794,12 @@ gremlin> result.memory.runtime
==>20356
gremlin> result.memory.keys()
==>a
-gremlin> result.memory.a
+gremlin> result.memory.get('~reducing')
==>a={marko=1, vadas=1}
==>e={peter=1}
==>i={ripple=1}
==>o={lop=1, josh=1}
-NOTE: Distributed graph computations on cluster-sized graphs can yield an enormous number of results. To be safe, Hadoop-Gremlin only returns up to 20 results to the console (with, of course, all the results being available in HDFS). If more traversal results are desired, access `result.memory().get('~traversers')`.
-
TIP: image:gremlin-sugar.png[width=50,float=left] For those wanting to use the <<sugar-plugin,SugarPlugin>> with their submitted traversal, do `:remote config useSugar true` as well as `:plugin use tinkerpop.sugar` at the start of the Gremlin Console session if it is not already activated.
The `TraversalSupplier1` class mentioned in the `hadoop-kryo.properties` file is distributed with Hadoop-Gremlin. It declares what Gremlin-Java traversal to execute on the loaded `HadoopGraph`. This is the means by which traversals can be programmatically executed in Hadoop-Gremlin.
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/CountGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/CountGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/CountGlobalStep.java
index e62153d..e54c96f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/CountGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/CountGlobalStep.java
@@ -58,7 +58,7 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
@Override
public MapReduce<MapReduce.NullObject, Long, MapReduce.NullObject, Long, Long> getMapReduce() {
- return CountMapReduce.instance();
+ return CountGlobalMapReduce.instance();
}
///////////
@@ -83,11 +83,11 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
///////////
- private static class CountMapReduce extends StaticMapReduce<MapReduce.NullObject, Long, MapReduce.NullObject, Long, Long> {
+ private static class CountGlobalMapReduce extends StaticMapReduce<MapReduce.NullObject, Long, MapReduce.NullObject, Long, Long> {
- private static CountMapReduce INSTANCE = new CountMapReduce();
+ private static CountGlobalMapReduce INSTANCE = new CountGlobalMapReduce();
- private CountMapReduce() {
+ private CountGlobalMapReduce() {
}
@@ -125,7 +125,7 @@ public final class CountGlobalStep<S> extends ReducingBarrierStep<S, Long> imple
return keyValues.hasNext() ? keyValues.next().getValue() : 0L;
}
- public static final CountMapReduce instance() {
+ public static final CountGlobalMapReduce instance() {
return INSTANCE;
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupCountStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupCountStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupCountStep.java
index c289b44..9f990c5 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupCountStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupCountStep.java
@@ -25,9 +25,9 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
import org.apache.tinkerpop.gremlin.process.graph.traversal.step.util.ReducingBarrierStep;
-import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.process.traverser.TraverserRequirement;
import org.apache.tinkerpop.gremlin.process.util.MapHelper;
@@ -49,7 +49,7 @@ import java.util.function.BiFunction;
*/
public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Long>> implements MapReducer, TraversalParent {
- private Traversal.Admin<S, E> groupTraversal = new IdentityTraversal<>();
+ private Traversal.Admin<S, E> groupTraversal = null;
public GroupCountStep(final Traversal.Admin traversal) {
super(traversal);
@@ -65,7 +65,7 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
@Override
public List<Traversal.Admin<S, E>> getLocalChildren() {
- return Collections.singletonList(this.groupTraversal);
+ return null == this.groupTraversal ? Collections.emptyList() : Collections.singletonList(this.groupTraversal);
}
@Override
@@ -81,7 +81,8 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
@Override
public GroupCountStep<S, E> clone() throws CloneNotSupportedException {
final GroupCountStep<S, E> clone = (GroupCountStep<S, E>) super.clone();
- clone.groupTraversal = this.integrateChild(this.groupTraversal.clone());
+ if (null != this.groupTraversal)
+ clone.groupTraversal = clone.integrateChild(this.groupTraversal.clone());
return clone;
}
@@ -89,12 +90,17 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
public Traverser<Map<E, Long>> processNextStart() {
if (this.byPass) {
final Traverser.Admin<S> traverser = this.starts.next();
- return traverser.asAdmin().split(TraversalUtil.apply(traverser, (Traversal.Admin<S, Map<E, Long>>) this.groupTraversal), this);
+ return traverser.asAdmin().split(TraversalUtil.applyNullable(traverser, (Traversal.Admin<S, Map<E, Long>>) this.groupTraversal), this);
} else {
return super.processNextStart();
}
}
+ @Override
+ public String toString() {
+ return TraversalHelper.makeStepString(this, this.groupTraversal);
+ }
+
///////////
private class GroupCountBiFunction implements BiFunction<Map<E, Long>, Traverser<S>, Map<E, Long>>, Serializable {
@@ -105,7 +111,7 @@ public final class GroupCountStep<S, E> extends ReducingBarrierStep<S, Map<E, Lo
@Override
public Map<E, Long> apply(final Map<E, Long> mutatingSeed, final Traverser<S> traverser) {
- MapHelper.incr(mutatingSeed, TraversalUtil.apply(traverser.get(), GroupCountStep.this.groupTraversal), traverser.bulk());
+ MapHelper.incr(mutatingSeed, TraversalUtil.applyNullable(traverser.asAdmin(), GroupCountStep.this.groupTraversal), traverser.bulk());
return mutatingSeed;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
index 3fd15fa..7c795f1 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/GroupStep.java
@@ -27,7 +27,6 @@ import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram;
import org.apache.tinkerpop.gremlin.process.graph.traversal.step.util.ReducingBarrierStep;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalMatrix;
-import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal;
import org.apache.tinkerpop.gremlin.process.traversal.step.MapReducer;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper;
@@ -39,6 +38,7 @@ import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.io.Serializable;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
@@ -56,19 +56,26 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
private char state = 'k';
- private Traversal.Admin<S, K> keyTraversal = new IdentityTraversal<>();
- private Traversal.Admin<S, V> valueTraversal = new IdentityTraversal<>();
+ private Traversal.Admin<S, K> keyTraversal = null;
+ private Traversal.Admin<S, V> valueTraversal = null;
private Traversal.Admin<Collection<V>, R> reduceTraversal = null;
public GroupStep(final Traversal.Admin traversal) {
super(traversal);
this.setSeedSupplier((Supplier) new GroupMapSupplier());
- this.setBiFunction((BiFunction) new GroupReducingBiFunction());
+ this.setBiFunction((BiFunction) new GroupBiFunction());
}
@Override
public <A, B> List<Traversal.Admin<A, B>> getLocalChildren() {
- return null == this.reduceTraversal ? (List) Arrays.asList(this.keyTraversal, this.valueTraversal) : (List) Arrays.asList(this.keyTraversal, this.valueTraversal, this.reduceTraversal);
+ final List<Traversal.Admin<A, B>> children = new ArrayList<>(3);
+ if (null != this.keyTraversal)
+ children.add((Traversal.Admin) this.keyTraversal);
+ if (null != this.valueTraversal)
+ children.add((Traversal.Admin) this.valueTraversal);
+ if (null != this.reduceTraversal)
+ children.add((Traversal.Admin) this.reduceTraversal);
+ return children;
}
public Traversal.Admin<Collection<V>, R> getReduceTraversal() {
@@ -99,8 +106,10 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
@Override
public GroupStep<S, K, V, R> clone() throws CloneNotSupportedException {
final GroupStep<S, K, V, R> clone = (GroupStep<S, K, V, R>) super.clone();
- clone.keyTraversal = clone.integrateChild(this.keyTraversal.clone());
- clone.valueTraversal = clone.integrateChild(this.valueTraversal.clone());
+ if (null != this.keyTraversal)
+ clone.keyTraversal = clone.integrateChild(this.keyTraversal.clone());
+ if (null != this.valueTraversal)
+ clone.valueTraversal = clone.integrateChild(this.valueTraversal.clone());
if (null != this.reduceTraversal)
clone.reduceTraversal = clone.integrateChild(this.reduceTraversal.clone());
return clone;
@@ -108,37 +117,42 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
@Override
public MapReduce<K, Collection<V>, K, R, Map<K, R>> getMapReduce() {
- return new GroupReducingMapReduce<>(this);
+ return new GroupMapReduce<>(this);
}
@Override
public Traverser<Map<K, R>> processNextStart() {
if (this.byPass) {
final Traverser.Admin<S> traverser = this.starts.next();
- final Object[] kvPair = new Object[]{TraversalUtil.apply(traverser, (Traversal.Admin<S, Map>) this.keyTraversal), TraversalUtil.apply(traverser, (Traversal.Admin<S, Map>) this.valueTraversal)};
+ final Object[] kvPair = new Object[]{TraversalUtil.applyNullable(traverser, (Traversal.Admin<S, Map>) this.keyTraversal), TraversalUtil.applyNullable(traverser, (Traversal.Admin<S, Map>) this.valueTraversal)};
return traverser.asAdmin().split(kvPair, (Step) this);
} else {
return super.processNextStart();
}
}
+ @Override
+ public String toString() {
+ return TraversalHelper.makeStepString(this, this.keyTraversal, this.valueTraversal, this.reduceTraversal);
+ }
+
///////////
- private class GroupReducingBiFunction implements BiFunction<Map<K, Collection<V>>, Traverser.Admin<S>, Map<K, Collection<V>>>, Serializable {
+ private class GroupBiFunction implements BiFunction<Map<K, Collection<V>>, Traverser.Admin<S>, Map<K, Collection<V>>>, Serializable {
- private GroupReducingBiFunction() {
+ private GroupBiFunction() {
}
@Override
public Map<K, Collection<V>> apply(final Map<K, Collection<V>> mutatingSeed, final Traverser.Admin<S> traverser) {
- this.doGroup(traverser, mutatingSeed, keyTraversal, valueTraversal);
+ this.doGroup(traverser, mutatingSeed, GroupStep.this.keyTraversal, GroupStep.this.valueTraversal);
return mutatingSeed;
}
private void doGroup(final Traverser.Admin<S> traverser, final Map<K, Collection<V>> groupMap, final Traversal.Admin<S, K> keyTraversal, final Traversal.Admin<S, V> valueTraversal) {
- final K key = TraversalUtil.apply(traverser, keyTraversal);
- final V value = TraversalUtil.apply(traverser, valueTraversal);
+ final K key = TraversalUtil.applyNullable(traverser, keyTraversal);
+ final V value = TraversalUtil.applyNullable(traverser, valueTraversal);
Collection<V> values = groupMap.get(key);
if (null == values) {
values = new BulkSet<>();
@@ -154,11 +168,11 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
@Override
public Map<K, R> getFinal() {
- if (null == reduceTraversal)
+ if (null == GroupStep.this.reduceTraversal)
return (Map<K, R>) this;
else {
final Map<K, R> reduceMap = new HashMap<>();
- this.forEach((k, vv) -> reduceMap.put(k, TraversalUtil.apply(vv, reduceTraversal)));
+ this.forEach((k, vv) -> reduceMap.put(k, TraversalUtil.applyNullable(vv, GroupStep.this.reduceTraversal)));
return reduceMap;
}
}
@@ -177,18 +191,18 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
///////////
- public static final class GroupReducingMapReduce<K, V, R> implements MapReduce<K, Collection<V>, K, R, Map<K, R>> {
+ public static final class GroupMapReduce<K, V, R> implements MapReduce<K, Collection<V>, K, R, Map<K, R>> {
public static final String GROUP_BY_STEP_STEP_ID = "gremlin.groupStep.stepId";
private String groupStepId;
private Traversal.Admin<Collection<V>, R> reduceTraversal;
- private GroupReducingMapReduce() {
+ private GroupMapReduce() {
}
- public GroupReducingMapReduce(final GroupStep step) {
+ public GroupMapReduce(final GroupStep<?, K, V, R> step) {
this.groupStepId = step.getId();
this.reduceTraversal = step.getReduceTraversal();
}
@@ -226,7 +240,7 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
public void reduce(final K key, final Iterator<Collection<V>> values, final ReduceEmitter<K, R> emitter) {
final Set<V> set = new BulkSet<>();
values.forEachRemaining(set::addAll);
- emitter.emit(key, null == this.reduceTraversal ? (R) set : TraversalUtil.apply(set, this.reduceTraversal));
+ emitter.emit(key, TraversalUtil.applyNullable(set, this.reduceTraversal));
}
@Override
@@ -242,8 +256,8 @@ public final class GroupStep<S, K, V, R> extends ReducingBarrierStep<S, Map<K, R
}
@Override
- public GroupReducingMapReduce<K, V, R> clone() throws CloneNotSupportedException {
- final GroupReducingMapReduce<K, V, R> clone = (GroupReducingMapReduce<K, V, R>) super.clone();
+ public GroupMapReduce<K, V, R> clone() throws CloneNotSupportedException {
+ final GroupMapReduce<K, V, R> clone = (GroupMapReduce<K, V, R>) super.clone();
if (null != clone.reduceTraversal)
clone.reduceTraversal = this.reduceTraversal.clone();
return clone;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MaxGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MaxGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MaxGlobalStep.java
index 1cefcf5..c6990cc 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MaxGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MaxGlobalStep.java
@@ -45,7 +45,7 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
public MaxGlobalStep(final Traversal.Admin traversal) {
super(traversal);
this.setSeedSupplier(new ConstantSupplier<>(null));
- this.setBiFunction(MaxBiFunction.<S>instance());
+ this.setBiFunction(MaxGlobalBiFunction.<S>instance());
}
@Override
@@ -55,16 +55,16 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public MapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> getMapReduce() {
- return MaxMapReduce.instance();
+ return MaxGlobalMapReduce.instance();
}
/////
- private static class MaxBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
+ private static class MaxGlobalBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
- private static final MaxBiFunction INSTANCE = new MaxBiFunction();
+ private static final MaxGlobalBiFunction INSTANCE = new MaxGlobalBiFunction();
- private MaxBiFunction() {
+ private MaxGlobalBiFunction() {
}
@@ -73,18 +73,18 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
return mutatingSeed != null && mutatingSeed.doubleValue() >= traverser.get().doubleValue() ? mutatingSeed : traverser.get();
}
- public final static <S extends Number> MaxBiFunction<S> instance() {
+ public final static <S extends Number> MaxGlobalBiFunction<S> instance() {
return INSTANCE;
}
}
///////////
- private static class MaxMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
+ private static class MaxGlobalMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
- private static final MaxMapReduce INSTANCE = new MaxMapReduce();
+ private static final MaxGlobalMapReduce INSTANCE = new MaxGlobalMapReduce();
- private MaxMapReduce() {
+ private MaxGlobalMapReduce() {
}
@@ -126,7 +126,7 @@ public final class MaxGlobalStep<S extends Number> extends ReducingBarrierStep<S
return keyValues.hasNext() ? keyValues.next().getValue() : Double.NaN;
}
- public static final MaxMapReduce instance() {
+ public static final MaxGlobalMapReduce instance() {
return INSTANCE;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MeanGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MeanGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MeanGlobalStep.java
index e069b88..51fc509 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MeanGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MeanGlobalStep.java
@@ -49,7 +49,7 @@ public final class MeanGlobalStep<S extends Number, E extends Number> extends Re
public MeanGlobalStep(final Traversal.Admin traversal) {
super(traversal);
this.setSeedSupplier((Supplier) MeanNumberSupplier.instance());
- this.setBiFunction((BiFunction) MeanBiFunction.instance());
+ this.setBiFunction((BiFunction) MeanGlobalBiFunction.instance());
}
@Override
@@ -59,16 +59,16 @@ public final class MeanGlobalStep<S extends Number, E extends Number> extends Re
@Override
public MapReduce<Number, Long, Number, Long, Double> getMapReduce() {
- return MeanMapReduce.instance();
+ return MeanGlobalMapReduce.instance();
}
/////
- private static class MeanBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
+ private static class MeanGlobalBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
- private static final MeanBiFunction INSTANCE = new MeanBiFunction();
+ private static final MeanGlobalBiFunction INSTANCE = new MeanGlobalBiFunction();
- private MeanBiFunction() {
+ private MeanGlobalBiFunction() {
}
@@ -77,18 +77,18 @@ public final class MeanGlobalStep<S extends Number, E extends Number> extends Re
return (S) ((MeanNumber) mutatingSeed).add(traverser.get(), traverser.bulk());
}
- public final static <S extends Number> MeanBiFunction<S> instance() {
+ public final static <S extends Number> MeanGlobalBiFunction<S> instance() {
return INSTANCE;
}
}
///////////
- private static final class MeanMapReduce extends StaticMapReduce<Number, Long, Number, Long, Double> {
+ private static final class MeanGlobalMapReduce extends StaticMapReduce<Number, Long, Number, Long, Double> {
- private static final MeanMapReduce INSTANCE = new MeanMapReduce();
+ private static final MeanGlobalMapReduce INSTANCE = new MeanGlobalMapReduce();
- private MeanMapReduce() {
+ private MeanGlobalMapReduce() {
}
@@ -137,7 +137,7 @@ public final class MeanGlobalStep<S extends Number, E extends Number> extends Re
return Double.NaN;
}
- public static final MeanMapReduce instance() {
+ public static final MeanGlobalMapReduce instance() {
return INSTANCE;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MinGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MinGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MinGlobalStep.java
index 111fe45..999ed5e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MinGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/MinGlobalStep.java
@@ -45,7 +45,7 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
public MinGlobalStep(final Traversal.Admin traversal) {
super(traversal);
this.setSeedSupplier(new ConstantSupplier<>(null));
- this.setBiFunction(MinBiFunction.instance());
+ this.setBiFunction(MinGlobalBiFunction.instance());
}
@Override
@@ -55,16 +55,16 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
@Override
public MapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> getMapReduce() {
- return MinMapReduce.instance();
+ return MinGlobalMapReduce.instance();
}
/////
- private static class MinBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
+ private static class MinGlobalBiFunction<S extends Number> implements BiFunction<S, Traverser<S>, S>, Serializable {
- private static final MinBiFunction INSTANCE = new MinBiFunction();
+ private static final MinGlobalBiFunction INSTANCE = new MinGlobalBiFunction();
- private MinBiFunction() {
+ private MinGlobalBiFunction() {
}
@@ -73,18 +73,18 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
return mutatingSeed != null && mutatingSeed.doubleValue() <= traverser.get().doubleValue() ? mutatingSeed : traverser.get();
}
- public final static <S extends Number> MinBiFunction<S> instance() {
+ public final static <S extends Number> MinGlobalBiFunction<S> instance() {
return INSTANCE;
}
}
///////////
- private static class MinMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
+ private static class MinGlobalMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
- private static final MinMapReduce INSTANCE = new MinMapReduce();
+ private static final MinGlobalMapReduce INSTANCE = new MinGlobalMapReduce();
- private MinMapReduce() {
+ private MinGlobalMapReduce() {
}
@@ -126,7 +126,7 @@ public final class MinGlobalStep<S extends Number> extends ReducingBarrierStep<S
return keyValues.hasNext() ? keyValues.next().getValue() : Double.NaN;
}
- public static final MinMapReduce instance() {
+ public static final MinGlobalMapReduce instance() {
return INSTANCE;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/SumGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/SumGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/SumGlobalStep.java
index fe528da..920c18c 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/SumGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/map/SumGlobalStep.java
@@ -50,7 +50,7 @@ public final class SumGlobalStep extends ReducingBarrierStep<Number, Double> imp
public SumGlobalStep(final Traversal.Admin traversal) {
super(traversal);
this.setSeedSupplier(new ConstantSupplier<>(0.0d));
- this.setBiFunction(SumBiFunction.instance());
+ this.setBiFunction(SumGlobalBiFunction.instance());
}
@@ -61,16 +61,16 @@ public final class SumGlobalStep extends ReducingBarrierStep<Number, Double> imp
@Override
public MapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> getMapReduce() {
- return SumMapReduce.instance();
+ return SumGlobalMapReduce.instance();
}
/////
- private static class SumBiFunction<S extends Number> implements BiFunction<Double, Traverser<S>, Double>, Serializable {
+ private static class SumGlobalBiFunction<S extends Number> implements BiFunction<Double, Traverser<S>, Double>, Serializable {
- private static final SumBiFunction INSTANCE = new SumBiFunction();
+ private static final SumGlobalBiFunction INSTANCE = new SumGlobalBiFunction();
- private SumBiFunction() {
+ private SumGlobalBiFunction() {
}
@@ -79,18 +79,18 @@ public final class SumGlobalStep extends ReducingBarrierStep<Number, Double> imp
return mutatingSeed + (traverser.get().doubleValue() * traverser.bulk());
}
- public final static <S extends Number> SumBiFunction<S> instance() {
+ public final static <S extends Number> SumGlobalBiFunction<S> instance() {
return INSTANCE;
}
}
///////////
- private static class SumMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
+ private static class SumGlobalMapReduce extends StaticMapReduce<MapReduce.NullObject, Number, MapReduce.NullObject, Number, Number> {
- private static final SumMapReduce INSTANCE = new SumMapReduce();
+ private static final SumGlobalMapReduce INSTANCE = new SumGlobalMapReduce();
- private SumMapReduce() {
+ private SumGlobalMapReduce() {
}
@@ -130,7 +130,7 @@ public final class SumGlobalStep extends ReducingBarrierStep<Number, Double> imp
return keyValues.hasNext() ? keyValues.next().getValue() : 0.0d;
}
- public static final SumMapReduce instance() {
+ public static final SumGlobalMapReduce instance() {
return INSTANCE;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/util/ReducingBarrierStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/util/ReducingBarrierStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/util/ReducingBarrierStep.java
index 1c2466e..4255ec2 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/util/ReducingBarrierStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/graph/traversal/step/util/ReducingBarrierStep.java
@@ -164,7 +164,6 @@ public abstract class ReducingBarrierStep<S, E> extends AbstractStep<S, E> imple
}
emitter.emit(FinalGet.tryFinalGet(mutatingSeed));
}
-
}
/////
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalUtil.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalUtil.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalUtil.java
index 89a6703..7c3d74e 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalUtil.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/util/TraversalUtil.java
@@ -44,6 +44,10 @@ public final class TraversalUtil {
}
}
+ public static final <S, E> E applyNullable(final Traverser.Admin<S> traverser, final Traversal.Admin<S, E> traversal) {
+ return null == traversal ? (E) traverser.get() : TraversalUtil.apply(traverser, traversal);
+ }
+
public static final <S, E> boolean test(final Traverser.Admin<S> traverser, final Traversal.Admin<S, E> traversal) {
final Traverser.Admin<S> split = traverser.split();
split.setSideEffects(traversal.getSideEffects());
@@ -65,6 +69,10 @@ public final class TraversalUtil {
}
}
+ public static final <S, E> E applyNullable(final S start, final Traversal.Admin<S, E> traversal) {
+ return null == traversal ? (E) start : TraversalUtil.apply(start, traversal);
+ }
+
public static final <S, E> boolean test(final S start, final Traversal.Admin<S, E> traversal) {
traversal.reset();
traversal.addStart(traversal.getTraverserGenerator().generate(start, traversal.getStartStep(), 1l));
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/08d3d7fb/hadoop-gremlin/conf/hadoop-kryo.properties
----------------------------------------------------------------------
diff --git a/hadoop-gremlin/conf/hadoop-kryo.properties b/hadoop-gremlin/conf/hadoop-kryo.properties
index f6e4539..d546da7 100644
--- a/hadoop-gremlin/conf/hadoop-kryo.properties
+++ b/hadoop-gremlin/conf/hadoop-kryo.properties
@@ -21,7 +21,7 @@ gremlin.hadoop.memoryOutputFormat=org.apache.hadoop.mapreduce.lib.output.Sequenc
gremlin.hadoop.deriveMemory=false
gremlin.hadoop.jarsInDistributedCache=true
-gremlin.hadoop.inputLocation=tinkergraph-modern-vertices.gio
+gremlin.hadoop.inputLocation=tinkerpop-modern-vertices.gio
gremlin.hadoop.outputLocation=output
#gremlin.vertexProgram=org.apache.tinkerpop.gremlin.process.computer.traversal.TraversalVertexProgram
#gremlin.traversalVertexProgram.traversalSupplier.type=CLASS