You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2015/05/22 21:01:08 UTC
incubator-tinkerpop git commit: Gremlin OLAP now supports
DedupGlobalStep if its an end step. Added two new tests to DedupTest to
verify behavior.
Repository: incubator-tinkerpop
Updated Branches:
refs/heads/master 63cccb9be -> 8eaa0c27a
Gremlin OLAP now supports DedupGlobalStep if its an end step. Added two new tests to DedupTest to verify behavior.
Project: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/commit/8eaa0c27
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/tree/8eaa0c27
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/diff/8eaa0c27
Branch: refs/heads/master
Commit: 8eaa0c27a454fa4824ea2a0a9a14197f5aa2808e
Parents: 63cccb9
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Fri May 22 13:00:28 2015 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Fri May 22 13:01:16 2015 -0600
----------------------------------------------------------------------
.../mapreduce/TraverserMapReduce.java | 13 ++++++-
.../traversal/step/filter/DedupGlobalStep.java | 12 +++++-
.../ComputerVerificationStrategy.java | 6 +--
.../step/filter/GroovyDedupTest.groovy | 32 ++++++----------
.../traversal/step/filter/DedupTest.java | 39 +++++++++++++++++++-
5 files changed, 73 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8eaa0c27/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
index e2446d6..14bb472 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/step/sideEffect/mapreduce/TraverserMapReduce.java
@@ -25,6 +25,7 @@ import org.apache.tinkerpop.gremlin.process.computer.util.StaticMapReduce;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
+import org.apache.tinkerpop.gremlin.process.traversal.step.filter.DedupGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TailGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderGlobalStep;
@@ -51,6 +52,7 @@ public final class TraverserMapReduce extends StaticMapReduce<Comparable, Traver
private Optional<CollectingBarrierStep<?>> collectingBarrierStep = Optional.empty();
private Optional<RangeGlobalStep<?>> rangeGlobalStep = Optional.empty();
private Optional<TailGlobalStep<?>> tailGlobalStep = Optional.empty();
+ private boolean dedupGlobal = false;
private TraverserMapReduce() {
}
@@ -75,11 +77,14 @@ public final class TraverserMapReduce extends StaticMapReduce<Comparable, Traver
this.rangeGlobalStep = Optional.of(((RangeGlobalStep) traversalEndStep).clone());
if (traversalEndStep instanceof TailGlobalStep)
this.tailGlobalStep = Optional.of(((TailGlobalStep) traversalEndStep).clone());
+ if (traversalEndStep instanceof DedupGlobalStep)
+ this.dedupGlobal = true;
+
}
@Override
public boolean doStage(final Stage stage) {
- return stage.equals(Stage.MAP) || this.collectingBarrierStep.isPresent() || this.rangeGlobalStep.isPresent() || this.tailGlobalStep.isPresent();
+ return stage.equals(Stage.MAP) || this.collectingBarrierStep.isPresent() || this.rangeGlobalStep.isPresent() || this.tailGlobalStep.isPresent() || this.dedupGlobal;
}
@Override
@@ -128,12 +133,16 @@ public final class TraverserMapReduce extends StaticMapReduce<Comparable, Traver
tailGlobalStep.setBypass(false);
tailGlobalStep.addStarts(IteratorUtils.map(keyValues, keyValue -> (Traverser) keyValue.getValue()));
return (Iterator) tailGlobalStep;
+ } else if (this.dedupGlobal) {
+ return IteratorUtils.map(keyValues, keyValue -> {
+ keyValue.getValue().asAdmin().setBulk(1l);
+ return keyValue.getValue();
+ });
} else {
return IteratorUtils.map(keyValues, KeyValue::getValue);
}
}
-
@Override
public String getMemoryKey() {
return TRAVERSERS;
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8eaa0c27/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
index c77f016..ffbd16d 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
@@ -21,9 +21,10 @@ package org.apache.tinkerpop.gremlin.process.traversal.step.filter;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.lambda.IdentityTraversal;
+import org.apache.tinkerpop.gremlin.process.traversal.step.Bypassing;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent;
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.TraverserRequirement;
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalUtil;
import org.apache.tinkerpop.gremlin.structure.util.StringFactory;
import java.util.Collections;
@@ -34,10 +35,11 @@ import java.util.Set;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
*/
-public final class DedupGlobalStep<S> extends FilterStep<S> implements TraversalParent {
+public final class DedupGlobalStep<S> extends FilterStep<S> implements TraversalParent, Bypassing {
private Traversal.Admin<S, Object> dedupTraversal = new IdentityTraversal<>();
private Set<Object> duplicateSet = new HashSet<>();
+ private boolean bypass = false;
public DedupGlobalStep(final Traversal.Admin traversal) {
super(traversal);
@@ -45,6 +47,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
@Override
protected boolean filter(final Traverser.Admin<S> traverser) {
+ if (this.bypass) return true;
traverser.setBulk(1);
return this.duplicateSet.add(TraversalUtil.apply(traverser, this.dedupTraversal));
}
@@ -83,4 +86,9 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
public Set<TraverserRequirement> getRequirements() {
return this.getSelfAndChildRequirements(TraverserRequirement.BULK);
}
+
+ @Override
+ public void setBypass(final boolean bypass) {
+ this.bypass = bypass;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8eaa0c27/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
index 11eac7a..8c893a7 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/strategy/verification/ComputerVerificationStrategy.java
@@ -58,7 +58,7 @@ public final class ComputerVerificationStrategy extends AbstractTraversalStrateg
private static final ComputerVerificationStrategy INSTANCE = new ComputerVerificationStrategy();
private static final Set<Class<?>> UNSUPPORTED_STEPS = new HashSet<>(Arrays.asList(
- DedupGlobalStep.class, InjectStep.class, MatchStep.class, Mutating.class, SubgraphStep.class
+ InjectStep.class, MatchStep.class, Mutating.class, SubgraphStep.class
));
private ComputerVerificationStrategy() {
@@ -86,12 +86,12 @@ public final class ComputerVerificationStrategy extends AbstractTraversalStrateg
throw new ComputerVerificationException("A final CollectingBarrierStep can not process an element beyond its id: " + endStep, traversal);
}
///
- if(endStep instanceof RangeGlobalStep || endStep instanceof TailGlobalStep)
+ if (endStep instanceof RangeGlobalStep || endStep instanceof TailGlobalStep || endStep instanceof DedupGlobalStep)
((Bypassing) endStep).setBypass(true);
}
for (final Step<?, ?> step : traversal.getSteps()) {
- if ((step instanceof ReducingBarrierStep || step instanceof SupplyingBarrierStep || step instanceof OrderGlobalStep || step instanceof RangeGlobalStep || step instanceof TailGlobalStep) && (step != endStep || !(traversal.getParent() instanceof EmptyStep)))
+ if ((step instanceof ReducingBarrierStep || step instanceof SupplyingBarrierStep || step instanceof OrderGlobalStep || step instanceof RangeGlobalStep || step instanceof TailGlobalStep || step instanceof DedupGlobalStep) && (step != endStep || !(traversal.getParent() instanceof EmptyStep)))
throw new ComputerVerificationException("Global traversals on GraphComputer may not contain mid-traversal barriers: " + step, traversal);
if (step instanceof TraversalParent) {
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8eaa0c27/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
index f0ca09c..009936e 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
@@ -18,8 +18,8 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step.filter
-import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalScriptHelper
import org.apache.tinkerpop.gremlin.process.traversal.Traversal
+import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalScriptHelper
import org.apache.tinkerpop.gremlin.structure.Vertex
/**
@@ -29,23 +29,6 @@ import org.apache.tinkerpop.gremlin.structure.Vertex
public abstract class GroovyDedupTest {
public static class Traversals extends DedupTest {
- /*@Override
- @Test
- @org.junit.Ignore("Traversal not supported by ComputerTraversalEngine.computer")
- public void g_V_both_dedup_name() {
- }
-
- @Override
- @Test
- @org.junit.Ignore("Traversal not supported by ComputerTraversalEngine.computer")
- public void g_V_both_hasXlabel_softwareX_dedup_byXlangX_name() {
- }
-
- @Override
- @Test
- @org.junit.Ignore("Traversal not supported by ComputerTraversalEngine.computer")
- public void g_V_both_name_orderXa_bX_dedup() {
- }*/
@Override
public Traversal<Vertex, String> get_g_V_both_dedup_name() {
@@ -57,7 +40,6 @@ public abstract class GroovyDedupTest {
TraversalScriptHelper.compute("g.V().group().by(T.label).by(bothE().values('weight').fold()).by(dedup(Scope.local))", g);
}
-
@Override
public Traversal<Vertex, String> get_g_V_both_hasXlabel_softwareX_dedup_byXlangX_name() {
TraversalScriptHelper.compute("g.V.both.has(T.label, 'software').dedup.by('lang').name", g);
@@ -65,7 +47,17 @@ public abstract class GroovyDedupTest {
@Override
public Traversal<Vertex, String> get_g_V_both_name_orderXa_bX_dedup() {
- return TraversalScriptHelper.compute("g.V().both().properties('name').order.by { a, b -> a.value() <=> b.value() }.dedup.value", g);
+ TraversalScriptHelper.compute("g.V().both().properties('name').order.by { a, b -> a.value() <=> b.value() }.dedup.value", g);
+ }
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_both_both_dedup() {
+ TraversalScriptHelper.compute("g.V.both.both.dedup", g)
+ }
+
+ @Override
+ public Traversal<Vertex, String> get_g_V_both_both_name_dedup() {
+ TraversalScriptHelper.compute("g.V.both.both.name.dedup", g);
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-tinkerpop/blob/8eaa0c27/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
index 2696012..9d2102d 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
@@ -22,9 +22,8 @@ import org.apache.tinkerpop.gremlin.LoadGraphWith;
import org.apache.tinkerpop.gremlin.process.AbstractGremlinProcessTest;
import org.apache.tinkerpop.gremlin.process.GremlinProcessRunner;
import org.apache.tinkerpop.gremlin.process.traversal.Scope;
-import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
-import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine;
+import org.apache.tinkerpop.gremlin.structure.T;
import org.apache.tinkerpop.gremlin.structure.Vertex;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -54,6 +53,10 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, String> get_g_V_both_name_orderXa_bX_dedup();
+ public abstract Traversal<Vertex, String> get_g_V_both_both_name_dedup();
+
+ public abstract Traversal<Vertex, Vertex> get_g_V_both_both_dedup();
+
public abstract Traversal<Vertex, Map<String, Set<Double>>> get_g_V_group_byXlabelX_byXbothE_valuesXweightX_foldX_byXdedupXlocalXX();
@Test
@@ -101,6 +104,28 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
@Test
@LoadGraphWith(MODERN)
+ public void g_V_both_both_name_dedup() {
+ final Traversal<Vertex, String> traversal = get_g_V_both_both_name_dedup();
+ printTraversalForm(traversal);
+ checkResults(Arrays.asList("marko", "vadas", "josh", "peter", "lop", "ripple"), traversal);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_both_both_dedup() {
+ final Traversal<Vertex, Vertex> traversal = get_g_V_both_both_dedup();
+ printTraversalForm(traversal);
+ checkResults(Arrays.asList(
+ convertToVertex(graph, "marko"),
+ convertToVertex(graph, "vadas"),
+ convertToVertex(graph, "josh"),
+ convertToVertex(graph, "peter"),
+ convertToVertex(graph, "lop"),
+ convertToVertex(graph, "ripple")), traversal);
+ }
+
+ @Test
+ @LoadGraphWith(MODERN)
public void g_V_group_byXlabelX_byXbothE_valuesXweightX_foldX_byXdedupXlocalXX() {
final Traversal<Vertex, Map<String, Set<Double>>> traversal =
get_g_V_group_byXlabelX_byXbothE_valuesXweightX_foldX_byXdedupXlocalXX();
@@ -133,6 +158,16 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
}
@Override
+ public Traversal<Vertex, String> get_g_V_both_both_name_dedup() {
+ return g.V().both().both().<String>values("name").dedup();
+ }
+
+ @Override
+ public Traversal<Vertex, Vertex> get_g_V_both_both_dedup() {
+ return g.V().both().both().dedup();
+ }
+
+ @Override
public Traversal<Vertex, Map<String, Set<Double>>> get_g_V_group_byXlabelX_byXbothE_valuesXweightX_foldX_byXdedupXlocalXX() {
return g.V().<String, Set<Double>>group().by(T.label).by(bothE().values("weight").fold()).by(dedup(Scope.local));
}