You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by ok...@apache.org on 2016/10/11 14:20:38 UTC
tinkerpop git commit: added GraphComputing.atMaster(boolean) to allow
steps (if the want) to know whether they are executing locally at master or
distributed across workers. This fixes a bug in dedup() on OLAP where
non-element traversers were not being
Repository: tinkerpop
Updated Branches:
refs/heads/TINKERPOP-1495 [created] 0dcf65921
added GraphComputing.atMaster(boolean) to allow steps (if the want) to know whether they are executing locally at master or distributed across workers. This fixes a bug in dedup() on OLAP where non-element traversers were not being dedup'd correctly.
Project: http://git-wip-us.apache.org/repos/asf/tinkerpop/repo
Commit: http://git-wip-us.apache.org/repos/asf/tinkerpop/commit/0dcf6592
Tree: http://git-wip-us.apache.org/repos/asf/tinkerpop/tree/0dcf6592
Diff: http://git-wip-us.apache.org/repos/asf/tinkerpop/diff/0dcf6592
Branch: refs/heads/TINKERPOP-1495
Commit: 0dcf65921b6045d734d1d6230309f2a1de4f4ee5
Parents: 193db1c
Author: Marko A. Rodriguez <ok...@gmail.com>
Authored: Tue Oct 11 08:20:31 2016 -0600
Committer: Marko A. Rodriguez <ok...@gmail.com>
Committed: Tue Oct 11 08:20:31 2016 -0600
----------------------------------------------------------------------
CHANGELOG.asciidoc | 2 ++
.../computer/traversal/MasterExecutor.java | 4 ++-
.../computer/traversal/WorkerExecutor.java | 4 ++-
.../process/traversal/step/GraphComputing.java | 30 ++++++++++++++++++++
.../traversal/step/filter/DedupGlobalStep.java | 12 ++++++--
.../step/filter/GroovyDedupTest.groovy | 6 +++-
.../traversal/step/filter/DedupTest.java | 30 +++++++++++++++++---
7 files changed, 79 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/CHANGELOG.asciidoc
----------------------------------------------------------------------
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 746d38c..fbb6043 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -26,6 +26,8 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
TinkerPop 3.2.3 (Release Date: NOT OFFICIALLY RELEASED YET)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+* Added `GraphComputing.atMaster(boolean)` to allow steps to know whether they are executing at master or distributed at workers.
+* Fixed a bug in OLAP where `DedupGlobalStep` wasn't de-duping local master traversers.
* Added `HasContainerHolder.removeHasContainer()`-method with default `UnsupportedOperationException` implementation.
* `TraversalSource.withComputer()` is simplified to add a `VertexProgramStrategy`. Easier for language variants.
* Fixed a `Set`, `List`, `Map` bug in the various `Translators` where such collections were not being internally translated.
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
index f81ca14..48e3a57 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/MasterExecutor.java
@@ -24,12 +24,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.Path;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
+import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.ConnectiveStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.RangeGlobalStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.filter.TailGlobalStep;
-import org.apache.tinkerpop.gremlin.process.traversal.step.filter.WherePredicateStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.IdStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.LabelStep;
import org.apache.tinkerpop.gremlin.process.traversal.step.map.PropertiesStep;
@@ -105,6 +105,7 @@ final class MasterExecutor {
else {
currentStep = traversalMatrix.getStepById(traverser.getStepId());
if (!currentStep.getId().equals(previousStep.getId()) && !(previousStep instanceof EmptyStep)) {
+ GraphComputing.atMaster(previousStep, true);
while (previousStep.hasNext()) {
final Traverser.Admin<Object> result = previousStep.next();
if (result.isHalted())
@@ -120,6 +121,7 @@ final class MasterExecutor {
}
}
if (!(currentStep instanceof EmptyStep)) {
+ GraphComputing.atMaster(currentStep, true);
while (currentStep.hasNext()) {
final Traverser.Admin<Object> traverser = currentStep.next();
if (traverser.isHalted())
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
index 0a3aad2..2571e7b 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/computer/traversal/WorkerExecutor.java
@@ -21,14 +21,15 @@ package org.apache.tinkerpop.gremlin.process.computer.traversal;
import org.apache.tinkerpop.gremlin.process.computer.Memory;
import org.apache.tinkerpop.gremlin.process.computer.MessageScope;
import org.apache.tinkerpop.gremlin.process.computer.Messenger;
-import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.Step;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalSideEffects;
import org.apache.tinkerpop.gremlin.process.traversal.Traverser;
import org.apache.tinkerpop.gremlin.process.traversal.step.Barrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.Bypassing;
+import org.apache.tinkerpop.gremlin.process.traversal.step.GraphComputing;
import org.apache.tinkerpop.gremlin.process.traversal.step.LocalBarrier;
import org.apache.tinkerpop.gremlin.process.traversal.step.util.EmptyStep;
+import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.traverser.util.TraverserSet;
import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalMatrix;
import org.apache.tinkerpop.gremlin.structure.Edge;
@@ -179,6 +180,7 @@ final class WorkerExecutor {
final Memory memory,
final boolean returnHaltedTraversers,
final HaltedTraverserStrategy haltedTraverserStrategy) {
+ GraphComputing.atMaster(step, false);
if (step instanceof Barrier) {
if (step instanceof Bypassing)
((Bypassing) step).setBypass(true);
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
index 0028eb1..bd9fc42 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/GraphComputing.java
@@ -18,6 +18,9 @@
*/
package org.apache.tinkerpop.gremlin.process.traversal.step;
+import org.apache.tinkerpop.gremlin.process.traversal.Step;
+import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+
/**
* A {@code GraphComputing} step is one that will change its behavior whether its on a {@link org.apache.tinkerpop.gremlin.process.computer.GraphComputer} or not.
* {@link org.apache.tinkerpop.gremlin.process.traversal.strategy.verification.ComputerVerificationStrategy} is responsible for calling the {@link GraphComputing#onGraphComputer()} method.
@@ -32,4 +35,31 @@ public interface GraphComputing {
*/
public void onGraphComputer();
+ /**
+ * Some steps should behave different whether they are executing at the master traversal or distributed across the worker traversals.
+ * The default implementation does nothing.
+ *
+ * @param atMaster whether the step is currently executing at master
+ */
+ public default void atMaster(boolean atMaster) {
+
+ }
+
+ public static void atMaster(final Step<?, ?> step, boolean atMaster) {
+ if (step instanceof GraphComputing)
+ ((GraphComputing) step).atMaster(atMaster);
+ if (step instanceof TraversalParent) {
+ for (final Traversal.Admin<?, ?> local : ((TraversalParent) step).getLocalChildren()) {
+ for (final Step<?, ?> s : local.getSteps()) {
+ GraphComputing.atMaster(s, atMaster);
+ }
+ }
+ for (final Traversal.Admin<?, ?> global : ((TraversalParent) step).getGlobalChildren()) {
+ for (final Step<?, ?> s : global.getSteps()) {
+ GraphComputing.atMaster(s, atMaster);
+ }
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
----------------------------------------------------------------------
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
index 8ef6455..220805f 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupGlobalStep.java
@@ -57,6 +57,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
private boolean onGraphComputer = false;
private final Set<String> dedupLabels;
private Set<String> keepLabels;
+ private boolean executingAtMaster = false;
public DedupGlobalStep(final Traversal.Admin traversal, final String... dedupLabels) {
super(traversal);
@@ -65,7 +66,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
@Override
protected boolean filter(final Traverser.Admin<S> traverser) {
- if (this.onGraphComputer) return true;
+ if (this.onGraphComputer && !this.executingAtMaster) return true;
traverser.setBulk(1);
if (null == this.dedupLabels) {
return this.duplicateSet.add(TraversalUtil.applyNullable(traverser, this.dedupTraversal));
@@ -77,6 +78,11 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
}
@Override
+ public void atMaster(final boolean atMaster) {
+ this.executingAtMaster = atMaster;
+ }
+
+ @Override
public ElementRequirement getMaxRequirement() {
return null == this.dedupLabels ? ElementRequirement.ID : PathProcessor.super.getMaxRequirement();
}
@@ -206,5 +212,7 @@ public final class DedupGlobalStep<S> extends FilterStep<S> implements Traversal
}
@Override
- public Set<String> getKeepLabels() { return this.keepLabels; }
+ public Set<String> getKeepLabels() {
+ return this.keepLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
----------------------------------------------------------------------
diff --git a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
index d03f526..a091c6f 100644
--- a/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
+++ b/gremlin-groovy-test/src/main/groovy/org/apache/tinkerpop/gremlin/process/traversal/step/filter/GroovyDedupTest.groovy
@@ -19,7 +19,6 @@
package org.apache.tinkerpop.gremlin.process.traversal.step.filter
import org.apache.tinkerpop.gremlin.process.traversal.Path
-import org.apache.tinkerpop.gremlin.process.traversal.Scope
import org.apache.tinkerpop.gremlin.process.traversal.Traversal
import org.apache.tinkerpop.gremlin.process.traversal.util.ScriptTraversal
import org.apache.tinkerpop.gremlin.structure.Vertex
@@ -95,5 +94,10 @@ public abstract class GroovyDedupTest {
public Traversal<Vertex, String> get_g_V_both_both_dedup_byXoutE_countX_name() {
new ScriptTraversal<>(g, "gremlin-groovy", "g.V.both.both.dedup.by(outE().count).name")
}
+
+ @Override
+ public Traversal<Vertex, Long> get_g_V_groupCount_selectXvaluesX_unfold_dedup() {
+ new ScriptTraversal<>(g, "gremlin-groovy", "g.V.groupCount.select(values).unfold.dedup")
+ }
}
}
http://git-wip-us.apache.org/repos/asf/tinkerpop/blob/0dcf6592/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
----------------------------------------------------------------------
diff --git a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
index fa55c8e..970a976 100644
--- a/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
+++ b/gremlin-test/src/main/java/org/apache/tinkerpop/gremlin/process/traversal/step/filter/DedupTest.java
@@ -32,16 +32,18 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import static org.apache.tinkerpop.gremlin.LoadGraphWith.GraphData.MODERN;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.both;
import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.bothE;
-import static org.hamcrest.MatcherAssert.assertThat;
-import static org.hamcrest.Matchers.containsInAnyOrder;
-import static org.hamcrest.Matchers.isIn;
+import static org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__.select;
+import static org.apache.tinkerpop.gremlin.structure.Column.values;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -81,6 +83,8 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
public abstract Traversal<Vertex, Map<String, String>> get_g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold();
+ public abstract Traversal<Vertex, Long> get_g_V_groupCount_selectXvaluesX_unfold_dedup();
+
@Test
@LoadGraphWith(MODERN)
public void g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold() {
@@ -269,6 +273,14 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
assertEquals(4, new HashSet<>(names).size());
}
+ @Test
+ @LoadGraphWith(MODERN)
+ public void g_V_groupCount_selectXvaluesX_unfold_dedup() {
+ final Traversal<Vertex, Long> traversal = get_g_V_groupCount_selectXvaluesX_unfold_dedup();
+ printTraversalForm(traversal);
+ checkResults(Collections.singletonList(1L), traversal);
+ }
+
public static class Traversals extends DedupTest {
@Override
public Traversal<Vertex, String> get_g_V_out_in_valuesXnameX_fold_dedupXlocalX_unfold() {
@@ -277,7 +289,7 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
@Override
public Traversal<Vertex, Map<String, String>> get_g_V_out_asXxX_in_asXyX_selectXx_yX_byXnameX_fold_dedupXlocal_x_yX_unfold() {
- return g.V().out().as("x").in().as("y").select("x","y").by("name").fold().dedup(Scope.local,"x","y").unfold();
+ return g.V().out().as("x").in().as("y").select("x", "y").by("name").fold().dedup(Scope.local, "x", "y").unfold();
}
@Override
@@ -334,5 +346,15 @@ public abstract class DedupTest extends AbstractGremlinProcessTest {
public Traversal<Vertex, String> get_g_V_both_both_dedup_byXoutE_countX_name() {
return g.V().both().both().dedup().by(__.outE().count()).values("name");
}
+
+ @Override
+ public Traversal<Vertex, Long> get_g_V_groupCount_selectXvaluesX_unfold_dedup() {
+ return g.V().groupCount().select(values).<Long>unfold().dedup();
+ }
+
+ /*@Override
+ public Traversal<Vertex,Collection<Vertex>> get_g_V_asXaX_repeatXbothX_timesX3X_emit_asXbX_group_byXselectXaXX_byXselectXbX_dedup_order_byXidX_foldX_selectXvaluesX_unfold_dedup() {
+ return g.V().as("a").repeat(both()).times(3).emit().as("b").group().by(select("a")).by(select("b").dedup().order().by(T.id).fold()).select(values).<Collection<Vertex>>unfold().dedup();
+ }*/
}
}