You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by sp...@apache.org on 2019/09/13 20:06:20 UTC
[tinkerpop] 01/01: TINKERPOP-1810 Support for withSack() that use
Lambdas with remotes
This is an automated email from the ASF dual-hosted git repository.
spmallette pushed a commit to branch TINKERPOP-1810
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit 0ac7f3acf04929043151becd6393df88bcb8ec9e
Author: Stephen Mallette <sp...@genoprime.com>
AuthorDate: Fri Sep 13 15:53:27 2019 -0400
TINKERPOP-1810 Support for withSack() that use Lambdas with remotes
To support this we needed to better detect the number of arguments that a lambda has so that we can properly deserialize to BinaryOperator or UnaryOperator. We also needed to better cast arguments for groovy given withSack() overlaods that aren't easily detected during eval() and generate errors otherwise. The method for detecting arguments isn't super nice - just some string parsing, but since Gremlin really only ever expects zero, one or two arg lambdas the risk of taking this appro [...]
---
CHANGELOG.asciidoc | 1 +
docs/src/reference/gremlin-variants.asciidoc | 8 ++++++
.../tinkerpop/gremlin/util/function/Lambda.java | 31 ++++++++++++++++++++++
.../src/Gremlin.Net/Process/Traversal/Lambda.cs | 4 +--
.../Process/Traversal/StringBasedLambda.cs | 21 ++++++++++++++-
.../GraphTraversalSourceTests.cs | 12 +++++++++
.../gremlin/groovy/jsr223/GroovyTranslator.java | 31 ++++++++++++++++++++--
.../groovy/jsr223/GroovyTranslatorTest.java | 28 ++++++++++++++++++-
.../gremlin_python/structure/io/graphsonV2d0.py | 8 ++++--
.../gremlin_python/structure/io/graphsonV3d0.py | 8 ++++--
.../tests/driver/test_driver_remote_connection.py | 11 ++++++++
11 files changed, 153 insertions(+), 10 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index f94bab3..2797d91 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -24,6 +24,7 @@ image::https://raw.githubusercontent.com/apache/tinkerpop/master/docs/static/ima
=== TinkerPop 3.3.9 (Release Date: NOT OFFICIALLY RELEASED YET)
* Added `ImmutableExplanation` for a `TraversalExplanation` that just contains data.
+* Added support for `UnaryOperator` and `BinaryOperator` for `Lambda` instances.
* Fixed `TraversalExplanation` deserialization in GraphSON 2 and 3 which was not supported before in Java.
* Added support for custom request headers in Python.
* Bumped jackson databind 2.9.9.3.
diff --git a/docs/src/reference/gremlin-variants.asciidoc b/docs/src/reference/gremlin-variants.asciidoc
index c356019..d141333 100644
--- a/docs/src/reference/gremlin-variants.asciidoc
+++ b/docs/src/reference/gremlin-variants.asciidoc
@@ -312,6 +312,10 @@ g.V().out().map(lambda: "x: len(x.get().value('name'))").sum().toList()
<7> The default lambda language is changed back to Gremlin-Python.
<8> If the `lambda`-prefix is not provided, then it is appended automatically in order to give a more natural look to the expression.
+TIP: When running into situations where Groovy cannot properly discern a method signature based on the `Lambda`
+instance created, it will help to fully define the closure in the lambda expression - so rather than
+`lambda: ("it.get().value("name')","gremlin-groovy")`, prefer `lambda: ("x -> x.get().value("name"),"gremlin-groovy")`.
+
=== Limitations
* Traversals that return a `Set` *might* be coerced to a `List` in Python. In the case of Python, number equality
@@ -504,6 +508,10 @@ g.V().Out().Map<int>(Lambda.Python("lambda x: len(x.get().value('name'))")).Sum<
The `ILambda` interface returned by these two methods inherits interfaces like `IFunction` and `IPredicate` that mirror
their Java counterparts which makes it possible to use lambdas with Gremlin.Net for the same steps as in Gremlin-Java.
+TIP: When running into situations where Groovy cannot properly discern a method signature based on the `Lambda`
+instance created, it will help to fully define the closure in the lambda expression - so rather than
+`Lambda.Groovy("it.get().value('name'))`, prefer `Lambda.Groovy("x -> x.get().value('name'))`.
+
[[gremlin-javascript]]
== Gremlin-JavaScript
diff --git a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/function/Lambda.java b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/function/Lambda.java
index 1031156..0ec32eb 100644
--- a/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/function/Lambda.java
+++ b/gremlin-core/src/main/java/org/apache/tinkerpop/gremlin/util/function/Lambda.java
@@ -19,16 +19,23 @@
package org.apache.tinkerpop.gremlin.util.function;
+import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine;
+
import java.io.Serializable;
import java.util.Comparator;
import java.util.function.BiFunction;
+import java.util.function.BinaryOperator;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
/**
+ * Provides a way to serialize string lambdas as scripts which can be evaluated by a {@link GremlinScriptEngine}.
+ *
* @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
*/
public interface Lambda extends Serializable {
@@ -103,6 +110,12 @@ public interface Lambda extends Serializable {
}
+ public static class UnaryLambda<A> extends OneArgLambda<A, A> implements UnaryOperator<A> {
+ public UnaryLambda(final String lambdaSource, final String lambdaLanguage) {
+ super(lambdaSource, lambdaLanguage);
+ }
+ }
+
public static class OneArgLambda<A, B> extends AbstractLambda implements Function<A, B>, Predicate<A>, Consumer<A> {
public OneArgLambda(final String lambdaSource, final String lambdaLanguage) {
@@ -125,6 +138,12 @@ public interface Lambda extends Serializable {
}
}
+ public static class BinaryLambda<A> extends TwoArgLambda<A, A, A> implements BinaryOperator<A> {
+ public BinaryLambda(final String lambdaSource, final String lambdaLanguage) {
+ super(lambdaSource, lambdaLanguage);
+ }
+ }
+
public static class TwoArgLambda<A, B, C> extends AbstractLambda implements BiFunction<A, B, C>, Comparator<A> {
public TwoArgLambda(final String lambdaSource, final String lambdaLanguage) {
@@ -147,6 +166,10 @@ public interface Lambda extends Serializable {
public static String DEFAULT_LAMBDA_LANGUAGE = "gremlin-groovy";
+ public static <A> UnaryOperator<A> unaryOperator(final String lambdaSource, final String lambdaLanguage) {
+ return new UnaryLambda<>(lambdaSource, lambdaLanguage);
+ }
+
public static <A, B> Function<A, B> function(final String lambdaSource, final String lambdaLanguage) {
return new OneArgLambda<>(lambdaSource, lambdaLanguage);
}
@@ -173,6 +196,10 @@ public interface Lambda extends Serializable {
//
+ public static <A> UnaryOperator<A> unaryOperator(final String lambdaSource) {
+ return new UnaryLambda<>(lambdaSource, DEFAULT_LAMBDA_LANGUAGE);
+ }
+
public static <A, B> Function<A, B> function(final String lambdaSource) {
return new OneArgLambda<>(lambdaSource, DEFAULT_LAMBDA_LANGUAGE);
}
@@ -196,4 +223,8 @@ public interface Lambda extends Serializable {
public static <A, B, C> BiFunction<A, B, C> biFunction(final String lambdaSource) {
return new TwoArgLambda<>(lambdaSource, DEFAULT_LAMBDA_LANGUAGE);
}
+
+ public static <A> BinaryOperator<A> binaryOperator(final String lambdaSource) {
+ return new BinaryLambda<>(lambdaSource, DEFAULT_LAMBDA_LANGUAGE);
+ }
}
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/Lambda.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/Lambda.cs
index c1a0e44..01bdbc6 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/Lambda.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/Lambda.cs
@@ -42,7 +42,7 @@ namespace Gremlin.Net.Process.Traversal
/// <summary>
/// Gets the arguments of this lambda.
/// </summary>
- object Arguments { get; }
+ int Arguments { get; }
}
/// <summary>
@@ -57,7 +57,7 @@ namespace Gremlin.Net.Process.Traversal
/// <returns>The created lambda.</returns>
public static ILambda Groovy(string expression)
{
- return new StringBasedLambda(expression, "gremlin-groovy");
+ return new GroovyStringBasedLambda(expression);
}
/// <summary>
diff --git a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/StringBasedLambda.cs b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/StringBasedLambda.cs
index e27b474..71a4651 100644
--- a/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/StringBasedLambda.cs
+++ b/gremlin-dotnet/src/Gremlin.Net/Process/Traversal/StringBasedLambda.cs
@@ -21,22 +21,41 @@
#endregion
+using System;
+
namespace Gremlin.Net.Process.Traversal
{
internal class StringBasedLambda : ILambda
{
private const int DefaultArgument = -1;
+ private int _arguments;
public StringBasedLambda(string expression, string language)
{
LambdaExpression = expression;
Language = language;
+ _arguments = DefaultArgument;
}
public string LambdaExpression { get; }
public string Language { get; }
- public object Arguments => DefaultArgument;
+ public int Arguments
+ {
+ get => _arguments;
+ protected set => _arguments = value;
+ }
+ }
+
+ internal class GroovyStringBasedLambda : StringBasedLambda
+ {
+ public GroovyStringBasedLambda(string expression) : base(expression, "gremlin-groovy")
+ {
+ // try to detect 1 or 2 argument lambda if possible otherwise go with unknown which is the default
+ if (!expression.Contains("->")) return;
+ var args = expression.Substring(0, expression.IndexOf("->", StringComparison.Ordinal));
+ Arguments = args.Contains(",") ? 2 : 1;
+ }
}
}
\ No newline at end of file
diff --git a/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Process/Traversal/DriverRemoteConnection/GraphTraversalSourceTests.cs b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Process/Traversal/DriverRemoteConnection/GraphTraversalSourceTests.cs
index 1bc45da..3aaa9b7 100644
--- a/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Process/Traversal/DriverRemoteConnection/GraphTraversalSourceTests.cs
+++ b/gremlin-dotnet/test/Gremlin.Net.IntegrationTest/Process/Traversal/DriverRemoteConnection/GraphTraversalSourceTests.cs
@@ -49,5 +49,17 @@ namespace Gremlin.Net.IntegrationTest.Process.Traversal.DriverRemoteConnection
Assert.Contains("josh", results);
Assert.Contains("peter", results);
}
+
+ [Fact]
+ public void ShouldHandleLambdasInWithSack()
+ {
+ var connection = _connectionFactory.CreateRemoteConnection();
+ var g = AnonymousTraversalSource.Traversal().WithRemote(connection);
+
+ Assert.Equal(24.0, g.WithSack(1.0, (IUnaryOperator) Lambda.Groovy("x -> x + 1")).V().Both().Sack<double>().Sum<double>().Next());
+ Assert.Equal(24.0, g.WithSack((ISupplier) Lambda.Groovy("{1.0d}"), (IUnaryOperator) Lambda.Groovy("x -> x + 1")).V().Both().Sack<double>().Sum<double>().Next());
+ Assert.Equal(48.0, g.WithSack(1.0, (IBinaryOperator) Lambda.Groovy("x, y -> x + y + 1")).V().Both().Sack<double>().Sum<double>().Next());
+ Assert.Equal(48.0, g.WithSack((ISupplier) Lambda.Groovy("{1.0d}"), (IBinaryOperator) Lambda.Groovy("x, y -> x + y + 1")).V().Both().Sack<double>().Sum<double>().Next());
+ }
}
}
\ No newline at end of file
diff --git a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
index 717e1bc..ba14996 100644
--- a/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
+++ b/gremlin-groovy/src/main/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslator.java
@@ -26,6 +26,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.P;
import org.apache.tinkerpop.gremlin.process.traversal.SackFunctions;
import org.apache.tinkerpop.gremlin.process.traversal.Translator;
import org.apache.tinkerpop.gremlin.process.traversal.Traversal;
+import org.apache.tinkerpop.gremlin.process.traversal.TraversalSource;
import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy;
import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalOptionParent;
import org.apache.tinkerpop.gremlin.process.traversal.strategy.TraversalStrategyProxy;
@@ -47,6 +48,9 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
+import java.util.function.BinaryOperator;
+import java.util.function.Supplier;
+import java.util.function.UnaryOperator;
/**
* Converts bytecode to a Groovy string of Gremlin.
@@ -105,8 +109,31 @@ public final class GroovyTranslator implements Translator.ScriptTranslator {
else {
traversalScript.append(".");
String temp = methodName + "(";
- for (final Object object : instruction.getArguments()) {
- temp = temp + convertToString(object) + ",";
+
+ // have to special case withSack() for Groovy because UnaryOperator and BinaryOperator signatures
+ // make it impossible for the interpreter to figure out which function to call. specifically we need
+ // to discern between:
+ // withSack(A initialValue, UnaryOperator<A> splitOperator)
+ // withSack(A initialValue, BinaryOperator<A> splitOperator)
+ // and:
+ // withSack(Supplier<A> initialValue, UnaryOperator<A> mergeOperator)
+ // withSack(Supplier<A> initialValue, BinaryOperator<A> mergeOperator)
+ if (methodName.equals(TraversalSource.Symbols.withSack) &&
+ instruction.getArguments().length == 2 && instruction.getArguments()[1] instanceof Lambda) {
+ final String castFirstArgTo = instruction.getArguments()[0] instanceof Lambda ?
+ Supplier.class.getName() : "";
+ final Lambda secondArg = (Lambda) instruction.getArguments()[1];
+ final String castSecondArgTo = secondArg.getLambdaArguments() == 1 ? UnaryOperator.class.getName() :
+ BinaryOperator.class.getName();
+ if (!castFirstArgTo.isEmpty())
+ temp = temp + String.format("(%s) ", castFirstArgTo);
+ temp = temp + String.format("%s, (%s) %s,",
+ convertToString(instruction.getArguments()[0]), castSecondArgTo,
+ convertToString(instruction.getArguments()[1]));
+ } else {
+ for (final Object object : instruction.getArguments()) {
+ temp = temp + convertToString(object) + ",";
+ }
}
traversalScript.append(temp.substring(0, temp.length() - 1)).append(")");
}
diff --git a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslatorTest.java b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslatorTest.java
index b097c3d..305717c 100644
--- a/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslatorTest.java
+++ b/gremlin-groovy/src/test/java/org/apache/tinkerpop/gremlin/groovy/jsr223/GroovyTranslatorTest.java
@@ -20,7 +20,6 @@
package org.apache.tinkerpop.gremlin.groovy.jsr223;
import org.apache.commons.configuration.MapConfiguration;
-import org.apache.tinkerpop.gremlin.AbstractGremlinTest;
import org.apache.tinkerpop.gremlin.jsr223.TranslatorCustomizer;
import org.apache.tinkerpop.gremlin.process.traversal.Order;
import org.apache.tinkerpop.gremlin.process.traversal.Pop;
@@ -71,6 +70,7 @@ import static org.junit.Assert.fail;
/**
* @author Marko A. Rodriguez (http://markorodriguez.com)
+ * @author Stephen Mallette (http://stephen.genoprime.com)
*/
public class GroovyTranslatorTest {
@@ -102,6 +102,32 @@ public class GroovyTranslatorTest {
}
@Test
+ public void shouldHandleConfusingSacks() {
+ final TinkerGraph graph = TinkerFactory.createModern();
+ final GraphTraversalSource g = graph.traversal();
+
+ final Traversal<Vertex,Double> tConstantUnary = g.withSack(1.0, Lambda.unaryOperator("it + 1")).V().sack();
+ final String scriptConstantUnary = GroovyTranslator.of("g").translate(tConstantUnary.asAdmin().getBytecode());
+ assertEquals("g.withSack(1.0d, (java.util.function.UnaryOperator) {it + 1}).V().sack()", scriptConstantUnary);
+ assertThatScriptOk(scriptConstantUnary, "g", g);
+
+ final Traversal<Vertex,Double> tSupplierUnary = g.withSack(Lambda.supplier("1.0d"), Lambda.<Double>unaryOperator("it + 1")).V().sack();
+ final String scriptSupplierUnary = GroovyTranslator.of("g").translate(tSupplierUnary.asAdmin().getBytecode());
+ assertEquals("g.withSack((java.util.function.Supplier) {1.0d}, (java.util.function.UnaryOperator) {it + 1}).V().sack()", scriptSupplierUnary);
+ assertThatScriptOk(scriptSupplierUnary, "g", g);
+
+ final Traversal<Vertex,Double> tConstantBinary = g.withSack(1.0, Lambda.binaryOperator("x,y -> x + y + 1")).V().sack();
+ final String scriptConstantBinary = GroovyTranslator.of("g").translate(tConstantBinary.asAdmin().getBytecode());
+ assertEquals("g.withSack(1.0d, (java.util.function.BinaryOperator) {x,y -> x + y + 1}).V().sack()", scriptConstantBinary);
+ assertThatScriptOk(scriptConstantBinary, "g", g);
+
+ final Traversal<Vertex,Double> tSupplierBinary = g.withSack(Lambda.supplier("1.0d"), Lambda.<Double>binaryOperator("x,y -> x + y + 1")).V().sack();
+ final String scriptSupplierBinary = GroovyTranslator.of("g").translate(tSupplierBinary.asAdmin().getBytecode());
+ assertEquals("g.withSack((java.util.function.Supplier) {1.0d}, (java.util.function.BinaryOperator) {x,y -> x + y + 1}).V().sack()", scriptSupplierBinary);
+ assertThatScriptOk(scriptSupplierBinary, "g", g);
+ }
+
+ @Test
public void shouldSupportStringSupplierLambdas() {
final TinkerGraph graph = TinkerFactory.createModern();
GraphTraversalSource g = graph.traversal();
diff --git a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV2d0.py b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV2d0.py
index c3cfb7d..6b5def5 100644
--- a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV2d0.py
+++ b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV2d0.py
@@ -20,10 +20,8 @@ under the License.
import calendar
import datetime
import json
-import time
import uuid
import math
-import base64
from collections import OrderedDict
from decimal import *
from datetime import timedelta
@@ -311,8 +309,14 @@ class LambdaSerializer(_GraphSONTypeIO):
script = "lambda " + script
out["script"] = script
out["arguments"] = six.get_function_code(eval(out["script"])).co_argcount
+ elif language == "gremlin-groovy" and "->" in script:
+ # if the user has explicitly added parameters to the groovy closure then we can easily detect one or two
+ # arg lambdas - if we can't detect 1 or 2 then we just go with "unknown"
+ args = script[0:script.find("->")]
+ out["arguments"] = 2 if "," in args else 1
else:
out["arguments"] = -1
+
return GraphSONUtil.typedValue("Lambda", out)
diff --git a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py
index 255ebfb..ffbf23a 100644
--- a/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py
+++ b/gremlin-python/src/main/jython/gremlin_python/structure/io/graphsonV3d0.py
@@ -19,10 +19,8 @@ under the License.
import calendar
import datetime
import json
-import time
import uuid
import math
-import base64
from collections import OrderedDict
from decimal import *
import logging
@@ -319,8 +317,14 @@ class LambdaSerializer(_GraphSONTypeIO):
script = "lambda " + script
out["script"] = script
out["arguments"] = six.get_function_code(eval(out["script"])).co_argcount
+ elif language == "gremlin-groovy" and "->" in script:
+ # if the user has explicitly added parameters to the groovy closure then we can easily detect one or two
+ # arg lambdas - if we can't detect 1 or 2 then we just go with "unknown"
+ args = script[0:script.find("->")]
+ out["arguments"] = 2 if "," in args else 1
else:
out["arguments"] = -1
+
return GraphSONUtil.typedValue("Lambda", out)
diff --git a/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py b/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py
index aa666d9..7bda287 100644
--- a/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py
+++ b/gremlin-python/src/main/jython/tests/driver/test_driver_remote_connection.py
@@ -92,6 +92,17 @@ class TestDriverRemoteConnection(object):
__.max_().is_(gt(0)), __.min_().is_(gt(0)))).range_(0, 1).id_().next()
assert 1 == results
+ def test_lambda_traversals(self, remote_connection):
+ statics.load_statics(globals())
+ assert "remoteconnection[ws://localhost:45940/gremlin,gmodern]" == str(remote_connection)
+ g = traversal().withRemote(remote_connection)
+
+ assert 24.0 == g.withSack(1.0, lambda: ("x -> x + 1", "gremlin-groovy")).V().both().sack().sum().next()
+ assert 24.0 == g.withSack(lambda: ("{1.0d}", "gremlin-groovy"), lambda: ("x -> x + 1", "gremlin-groovy")).V().both().sack().sum().next()
+
+ assert 48.0 == g.withSack(1.0, lambda: ("x, y -> x + y + 1", "gremlin-groovy")).V().both().sack().sum().next()
+ assert 48.0 == g.withSack(lambda: ("{1.0d}", "gremlin-groovy"), lambda: ("x, y -> x + y + 1", "gremlin-groovy")).V().both().sack().sum().next()
+
def test_iteration(self, remote_connection):
statics.load_statics(globals())
assert "remoteconnection[ws://localhost:45940/gremlin,gmodern]" == str(remote_connection)