You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by xi...@apache.org on 2023/10/13 05:23:42 UTC
[tinkerpop] 01/01: Enable remote connection over HTTP for gremlin python
This is an automated email from the ASF dual-hosted git repository.
xiazcy pushed a commit to branch add-http-gremlin-python
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit b2ef3dbb37a9e34e91319214a751b575eee94a14
Author: Yang Xia <55...@users.noreply.github.com>
AuthorDate: Thu Oct 12 22:23:23 2023 -0700
Enable remote connection over HTTP for gremlin python
---
CHANGELOG.asciidoc | 2 +
gremlin-python/docker-compose.yml | 2 +
.../gremlin_python/driver/aiohttp/transport.py | 101 ++++++++++
.../main/python/gremlin_python/driver/client.py | 34 +++-
.../main/python/gremlin_python/driver/protocol.py | 95 ++++++++-
.../python/gremlin_python/driver/serializer.py | 7 +-
.../python/gremlin_python/process/translator.py | 110 +++++++----
gremlin-python/src/main/python/tests/conftest.py | 58 +++++-
.../driver/test_driver_remote_connection_http.py | 214 +++++++++++++++++++++
.../main/python/tests/process/test_translator.py | 37 +++-
10 files changed, 605 insertions(+), 55 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 6ee4818231..152f66a8fa 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,6 +28,8 @@ This release also includes changes from <<release-3-5-8, 3.5.8>>.
* Fixed a javadoc comment in `GraphTraversal.not()` method.
* Allowed `gremlin-driver` to be used over HTTP for experimental purposes.
* Deprecated the `HandshakeInterceptor` in favor of a more generic `RequestInterceptor`.
+* Allowed `gremlin-python` to be used over HTTP for experimental purposes.
+* Updated `gremlin-python` translator to enable additional traversals over HTTP.
* Fixed a bug in `StarGraph` where `EdgeFilter` did not remove associated Edge Properties.
* Added translator to the Go GLV.
* Fixed bug with filtering for `group()` when the side-effect label was defined for it.
diff --git a/gremlin-python/docker-compose.yml b/gremlin-python/docker-compose.yml
index 6262c61da8..02e693ee5b 100644
--- a/gremlin-python/docker-compose.yml
+++ b/gremlin-python/docker-compose.yml
@@ -58,6 +58,8 @@ services:
- KRB5CCNAME=./test-tkt.cc
- GREMLIN_SERVER_URL=ws://gremlin-server-test-python:{}/gremlin
- GREMLIN_SERVER_BASIC_AUTH_URL=wss://gremlin-server-test-python:{}/gremlin
+ - GREMLIN_SERVER_URL_HTTP=http://gremlin-server-test-python:{}/
+ - GREMLIN_SERVER_BASIC_AUTH_URL_HTTP=https://gremlin-server-test-python:{}/
- KRB_HOSTNAME=${KRB_HOSTNAME:-gremlin-server-test}
- VERSION=${VERSION}
working_dir: /python_app
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
index 34dab16b53..4b9ead23cc 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
@@ -138,3 +138,104 @@ class AiohttpTransport(AbstractBaseTransport):
def closed(self):
# Connection is closed if either the websocket or the client session is closed.
return self._websocket.closed or self._client_session.closed
+
+
+class AiohttpHTTPTransport(AbstractBaseTransport):
+ nest_asyncio_applied = False
+
+ def __init__(self, call_from_event_loop=None, read_timeout=None, write_timeout=None, **kwargs):
+ if call_from_event_loop is not None and call_from_event_loop and not AiohttpTransport.nest_asyncio_applied:
+ """
+ The AiohttpTransport implementation uses the asyncio event loop. Because of this, it cannot be called
+ within an event loop without nest_asyncio. If the code is ever refactored so that it can be called
+ within an event loop this import and call can be removed. Without this, applications which use the
+ event loop to call gremlin-python (such as Jupyter) will not work.
+ """
+ import nest_asyncio
+ nest_asyncio.apply()
+ AiohttpTransport.nest_asyncio_applied = True
+
+ # Start event loop and initialize client session and response to None
+ self._loop = asyncio.new_event_loop()
+ self._client_session = None
+ self._http_req_resp = None
+ self._enable_ssl = False
+
+ # Set all inner variables to parameters passed in.
+ self._aiohttp_kwargs = kwargs
+ self._write_timeout = write_timeout
+ self._read_timeout = read_timeout
+ if "ssl_options" in self._aiohttp_kwargs:
+ self._ssl_context = self._aiohttp_kwargs.pop("ssl_options")
+ self._enable_ssl = True
+
+ # if "ssl_options" in self._aiohttp_kwargs:
+ # self._aiohttp_kwargs["ssl_context"] = self._aiohttp_kwargs.pop("ssl_options")
+
+ def __del__(self):
+ # Close will only actually close if things are left open, so this is safe to call.
+ # Clean up any connection resources and close the event loop.
+ self.close()
+
+ def connect(self, url, headers=None):
+ # Inner function to perform async connect.
+ async def async_connect():
+ # Start client session and use it to send all HTTP requests. Base url is the endpoint, headers are set here
+ # Base url can only parse basic url with no path, see https://github.com/aio-libs/aiohttp/issues/6647
+ if self._enable_ssl:
+ # ssl context is established through tcp connector
+ tcp_conn = aiohttp.TCPConnector(ssl_context=self._ssl_context)
+ self._client_session = aiohttp.ClientSession(connector=tcp_conn,
+ base_url=url, headers=headers, loop=self._loop)
+ else:
+ self._client_session = aiohttp.ClientSession(base_url=url, headers=headers, loop=self._loop)
+
+
+ # Execute the async connect synchronously.
+ self._loop.run_until_complete(async_connect())
+
+ def write(self, message):
+ # Inner function to perform async write.
+ async def async_write():
+ basic_auth = None
+ # basic password authentication for https connections
+ if message['auth']:
+ basic_auth = aiohttp.BasicAuth(message['auth']['username'], message['auth']['password'])
+ async with async_timeout.timeout(self._write_timeout):
+ self._http_req_resp = await self._client_session.post(url="/gremlin",
+ auth=basic_auth,
+ data=message['payload'],
+ headers=message['headers'],
+ **self._aiohttp_kwargs)
+
+ # Execute the async write synchronously.
+ self._loop.run_until_complete(async_write())
+
+ def read(self):
+ # Inner function to perform async read.
+ async def async_read():
+ async with async_timeout.timeout(self._read_timeout):
+ # using http request read()
+ return await self._http_req_resp.text()
+
+ return self._loop.run_until_complete(async_read())
+
+ def close(self):
+ # Inner function to perform async close.
+ async def async_close():
+ if self._client_session is not None and not self._client_session.closed:
+ await self._client_session.close()
+ self._client_session = None
+
+ # If the loop is not closed (connection hasn't already been closed)
+ if not self._loop.is_closed():
+ # Execute the async close synchronously.
+ self._loop.run_until_complete(async_close())
+
+ # Close the event loop.
+ self._loop.close()
+
+ @property
+ def closed(self):
+ # Connection is closed when client session is closed.
+ return self._client_session.closed
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/client.py b/gremlin-python/src/main/python/gremlin_python/driver/client.py
index ec7078d593..6ad9fd5517 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/client.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/client.py
@@ -19,6 +19,7 @@
import logging
import warnings
import queue
+import re
from concurrent.futures import ThreadPoolExecutor
from gremlin_python.driver import connection, protocol, request, serializer
@@ -45,15 +46,20 @@ class Client:
kerberized_service="", headers=None, session=None,
enable_user_agent_on_connect=True, **transport_kwargs):
log.info("Creating Client with url '%s'", url)
+
+ # check via url that we are using http protocol
+ self._use_http = re.search('^http', url)
+
self._closed = False
self._url = url
self._headers = headers
self._enable_user_agent_on_connect = enable_user_agent_on_connect
self._traversal_source = traversal_source
- if "max_content_length" not in transport_kwargs:
+ if not self._use_http and "max_content_length" not in transport_kwargs:
transport_kwargs["max_content_length"] = 10 * 1024 * 1024
if message_serializer is None:
message_serializer = serializer.GraphBinarySerializersV1()
+ # message_serializer = serializer.GraphSONMessageSerializer()
self._message_serializer = message_serializer
self._username = username
@@ -63,21 +69,31 @@ class Client:
if transport_factory is None:
try:
from gremlin_python.driver.aiohttp.transport import (
- AiohttpTransport)
+ AiohttpTransport, AiohttpHTTPTransport)
except ImportError:
raise Exception("Please install AIOHTTP or pass "
"custom transport factory")
else:
def transport_factory():
- return AiohttpTransport(**transport_kwargs)
+ if self._use_http:
+ return AiohttpHTTPTransport(**transport_kwargs)
+ else:
+ return AiohttpTransport(**transport_kwargs)
self._transport_factory = transport_factory
if protocol_factory is None:
- def protocol_factory(): return protocol.GremlinServerWSProtocol(
- self._message_serializer,
- username=self._username,
- password=self._password,
- kerberized_service=kerberized_service,
- max_content_length=transport_kwargs["max_content_length"])
+ def protocol_factory():
+ if self._use_http:
+ return protocol.GremlinServerHTTPProtocol(
+ self._message_serializer,
+ username=self._username,
+ password=self._password)
+ else:
+ return protocol.GremlinServerWSProtocol(
+ self._message_serializer,
+ username=self._username,
+ password=self._password,
+ kerberized_service=kerberized_service,
+ max_content_length=transport_kwargs["max_content_length"])
self._protocol_factory = protocol_factory
if self._session_enabled:
if pool_size is None:
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
index 9323210356..6aaa9b8efe 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
#
+import json
import logging
import abc
import base64
@@ -25,6 +26,8 @@ import struct
from gremlin_python.driver import request
from gremlin_python.driver.resultset import ResultSet
+from gremlin_python.process.translator import Translator
+from gremlin_python.process.traversal import Bytecode
log = logging.getLogger("gremlinpython")
@@ -63,7 +66,6 @@ class AbstractBaseProtocol(metaclass=abc.ABCMeta):
class GremlinServerWSProtocol(AbstractBaseProtocol):
-
QOP_AUTH_BIT = 1
_kerberos_context = None
_max_content_length = 10 * 1024 * 1024
@@ -133,7 +135,7 @@ class GremlinServerWSProtocol(AbstractBaseProtocol):
# This message is going to be huge and kind of hard to read, but in the event of an error,
# it can provide invaluable info, so space it out appropriately.
log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'",
- str(message), str(results_dict))
+ str(message), str(results_dict))
del results_dict[request_id]
raise GremlinServerError(message['status'])
@@ -185,8 +187,95 @@ class GremlinServerWSProtocol(AbstractBaseProtocol):
name_length = len(self._username)
fmt = '!I' + str(name_length) + 's'
word = self.QOP_AUTH_BIT << 24 | self._max_content_length
- out = struct.pack(fmt, word, self._username.encode("utf-8"),)
+ out = struct.pack(fmt, word, self._username.encode("utf-8"), )
encoded = base64.b64encode(out).decode('ascii')
kerberos.authGSSClientWrap(self._kerberos_context, encoded)
auth = kerberos.authGSSClientResponse(self._kerberos_context)
return request.RequestMessage('', 'authentication', {'sasl': auth})
+
+
+class GremlinServerHTTPProtocol(AbstractBaseProtocol):
+
+ def __init__(self,
+ message_serializer,
+ username='', password=''):
+ self._message_serializer = message_serializer
+ self._username = username
+ self._password = password
+
+ def connection_made(self, transport):
+ super(GremlinServerHTTPProtocol, self).connection_made(transport)
+
+ # Transforms request message into string
+ def write(self, request_id, request_message):
+ payload = {
+ 'requestId': request_id
+ }
+
+ gremlinArgs = request_message.args['gremlin']
+ useBytecode = isinstance(gremlinArgs, Bytecode)
+
+ # translate bytecode into scripts
+ if useBytecode:
+ request_message.args['gremlin'] = Translator().of('g').translate(gremlinArgs)
+ payload['op'] = 'bytecode'
+
+ # add all args into the payload
+ for k, v in request_message.args.items():
+ payload[k] = v
+
+ json_data = json.dumps(payload)
+
+ basic_auth = {}
+ if self._username and self._password:
+ basic_auth['username'] = self._username
+ basic_auth['password'] = self._password
+
+ message = {
+ 'headers': {'CONTENT-TYPE': 'application/json',
+ 'ACCEPT': str(self._message_serializer.version, encoding='utf-8')},
+ 'payload': json_data,
+ 'auth': basic_auth
+ }
+
+ self._transport.write(message)
+
+ def data_received(self, message, results_dict):
+ # if Gremlin Server cuts off then we get a None for the message
+ if message is None:
+ log.error("Received empty message from server.")
+ raise GremlinServerError({'code': 500,
+ 'message': 'Server disconnected - please try to reconnect', 'attributes': {}})
+
+ # if a request query cannot be compiled by Gremlin Server, the HTTP handler will send back the exception in
+ # json string without a requestId
+ if 'message' in message and 'requestId' not in message:
+ log.error("\r\nReceived error message from server '%s'", str(message))
+ raise GremlinServerError({'code': 400,
+ 'message': message, 'attributes': {}})
+
+ message = self._message_serializer.deserialize_message(message)
+ request_id = message['requestId']
+ result_set = results_dict[request_id] if request_id in results_dict else ResultSet(None, None)
+ status_code = message['status']['code']
+ aggregate_to = message['result']['meta'].get('aggregateTo', 'list')
+ data = message['result']['data']
+ result_set.aggregate_to = aggregate_to
+
+ if status_code == 204:
+ result_set.stream.put_nowait([])
+ del results_dict[request_id]
+ return status_code
+ elif status_code in [200, 206]:
+ result_set.stream.put_nowait(data)
+ if status_code == 200:
+ result_set.status_attributes = message['status']['attributes']
+ del results_dict[request_id]
+ return status_code
+ else:
+ # This message is going to be huge and kind of hard to read, but in the event of an error,
+ # it can provide invaluable info, so space it out appropriately.
+ log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'",
+ str(message), str(results_dict))
+ del results_dict[request_id]
+ raise GremlinServerError(message['status'])
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
index a0b8832119..74ed99523c 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
@@ -17,6 +17,7 @@
# under the License.
#
+import base64
import logging
import struct
import uuid
@@ -156,7 +157,8 @@ class GraphSONMessageSerializer(object):
return message
def deserialize_message(self, message):
- msg = json.loads(message.decode('utf-8'))
+ # for parsing string message via HTTP connections
+ msg = json.loads(message if isinstance(message, str) else message.decode('utf-8'))
return self._graphson_reader.to_object(msg)
@@ -268,7 +270,8 @@ class GraphBinarySerializersV1(object):
return bytes(ba)
def deserialize_message(self, message):
- b = io.BytesIO(message)
+ # for parsing string message via HTTP connections
+ b = io.BytesIO(base64.b64decode(message) if isinstance(message, str) else message)
b.read(1) # version
diff --git a/gremlin-python/src/main/python/gremlin_python/process/translator.py b/gremlin-python/src/main/python/gremlin_python/process/translator.py
index efbc4b42e3..9f29e44eb3 100755
--- a/gremlin-python/src/main/python/gremlin_python/process/translator.py
+++ b/gremlin-python/src/main/python/gremlin_python/process/translator.py
@@ -24,11 +24,11 @@ sent to any TinkerPop compliant HTTP endpoint.
"""
__author__ = 'Kelvin R. Lawrence (gfxman)'
-from gremlin_python.process.graph_traversal import __
-from gremlin_python.process.anonymous_traversal import traversal
+import re
+
from gremlin_python.process.traversal import *
-from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from gremlin_python.process.strategies import *
+from gremlin_python.structure.graph import Vertex, Edge, VertexProperty
from datetime import datetime
@@ -39,16 +39,16 @@ class Translator:
# Dictionary used to reverse-map token IDs to strings
options = {
- WithOptions.tokens: 'tokens',
- WithOptions.none: 'none',
- WithOptions.ids: 'ids',
- WithOptions.labels: 'labels',
- WithOptions.keys: 'keys',
- WithOptions.values: 'values',
- WithOptions.all: 'all',
- WithOptions.indexer: 'indexer',
- WithOptions.list: 'list',
- WithOptions.map: 'map'
+ WithOptions.tokens: 'tokens',
+ WithOptions.none: 'none',
+ WithOptions.ids: 'ids',
+ WithOptions.labels: 'labels',
+ WithOptions.keys: 'keys',
+ WithOptions.values: 'values',
+ WithOptions.all: 'all',
+ WithOptions.indexer: 'indexer',
+ WithOptions.list: 'list',
+ WithOptions.map: 'map'
}
def __init__(self, traversal_source=None):
@@ -60,20 +60,22 @@ class Translator:
def get_target_language(self):
return "gremlin-groovy"
- def of(self,traversal_source):
+ def of(self, traversal_source):
self.traversal_source = traversal_source
return self
# Do any needed special processing for the representation
- # of strings and dates.
+ # of strings and dates and boolean.
def fixup(self, v):
if isinstance(v, str):
return f'\'{v}\''
elif type(v) == datetime:
return self.process_date(v)
+ elif type(v) == bool:
+ return 'true' if v else 'false'
else:
return str(v)
-
+
# Turn a Python datetime into the equivalent new Date(...)
def process_date(self, date):
y = date.year - 1900
@@ -97,27 +99,56 @@ class Translator:
else:
res += self.fixup(p.value)
if p.other is not None:
- res+= ',' + self.fixup(p.other)
+ res += ',' + self.fixup(p.other)
res += ')'
return res
# Special processing to handle strategies
def process_strategy(self, s):
c = 0
- res = f'new {str(s)}('
- for key in s.configuration:
- res += ',' if c > 0 else ''
- res += key + ':'
- val = s.configuration[key]
- if isinstance(val, Traversal):
- res += self.translate(val.bytecode, child=True)
- else:
- res += self.fixup(val)
- c += 1
- res += ')'
+ res = ''
+ # if parameter is empty, only pass class name (referenced GroovyTranslator.java)
+ if not s.configuration:
+ res += s.__class__.__name__
+ else:
+ res = f'new {str(s)}('
+ for key in s.configuration:
+ res += ',' if c > 0 else ''
+ res += key + ':'
+ val = s.configuration[key]
+ if isinstance(val, Traversal):
+ res += self.translate(val.bytecode, child=True)
+ else:
+ res += self.fixup(val)
+ c += 1
+ res += ')'
return res
pass
+ # Special processing to handle vertices
+ def process_vertex(self, vertex):
+ return f'new ReferenceVertex({str(vertex.id)},\'{vertex.label}\')'
+
+ # Special processing to handle edges
+ def process_edge(self, edge):
+ return f'new ReferenceEdge({str(edge.id)},\'{edge.label}\',' \
+ f'new ReferenceVertex({str(edge.inV.id)},\'{edge.inV.label}\'),' \
+ f'new ReferenceVertex({str(edge.outV.id)},\'{edge.outV.label}\'))'
+
+ # Special processing to handle vertex property
+ def process_vertex_property(self, vp):
+ return f'new ReferenceVertexProperty({str(vp.id)},\'{vp.label}\',{self.fixup(vp.value)})'
+
+ # Special processing to handle lambda
+ def process_lambda(self, lam):
+ lambda_result = lam()
+ script = lambda_result if isinstance(lambda_result, str) else lambda_result[0]
+ return f'{script}' if re.match(r"^\{.*\}$", script) else f'{{{script}}}'
+
+ # Special processing to handle bindings inside of traversals
+ def process_binding(self, binding):
+ return f'Bindings.instance().of(\'{binding.key}\', {str(binding.value)})'
+
# Main driver of the translation. Different parts of
# a Traversal are handled appropriately.
def do_translation(self, step):
@@ -130,16 +161,21 @@ class Translator:
for p in params:
script += ',' if c > 0 else ''
if with_opts:
- script += f'WithOptions.{self.options[p]}'
+ script += f'WithOptions.{self.options[p]}'
elif type(p) == Bytecode:
script += self.translate(p, True)
elif isinstance(p, P):
script += self.process_predicate(p)
+ elif type(p) == Vertex:
+ script += self.process_vertex(p)
+ elif type(p) == Edge:
+ script += self.process_edge(p)
+ elif type(p) == VertexProperty:
+ script += self.process_vertex_property(p)
elif type(p) in [Cardinality, Pop, Operator]:
tmp = str(p)
- script += tmp[0:-1] if tmp.endswith('_') else tmp
- elif type(p) in [ReadOnlyStrategy, SubgraphStrategy, VertexProgramStrategy,
- OptionsStrategy, PartitionStrategy]:
+ script += tmp[0:-1] if tmp.endswith('_') else tmp
+ elif isinstance(p, TraversalStrategy): # this will capture all strategies
script += self.process_strategy(p)
elif type(p) == datetime:
script += self.process_date(p)
@@ -150,8 +186,14 @@ class Translator:
script += f'\'{p}\''
elif type(p) == bool:
script += 'true' if p else 'false'
+ elif isinstance(p, type(lambda: None)) and p.__name__ == (lambda: None).__name__:
+ script += self.process_lambda(p)
+ elif type(p) == Binding:
+ script += self.process_binding(p)
elif p is None:
script += 'null'
+ elif isinstance(p, type):
+ script += p.__name__
else:
script += str(p)
c += 1
@@ -165,11 +207,11 @@ class Translator:
# anonymous traversal style syntax.
def translate(self, bytecode, child=False):
script = '__' if child else self.traversal_source
-
+
for step in bytecode.source_instructions:
script += self.do_translation(step)
for step in bytecode.step_instructions:
script += self.do_translation(step)
-
+
return script
diff --git a/gremlin-python/src/main/python/tests/conftest.py b/gremlin-python/src/main/python/tests/conftest.py
index a3b76a4362..c80d107d26 100644
--- a/gremlin-python/src/main/python/tests/conftest.py
+++ b/gremlin-python/src/main/python/tests/conftest.py
@@ -34,7 +34,7 @@ from gremlin_python.driver.protocol import GremlinServerWSProtocol
from gremlin_python.driver.serializer import (
GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0,
GraphBinarySerializersV1)
-from gremlin_python.driver.aiohttp.transport import AiohttpTransport
+from gremlin_python.driver.aiohttp.transport import AiohttpTransport, AiohttpHTTPTransport
gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin')
gremlin_basic_auth_url = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL', 'wss://localhost:{}/gremlin')
@@ -43,6 +43,10 @@ anonymous_url = gremlin_server_url.format(45940)
basic_url = gremlin_basic_auth_url.format(45941)
kerberos_url = gremlin_server_url.format(45942)
kerberized_service = 'test-service@{}'.format(kerberos_hostname)
+gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/')
+gremlin_basic_auth_url_http = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL_HTTP', 'https://localhost:{}/')
+anonymous_url_http = gremlin_server_url_http.format(45940)
+basic_url_http = gremlin_basic_auth_url_http.format(45941)
verbose_logging = False
logging.basicConfig(format='%(asctime)s [%(levelname)8s] [%(filename)15s:%(lineno)d - %(funcName)10s()] - %(message)s',
@@ -209,3 +213,55 @@ def graphson_serializer_v3(request):
@pytest.fixture
def graphbinary_serializer_v1(request):
return GraphBinarySerializersV1()
+
+
+@pytest.fixture(params=['graphsonv2', 'graphsonv3', 'graphbinaryv1'])
+def remote_connection_http(request):
+ try:
+ if request.param == 'graphbinaryv1':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphBinarySerializersV1())
+ elif request.param == 'graphsonv2':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphSONSerializersV2d0())
+ elif request.param == 'graphsonv3':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphSONSerializersV3d0())
+ else:
+ raise ValueError("Invalid serializer option - " + request.param)
+ except OSError:
+ pytest.skip('Gremlin Server is not running')
+ else:
+ def fin():
+ remote_conn.close()
+
+ request.addfinalizer(fin)
+ return remote_conn
+
+
+"""
+# The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection will
+# only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now
+
+@pytest.fixture(params=['basic'])
+def remote_connection_http_authenticated(request):
+ try:
+ if request.param == 'basic':
+ # turn off certificate verification for testing purposes only
+ ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+ ssl_opts.verify_mode = ssl.CERT_NONE
+ remote_conn = DriverRemoteConnection(basic_url_http, 'gmodern',
+ username='stephen', password='password',
+ message_serializer=serializer.GraphSONSerializersV2d0(),
+ transport_factory=lambda: AiohttpHTTPTransport(ssl_options=ssl_opts))
+ else:
+ raise ValueError("Invalid authentication option - " + request.param)
+ except OSError:
+ pytest.skip('Gremlin Server is not running')
+ else:
+ def fin():
+ remote_conn.close()
+
+ request.addfinalizer(fin)
+ return remote_conn
+"""
diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py
new file mode 100644
index 0000000000..4d1fd82e17
--- /dev/null
+++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py
@@ -0,0 +1,214 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os
+
+from gremlin_python import statics
+from gremlin_python.driver.protocol import GremlinServerError
+from gremlin_python.statics import long
+from gremlin_python.process.traversal import Traverser
+from gremlin_python.process.traversal import TraversalStrategy
+from gremlin_python.process.traversal import Bindings
+from gremlin_python.process.traversal import P, Order, T
+from gremlin_python.process.graph_traversal import __
+from gremlin_python.process.anonymous_traversal import traversal
+from gremlin_python.structure.graph import Vertex
+from gremlin_python.process.strategies import SubgraphStrategy, ReservedKeysVerificationStrategy, SeedStrategy
+from gremlin_python.structure.io.util import HashableDict
+from gremlin_python.driver.serializer import GraphSONSerializersV2d0
+
+__author__ = 'Marko A. Rodriguez (http://markorodriguez.com)'
+
+gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/')
+test_no_auth_http_url = gremlin_server_url_http.format(45940)
+
+# due to the limitation of relying on python translator for sending scripts over HTTP, certain tests are omitted
+class TestDriverRemoteConnectionHttp(object):
+ def test_traversals(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http)
+
+ assert long(6) == g.V().count().toList()[0]
+ # #
+ assert Vertex(1) == g.V(1).next()
+ assert Vertex(1) == g.V(Vertex(1)).next()
+ assert 1 == g.V(1).id_().next()
+ assert Traverser(Vertex(1)) == g.V(1).nextTraverser()
+ assert 1 == len(g.V(1).toList())
+ assert isinstance(g.V(1).toList(), list)
+ results = g.V().repeat(__.out()).times(2).name
+ results = results.toList()
+ assert 2 == len(results)
+ assert "lop" in results
+ assert "ripple" in results
+ # #
+ assert 10 == g.V().repeat(__.both()).times(5)[0:10].count().next()
+ assert 1 == g.V().repeat(__.both()).times(5)[0:1].count().next()
+ assert 0 == g.V().repeat(__.both()).times(5)[0:0].count().next()
+ assert 4 == g.V()[2:].count().next()
+ assert 2 == g.V()[:2].count().next()
+ # #
+ results = g.withSideEffect('a', ['josh', 'peter']).V(1).out('created').in_('created').values('name').where(
+ P.within('a')).toList()
+ assert 2 == len(results)
+ assert 'josh' in results
+ assert 'peter' in results
+ # #
+ results = g.V().out().profile().toList()
+ assert 1 == len(results)
+ assert 'metrics' in results[0]
+ assert 'dur' in results[0]
+ # #
+ results = g.V().has('name', 'peter').as_('a').out('created').as_('b').select('a', 'b').by(
+ __.valueMap()).toList()
+ assert 1 == len(results)
+ assert 'peter' == results[0]['a']['name'][0]
+ assert 35 == results[0]['a']['age'][0]
+ assert 'lop' == results[0]['b']['name'][0]
+ assert 'java' == results[0]['b']['lang'][0]
+ assert 2 == len(results[0]['a'])
+ assert 2 == len(results[0]['b'])
+ # #
+ results = g.V(1).inject(g.V(2).next()).values('name').toList()
+ assert 2 == len(results)
+ assert 'marko' in results
+ assert 'vadas' in results
+ # #
+ results = g.V().has('person', 'name', 'marko').map(
+ lambda: ("it.get().value('name')", "gremlin-groovy")).toList()
+ assert 1 == len(results)
+ assert 'marko' in results
+ # #
+ # this test just validates that the underscored versions of steps conflicting with Gremlin work
+ # properly and can be removed when the old steps are removed - TINKERPOP-2272
+ results = g.V().filter_(__.values('age').sum_().and_(
+ __.max_().is_(P.gt(0)), __.min_().is_(P.gt(0)))).range_(0, 1).id_().next()
+ assert 1 == results
+ # #
+ # test binding in P
+ results = g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count().next()
+ assert 2 == results
+ # #
+ # test dict keys which can only work on GraphBinary and GraphSON3 which include specific serialization
+ # types for dict
+ if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0):
+ results = g.V().has('person', 'name', 'marko').elementMap("name").groupCount().next()
+ assert {HashableDict.of({T.id: 1, T.label: 'person', 'name': 'marko'}): 1} == results
+ if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0):
+ results = g.V().has('person', 'name', 'marko').both('knows').groupCount().by(
+ __.values('name').fold()).next()
+ assert {tuple(['vadas']): 1, tuple(['josh']): 1} == results
+
+ def test_iteration(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http)
+
+ t = g.V().count()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert 6 == t.next()
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+
+ t = g.V().has('name', P.within('marko', 'peter')).values('name').order()
+ assert "marko" == t.next()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert "peter" == t.next()
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+
+ try:
+ t.next()
+ assert False
+ except StopIteration:
+ assert True
+
+ def test_strategies(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(TraversalStrategy("SubgraphStrategy",
+ {"vertices": __.hasLabel("person"),
+ "edges": __.hasLabel("created")},
+ "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy"))
+ assert 4 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert 1 == g.V().label().dedup().count().next()
+ assert 4 == g.V().filter_(lambda: ("x -> true", "gremlin-groovy")).count().next()
+ assert "person" == g.V().label().dedup().next()
+ #
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created")))
+ assert 4 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert 1 == g.V().label().dedup().count().next()
+ assert "person" == g.V().label().dedup().next()
+ #
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(SubgraphStrategy(edges=__.hasLabel("created")))
+ assert 6 == g.V().count().next()
+ assert 4 == g.E().count().next()
+ assert 1 == g.E().label().dedup().count().next()
+ assert "created" == g.E().label().dedup().next()
+ #
+ g = g.withoutStrategies(SubgraphStrategy). \
+ withComputer(vertices=__.has("name", "marko"), edges=__.limit(0))
+ assert 1 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert "person" == g.V().label().next()
+ assert "marko" == g.V().name.next()
+ #
+ g = traversal().withRemote(remote_connection_http).withComputer()
+ assert 6 == g.V().count().next()
+ assert 6 == g.E().count().next()
+ #
+ g = traversal().withRemote(remote_connection_http).withStrategies(SeedStrategy(12345))
+ shuffledResult = g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+
+ def test_clone(self, remote_connection_http):
+ g = traversal().withRemote(remote_connection_http)
+ t = g.V().both()
+ assert 12 == len(t.toList())
+ assert 5 == t.clone().limit(5).count().next()
+ assert 10 == t.clone().limit(10).count().next()
+
+ """
+ # The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection
+ # only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now
+
+ def test_authenticated(self, remote_connection_http_authenticated):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http_authenticated)
+
+ assert long(6) == g.V().count().toList()[0]
+ """
diff --git a/gremlin-python/src/main/python/tests/process/test_translator.py b/gremlin-python/src/main/python/tests/process/test_translator.py
index 2afa148372..8a121a5437 100644
--- a/gremlin-python/src/main/python/tests/process/test_translator.py
+++ b/gremlin-python/src/main/python/tests/process/test_translator.py
@@ -21,13 +21,14 @@ Unit tests for the Translator Class.
"""
__author__ = 'Kelvin R. Lawrence (gfxman)'
-from gremlin_python.structure.graph import Graph
+from gremlin_python.structure.graph import Graph, Vertex, Edge, VertexProperty
+from gremlin_python.process.anonymous_traversal import traversal
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.translator import *
from datetime import datetime
-class TestTraversalStrategies(object):
+class TestTranslator(object):
def test_translations(self):
g = traversal().withGraph(Graph())
@@ -317,7 +318,7 @@ class TestTraversalStrategies(object):
"g.V('44').valueMap().with(WithOptions.tokens)"])
# 89
tests.append([g.withStrategies(ReadOnlyStrategy()).addV('test'),
- "g.withStrategies(new ReadOnlyStrategy()).addV('test')"])
+ "g.withStrategies(ReadOnlyStrategy).addV('test')"])
# 90
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route'))
tests.append([g.withStrategies(strategy).V().count(),
@@ -333,11 +334,11 @@ class TestTraversalStrategies(object):
# 93
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route'))
tests.append([g.withStrategies(ReadOnlyStrategy(),strategy).V().count(),
- "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"])
+ "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"])
# 94
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'))
tests.append([g.withStrategies(ReadOnlyStrategy(), strategy).V().count(),
- "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"])
+ "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"])
# 95
tests.append([g.with_('evaluationTimeout', 500).V().count(),
"g.withStrategies(new OptionsStrategy(evaluationTimeout:500)).V().count()"])
@@ -349,7 +350,7 @@ class TestTraversalStrategies(object):
"g.withStrategies(new PartitionStrategy(partitionKey:'partition',writePartition:'a',readPartitions:['a'])).addV('test')"])
# 98
tests.append([g.withComputer().V().shortestPath().with_(ShortestPath.target, __.has('name','peter')),
- "g.withStrategies(new VertexProgramStrategy()).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"])
+ "g.withStrategies(VertexProgramStrategy).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"])
# 99
tests.append([g.V().has("p1", starting_with("foo")),
@@ -373,6 +374,30 @@ class TestTraversalStrategies(object):
tests.append([g.V().has("p1", None),
"g.V().has('p1',null)"])
+ # 104
+ vertex = Vertex(0, "person")
+ tests.append([g.V(vertex),
+ "g.V(new ReferenceVertex(0,'person'))"])
+
+ # 105
+ outVertex = Vertex(0, "person")
+ inVertex = Vertex(1, "person")
+ edge = Edge(2, outVertex, "knows", inVertex)
+ tests.append([g.inject(edge),
+ "g.inject(new ReferenceEdge(2,'knows',new ReferenceVertex(1,'person'),new ReferenceVertex(0,'person')))"])
+
+ # 106
+ vp = VertexProperty(3, "time", "18:00", None)
+ tests.append([g.inject(vp),
+ "g.inject(new ReferenceVertexProperty(3,'time','18:00'))"])
+
+ # 107
+ tests.append([g.V().has('person', 'name', 'marko').map(lambda: ("it.get().value('name')", "gremlin-groovy")),
+ "g.V().has('person','name','marko').map({it.get().value('name')})"])
+
+ # 108
+ tests.append([g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count(),
+ "g.V().has('person','age',Bindings.instance().of('x', lt(30))).count()"])
tlr = Translator().of('g')