You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tinkerpop.apache.org by xi...@apache.org on 2023/10/13 05:23:41 UTC
[tinkerpop] branch add-http-gremlin-python created (now b2ef3dbb37)
This is an automated email from the ASF dual-hosted git repository.
xiazcy pushed a change to branch add-http-gremlin-python
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
at b2ef3dbb37 Enable remote connection over HTTP for gremlin python
This branch includes the following new commits:
new b2ef3dbb37 Enable remote connection over HTTP for gremlin python
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[tinkerpop] 01/01: Enable remote connection over HTTP for gremlin python
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiazcy pushed a commit to branch add-http-gremlin-python
in repository https://gitbox.apache.org/repos/asf/tinkerpop.git
commit b2ef3dbb37a9e34e91319214a751b575eee94a14
Author: Yang Xia <55...@users.noreply.github.com>
AuthorDate: Thu Oct 12 22:23:23 2023 -0700
Enable remote connection over HTTP for gremlin python
---
CHANGELOG.asciidoc | 2 +
gremlin-python/docker-compose.yml | 2 +
.../gremlin_python/driver/aiohttp/transport.py | 101 ++++++++++
.../main/python/gremlin_python/driver/client.py | 34 +++-
.../main/python/gremlin_python/driver/protocol.py | 95 ++++++++-
.../python/gremlin_python/driver/serializer.py | 7 +-
.../python/gremlin_python/process/translator.py | 110 +++++++----
gremlin-python/src/main/python/tests/conftest.py | 58 +++++-
.../driver/test_driver_remote_connection_http.py | 214 +++++++++++++++++++++
.../main/python/tests/process/test_translator.py | 37 +++-
10 files changed, 605 insertions(+), 55 deletions(-)
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
index 6ee4818231..152f66a8fa 100644
--- a/CHANGELOG.asciidoc
+++ b/CHANGELOG.asciidoc
@@ -28,6 +28,8 @@ This release also includes changes from <<release-3-5-8, 3.5.8>>.
* Fixed a javadoc comment in `GraphTraversal.not()` method.
* Allowed `gremlin-driver` to be used over HTTP for experimental purposes.
* Deprecated the `HandshakeInterceptor` in favor of a more generic `RequestInterceptor`.
+* Allowed `gremlin-python` to be used over HTTP for experimental purposes.
+* Updated `gremlin-python` translator to enable additional traversals over HTTP.
* Fixed a bug in `StarGraph` where `EdgeFilter` did not remove associated Edge Properties.
* Added translator to the Go GLV.
* Fixed bug with filtering for `group()` when the side-effect label was defined for it.
diff --git a/gremlin-python/docker-compose.yml b/gremlin-python/docker-compose.yml
index 6262c61da8..02e693ee5b 100644
--- a/gremlin-python/docker-compose.yml
+++ b/gremlin-python/docker-compose.yml
@@ -58,6 +58,8 @@ services:
- KRB5CCNAME=./test-tkt.cc
- GREMLIN_SERVER_URL=ws://gremlin-server-test-python:{}/gremlin
- GREMLIN_SERVER_BASIC_AUTH_URL=wss://gremlin-server-test-python:{}/gremlin
+ - GREMLIN_SERVER_URL_HTTP=http://gremlin-server-test-python:{}/
+ - GREMLIN_SERVER_BASIC_AUTH_URL_HTTP=https://gremlin-server-test-python:{}/
- KRB_HOSTNAME=${KRB_HOSTNAME:-gremlin-server-test}
- VERSION=${VERSION}
working_dir: /python_app
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
index 34dab16b53..4b9ead23cc 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/aiohttp/transport.py
@@ -138,3 +138,104 @@ class AiohttpTransport(AbstractBaseTransport):
def closed(self):
# Connection is closed if either the websocket or the client session is closed.
return self._websocket.closed or self._client_session.closed
+
+
+class AiohttpHTTPTransport(AbstractBaseTransport):
+ nest_asyncio_applied = False
+
+ def __init__(self, call_from_event_loop=None, read_timeout=None, write_timeout=None, **kwargs):
+ if call_from_event_loop is not None and call_from_event_loop and not AiohttpTransport.nest_asyncio_applied:
+ """
+ The AiohttpTransport implementation uses the asyncio event loop. Because of this, it cannot be called
+ within an event loop without nest_asyncio. If the code is ever refactored so that it can be called
+ within an event loop this import and call can be removed. Without this, applications which use the
+ event loop to call gremlin-python (such as Jupyter) will not work.
+ """
+ import nest_asyncio
+ nest_asyncio.apply()
+ AiohttpTransport.nest_asyncio_applied = True
+
+ # Start event loop and initialize client session and response to None
+ self._loop = asyncio.new_event_loop()
+ self._client_session = None
+ self._http_req_resp = None
+ self._enable_ssl = False
+
+ # Set all inner variables to parameters passed in.
+ self._aiohttp_kwargs = kwargs
+ self._write_timeout = write_timeout
+ self._read_timeout = read_timeout
+ if "ssl_options" in self._aiohttp_kwargs:
+ self._ssl_context = self._aiohttp_kwargs.pop("ssl_options")
+ self._enable_ssl = True
+
+ # if "ssl_options" in self._aiohttp_kwargs:
+ # self._aiohttp_kwargs["ssl_context"] = self._aiohttp_kwargs.pop("ssl_options")
+
+ def __del__(self):
+ # Close will only actually close if things are left open, so this is safe to call.
+ # Clean up any connection resources and close the event loop.
+ self.close()
+
+ def connect(self, url, headers=None):
+ # Inner function to perform async connect.
+ async def async_connect():
+ # Start client session and use it to send all HTTP requests. Base url is the endpoint, headers are set here
+ # Base url can only parse basic url with no path, see https://github.com/aio-libs/aiohttp/issues/6647
+ if self._enable_ssl:
+ # ssl context is established through tcp connector
+ tcp_conn = aiohttp.TCPConnector(ssl_context=self._ssl_context)
+ self._client_session = aiohttp.ClientSession(connector=tcp_conn,
+ base_url=url, headers=headers, loop=self._loop)
+ else:
+ self._client_session = aiohttp.ClientSession(base_url=url, headers=headers, loop=self._loop)
+
+
+ # Execute the async connect synchronously.
+ self._loop.run_until_complete(async_connect())
+
+ def write(self, message):
+ # Inner function to perform async write.
+ async def async_write():
+ basic_auth = None
+ # basic password authentication for https connections
+ if message['auth']:
+ basic_auth = aiohttp.BasicAuth(message['auth']['username'], message['auth']['password'])
+ async with async_timeout.timeout(self._write_timeout):
+ self._http_req_resp = await self._client_session.post(url="/gremlin",
+ auth=basic_auth,
+ data=message['payload'],
+ headers=message['headers'],
+ **self._aiohttp_kwargs)
+
+ # Execute the async write synchronously.
+ self._loop.run_until_complete(async_write())
+
+ def read(self):
+ # Inner function to perform async read.
+ async def async_read():
+ async with async_timeout.timeout(self._read_timeout):
+ # using http request read()
+ return await self._http_req_resp.text()
+
+ return self._loop.run_until_complete(async_read())
+
+ def close(self):
+ # Inner function to perform async close.
+ async def async_close():
+ if self._client_session is not None and not self._client_session.closed:
+ await self._client_session.close()
+ self._client_session = None
+
+ # If the loop is not closed (connection hasn't already been closed)
+ if not self._loop.is_closed():
+ # Execute the async close synchronously.
+ self._loop.run_until_complete(async_close())
+
+ # Close the event loop.
+ self._loop.close()
+
+ @property
+ def closed(self):
+ # Connection is closed when client session is closed.
+ return self._client_session.closed
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/client.py b/gremlin-python/src/main/python/gremlin_python/driver/client.py
index ec7078d593..6ad9fd5517 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/client.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/client.py
@@ -19,6 +19,7 @@
import logging
import warnings
import queue
+import re
from concurrent.futures import ThreadPoolExecutor
from gremlin_python.driver import connection, protocol, request, serializer
@@ -45,15 +46,20 @@ class Client:
kerberized_service="", headers=None, session=None,
enable_user_agent_on_connect=True, **transport_kwargs):
log.info("Creating Client with url '%s'", url)
+
+ # check via url that we are using http protocol
+ self._use_http = re.search('^http', url)
+
self._closed = False
self._url = url
self._headers = headers
self._enable_user_agent_on_connect = enable_user_agent_on_connect
self._traversal_source = traversal_source
- if "max_content_length" not in transport_kwargs:
+ if not self._use_http and "max_content_length" not in transport_kwargs:
transport_kwargs["max_content_length"] = 10 * 1024 * 1024
if message_serializer is None:
message_serializer = serializer.GraphBinarySerializersV1()
+ # message_serializer = serializer.GraphSONMessageSerializer()
self._message_serializer = message_serializer
self._username = username
@@ -63,21 +69,31 @@ class Client:
if transport_factory is None:
try:
from gremlin_python.driver.aiohttp.transport import (
- AiohttpTransport)
+ AiohttpTransport, AiohttpHTTPTransport)
except ImportError:
raise Exception("Please install AIOHTTP or pass "
"custom transport factory")
else:
def transport_factory():
- return AiohttpTransport(**transport_kwargs)
+ if self._use_http:
+ return AiohttpHTTPTransport(**transport_kwargs)
+ else:
+ return AiohttpTransport(**transport_kwargs)
self._transport_factory = transport_factory
if protocol_factory is None:
- def protocol_factory(): return protocol.GremlinServerWSProtocol(
- self._message_serializer,
- username=self._username,
- password=self._password,
- kerberized_service=kerberized_service,
- max_content_length=transport_kwargs["max_content_length"])
+ def protocol_factory():
+ if self._use_http:
+ return protocol.GremlinServerHTTPProtocol(
+ self._message_serializer,
+ username=self._username,
+ password=self._password)
+ else:
+ return protocol.GremlinServerWSProtocol(
+ self._message_serializer,
+ username=self._username,
+ password=self._password,
+ kerberized_service=kerberized_service,
+ max_content_length=transport_kwargs["max_content_length"])
self._protocol_factory = protocol_factory
if self._session_enabled:
if pool_size is None:
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
index 9323210356..6aaa9b8efe 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/protocol.py
@@ -16,6 +16,7 @@
# specific language governing permissions and limitations
# under the License.
#
+import json
import logging
import abc
import base64
@@ -25,6 +26,8 @@ import struct
from gremlin_python.driver import request
from gremlin_python.driver.resultset import ResultSet
+from gremlin_python.process.translator import Translator
+from gremlin_python.process.traversal import Bytecode
log = logging.getLogger("gremlinpython")
@@ -63,7 +66,6 @@ class AbstractBaseProtocol(metaclass=abc.ABCMeta):
class GremlinServerWSProtocol(AbstractBaseProtocol):
-
QOP_AUTH_BIT = 1
_kerberos_context = None
_max_content_length = 10 * 1024 * 1024
@@ -133,7 +135,7 @@ class GremlinServerWSProtocol(AbstractBaseProtocol):
# This message is going to be huge and kind of hard to read, but in the event of an error,
# it can provide invaluable info, so space it out appropriately.
log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'",
- str(message), str(results_dict))
+ str(message), str(results_dict))
del results_dict[request_id]
raise GremlinServerError(message['status'])
@@ -185,8 +187,95 @@ class GremlinServerWSProtocol(AbstractBaseProtocol):
name_length = len(self._username)
fmt = '!I' + str(name_length) + 's'
word = self.QOP_AUTH_BIT << 24 | self._max_content_length
- out = struct.pack(fmt, word, self._username.encode("utf-8"),)
+ out = struct.pack(fmt, word, self._username.encode("utf-8"), )
encoded = base64.b64encode(out).decode('ascii')
kerberos.authGSSClientWrap(self._kerberos_context, encoded)
auth = kerberos.authGSSClientResponse(self._kerberos_context)
return request.RequestMessage('', 'authentication', {'sasl': auth})
+
+
+class GremlinServerHTTPProtocol(AbstractBaseProtocol):
+
+ def __init__(self,
+ message_serializer,
+ username='', password=''):
+ self._message_serializer = message_serializer
+ self._username = username
+ self._password = password
+
+ def connection_made(self, transport):
+ super(GremlinServerHTTPProtocol, self).connection_made(transport)
+
+ # Transforms request message into string
+ def write(self, request_id, request_message):
+ payload = {
+ 'requestId': request_id
+ }
+
+ gremlinArgs = request_message.args['gremlin']
+ useBytecode = isinstance(gremlinArgs, Bytecode)
+
+ # translate bytecode into scripts
+ if useBytecode:
+ request_message.args['gremlin'] = Translator().of('g').translate(gremlinArgs)
+ payload['op'] = 'bytecode'
+
+ # add all args into the payload
+ for k, v in request_message.args.items():
+ payload[k] = v
+
+ json_data = json.dumps(payload)
+
+ basic_auth = {}
+ if self._username and self._password:
+ basic_auth['username'] = self._username
+ basic_auth['password'] = self._password
+
+ message = {
+ 'headers': {'CONTENT-TYPE': 'application/json',
+ 'ACCEPT': str(self._message_serializer.version, encoding='utf-8')},
+ 'payload': json_data,
+ 'auth': basic_auth
+ }
+
+ self._transport.write(message)
+
+ def data_received(self, message, results_dict):
+ # if Gremlin Server cuts off then we get a None for the message
+ if message is None:
+ log.error("Received empty message from server.")
+ raise GremlinServerError({'code': 500,
+ 'message': 'Server disconnected - please try to reconnect', 'attributes': {}})
+
+ # if a request query cannot be compiled by Gremlin Server, the HTTP handler will send back the exception in
+ # json string without a requestId
+ if 'message' in message and 'requestId' not in message:
+ log.error("\r\nReceived error message from server '%s'", str(message))
+ raise GremlinServerError({'code': 400,
+ 'message': message, 'attributes': {}})
+
+ message = self._message_serializer.deserialize_message(message)
+ request_id = message['requestId']
+ result_set = results_dict[request_id] if request_id in results_dict else ResultSet(None, None)
+ status_code = message['status']['code']
+ aggregate_to = message['result']['meta'].get('aggregateTo', 'list')
+ data = message['result']['data']
+ result_set.aggregate_to = aggregate_to
+
+ if status_code == 204:
+ result_set.stream.put_nowait([])
+ del results_dict[request_id]
+ return status_code
+ elif status_code in [200, 206]:
+ result_set.stream.put_nowait(data)
+ if status_code == 200:
+ result_set.status_attributes = message['status']['attributes']
+ del results_dict[request_id]
+ return status_code
+ else:
+ # This message is going to be huge and kind of hard to read, but in the event of an error,
+ # it can provide invaluable info, so space it out appropriately.
+ log.error("\r\nReceived error message '%s'\r\n\r\nWith results dictionary '%s'",
+ str(message), str(results_dict))
+ del results_dict[request_id]
+ raise GremlinServerError(message['status'])
diff --git a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
index a0b8832119..74ed99523c 100644
--- a/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
+++ b/gremlin-python/src/main/python/gremlin_python/driver/serializer.py
@@ -17,6 +17,7 @@
# under the License.
#
+import base64
import logging
import struct
import uuid
@@ -156,7 +157,8 @@ class GraphSONMessageSerializer(object):
return message
def deserialize_message(self, message):
- msg = json.loads(message.decode('utf-8'))
+ # for parsing string message via HTTP connections
+ msg = json.loads(message if isinstance(message, str) else message.decode('utf-8'))
return self._graphson_reader.to_object(msg)
@@ -268,7 +270,8 @@ class GraphBinarySerializersV1(object):
return bytes(ba)
def deserialize_message(self, message):
- b = io.BytesIO(message)
+ # for parsing string message via HTTP connections
+ b = io.BytesIO(base64.b64decode(message) if isinstance(message, str) else message)
b.read(1) # version
diff --git a/gremlin-python/src/main/python/gremlin_python/process/translator.py b/gremlin-python/src/main/python/gremlin_python/process/translator.py
index efbc4b42e3..9f29e44eb3 100755
--- a/gremlin-python/src/main/python/gremlin_python/process/translator.py
+++ b/gremlin-python/src/main/python/gremlin_python/process/translator.py
@@ -24,11 +24,11 @@ sent to any TinkerPop compliant HTTP endpoint.
"""
__author__ = 'Kelvin R. Lawrence (gfxman)'
-from gremlin_python.process.graph_traversal import __
-from gremlin_python.process.anonymous_traversal import traversal
+import re
+
from gremlin_python.process.traversal import *
-from gremlin_python.driver.driver_remote_connection import DriverRemoteConnection
from gremlin_python.process.strategies import *
+from gremlin_python.structure.graph import Vertex, Edge, VertexProperty
from datetime import datetime
@@ -39,16 +39,16 @@ class Translator:
# Dictionary used to reverse-map token IDs to strings
options = {
- WithOptions.tokens: 'tokens',
- WithOptions.none: 'none',
- WithOptions.ids: 'ids',
- WithOptions.labels: 'labels',
- WithOptions.keys: 'keys',
- WithOptions.values: 'values',
- WithOptions.all: 'all',
- WithOptions.indexer: 'indexer',
- WithOptions.list: 'list',
- WithOptions.map: 'map'
+ WithOptions.tokens: 'tokens',
+ WithOptions.none: 'none',
+ WithOptions.ids: 'ids',
+ WithOptions.labels: 'labels',
+ WithOptions.keys: 'keys',
+ WithOptions.values: 'values',
+ WithOptions.all: 'all',
+ WithOptions.indexer: 'indexer',
+ WithOptions.list: 'list',
+ WithOptions.map: 'map'
}
def __init__(self, traversal_source=None):
@@ -60,20 +60,22 @@ class Translator:
def get_target_language(self):
return "gremlin-groovy"
- def of(self,traversal_source):
+ def of(self, traversal_source):
self.traversal_source = traversal_source
return self
# Do any needed special processing for the representation
- # of strings and dates.
+ # of strings and dates and boolean.
def fixup(self, v):
if isinstance(v, str):
return f'\'{v}\''
elif type(v) == datetime:
return self.process_date(v)
+ elif type(v) == bool:
+ return 'true' if v else 'false'
else:
return str(v)
-
+
# Turn a Python datetime into the equivalent new Date(...)
def process_date(self, date):
y = date.year - 1900
@@ -97,27 +99,56 @@ class Translator:
else:
res += self.fixup(p.value)
if p.other is not None:
- res+= ',' + self.fixup(p.other)
+ res += ',' + self.fixup(p.other)
res += ')'
return res
# Special processing to handle strategies
def process_strategy(self, s):
c = 0
- res = f'new {str(s)}('
- for key in s.configuration:
- res += ',' if c > 0 else ''
- res += key + ':'
- val = s.configuration[key]
- if isinstance(val, Traversal):
- res += self.translate(val.bytecode, child=True)
- else:
- res += self.fixup(val)
- c += 1
- res += ')'
+ res = ''
+ # if parameter is empty, only pass class name (referenced GroovyTranslator.java)
+ if not s.configuration:
+ res += s.__class__.__name__
+ else:
+ res = f'new {str(s)}('
+ for key in s.configuration:
+ res += ',' if c > 0 else ''
+ res += key + ':'
+ val = s.configuration[key]
+ if isinstance(val, Traversal):
+ res += self.translate(val.bytecode, child=True)
+ else:
+ res += self.fixup(val)
+ c += 1
+ res += ')'
return res
pass
+ # Special processing to handle vertices
+ def process_vertex(self, vertex):
+ return f'new ReferenceVertex({str(vertex.id)},\'{vertex.label}\')'
+
+ # Special processing to handle edges
+ def process_edge(self, edge):
+ return f'new ReferenceEdge({str(edge.id)},\'{edge.label}\',' \
+ f'new ReferenceVertex({str(edge.inV.id)},\'{edge.inV.label}\'),' \
+ f'new ReferenceVertex({str(edge.outV.id)},\'{edge.outV.label}\'))'
+
+ # Special processing to handle vertex property
+ def process_vertex_property(self, vp):
+ return f'new ReferenceVertexProperty({str(vp.id)},\'{vp.label}\',{self.fixup(vp.value)})'
+
+ # Special processing to handle lambda
+ def process_lambda(self, lam):
+ lambda_result = lam()
+ script = lambda_result if isinstance(lambda_result, str) else lambda_result[0]
+ return f'{script}' if re.match(r"^\{.*\}$", script) else f'{{{script}}}'
+
+ # Special processing to handle bindings inside of traversals
+ def process_binding(self, binding):
+ return f'Bindings.instance().of(\'{binding.key}\', {str(binding.value)})'
+
# Main driver of the translation. Different parts of
# a Traversal are handled appropriately.
def do_translation(self, step):
@@ -130,16 +161,21 @@ class Translator:
for p in params:
script += ',' if c > 0 else ''
if with_opts:
- script += f'WithOptions.{self.options[p]}'
+ script += f'WithOptions.{self.options[p]}'
elif type(p) == Bytecode:
script += self.translate(p, True)
elif isinstance(p, P):
script += self.process_predicate(p)
+ elif type(p) == Vertex:
+ script += self.process_vertex(p)
+ elif type(p) == Edge:
+ script += self.process_edge(p)
+ elif type(p) == VertexProperty:
+ script += self.process_vertex_property(p)
elif type(p) in [Cardinality, Pop, Operator]:
tmp = str(p)
- script += tmp[0:-1] if tmp.endswith('_') else tmp
- elif type(p) in [ReadOnlyStrategy, SubgraphStrategy, VertexProgramStrategy,
- OptionsStrategy, PartitionStrategy]:
+ script += tmp[0:-1] if tmp.endswith('_') else tmp
+ elif isinstance(p, TraversalStrategy): # this will capture all strategies
script += self.process_strategy(p)
elif type(p) == datetime:
script += self.process_date(p)
@@ -150,8 +186,14 @@ class Translator:
script += f'\'{p}\''
elif type(p) == bool:
script += 'true' if p else 'false'
+ elif isinstance(p, type(lambda: None)) and p.__name__ == (lambda: None).__name__:
+ script += self.process_lambda(p)
+ elif type(p) == Binding:
+ script += self.process_binding(p)
elif p is None:
script += 'null'
+ elif isinstance(p, type):
+ script += p.__name__
else:
script += str(p)
c += 1
@@ -165,11 +207,11 @@ class Translator:
# anonymous traversal style syntax.
def translate(self, bytecode, child=False):
script = '__' if child else self.traversal_source
-
+
for step in bytecode.source_instructions:
script += self.do_translation(step)
for step in bytecode.step_instructions:
script += self.do_translation(step)
-
+
return script
diff --git a/gremlin-python/src/main/python/tests/conftest.py b/gremlin-python/src/main/python/tests/conftest.py
index a3b76a4362..c80d107d26 100644
--- a/gremlin-python/src/main/python/tests/conftest.py
+++ b/gremlin-python/src/main/python/tests/conftest.py
@@ -34,7 +34,7 @@ from gremlin_python.driver.protocol import GremlinServerWSProtocol
from gremlin_python.driver.serializer import (
GraphSONMessageSerializer, GraphSONSerializersV2d0, GraphSONSerializersV3d0,
GraphBinarySerializersV1)
-from gremlin_python.driver.aiohttp.transport import AiohttpTransport
+from gremlin_python.driver.aiohttp.transport import AiohttpTransport, AiohttpHTTPTransport
gremlin_server_url = os.environ.get('GREMLIN_SERVER_URL', 'ws://localhost:{}/gremlin')
gremlin_basic_auth_url = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL', 'wss://localhost:{}/gremlin')
@@ -43,6 +43,10 @@ anonymous_url = gremlin_server_url.format(45940)
basic_url = gremlin_basic_auth_url.format(45941)
kerberos_url = gremlin_server_url.format(45942)
kerberized_service = 'test-service@{}'.format(kerberos_hostname)
+gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/')
+gremlin_basic_auth_url_http = os.environ.get('GREMLIN_SERVER_BASIC_AUTH_URL_HTTP', 'https://localhost:{}/')
+anonymous_url_http = gremlin_server_url_http.format(45940)
+basic_url_http = gremlin_basic_auth_url_http.format(45941)
verbose_logging = False
logging.basicConfig(format='%(asctime)s [%(levelname)8s] [%(filename)15s:%(lineno)d - %(funcName)10s()] - %(message)s',
@@ -209,3 +213,55 @@ def graphson_serializer_v3(request):
@pytest.fixture
def graphbinary_serializer_v1(request):
return GraphBinarySerializersV1()
+
+
+@pytest.fixture(params=['graphsonv2', 'graphsonv3', 'graphbinaryv1'])
+def remote_connection_http(request):
+ try:
+ if request.param == 'graphbinaryv1':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphBinarySerializersV1())
+ elif request.param == 'graphsonv2':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphSONSerializersV2d0())
+ elif request.param == 'graphsonv3':
+ remote_conn = DriverRemoteConnection(anonymous_url_http, 'gmodern',
+ message_serializer=serializer.GraphSONSerializersV3d0())
+ else:
+ raise ValueError("Invalid serializer option - " + request.param)
+ except OSError:
+ pytest.skip('Gremlin Server is not running')
+ else:
+ def fin():
+ remote_conn.close()
+
+ request.addfinalizer(fin)
+ return remote_conn
+
+
+"""
+# The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection will
+# only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now
+
+@pytest.fixture(params=['basic'])
+def remote_connection_http_authenticated(request):
+ try:
+ if request.param == 'basic':
+ # turn off certificate verification for testing purposes only
+ ssl_opts = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+ ssl_opts.verify_mode = ssl.CERT_NONE
+ remote_conn = DriverRemoteConnection(basic_url_http, 'gmodern',
+ username='stephen', password='password',
+ message_serializer=serializer.GraphSONSerializersV2d0(),
+ transport_factory=lambda: AiohttpHTTPTransport(ssl_options=ssl_opts))
+ else:
+ raise ValueError("Invalid authentication option - " + request.param)
+ except OSError:
+ pytest.skip('Gremlin Server is not running')
+ else:
+ def fin():
+ remote_conn.close()
+
+ request.addfinalizer(fin)
+ return remote_conn
+"""
diff --git a/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py
new file mode 100644
index 0000000000..4d1fd82e17
--- /dev/null
+++ b/gremlin-python/src/main/python/tests/driver/test_driver_remote_connection_http.py
@@ -0,0 +1,214 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+import os
+
+from gremlin_python import statics
+from gremlin_python.driver.protocol import GremlinServerError
+from gremlin_python.statics import long
+from gremlin_python.process.traversal import Traverser
+from gremlin_python.process.traversal import TraversalStrategy
+from gremlin_python.process.traversal import Bindings
+from gremlin_python.process.traversal import P, Order, T
+from gremlin_python.process.graph_traversal import __
+from gremlin_python.process.anonymous_traversal import traversal
+from gremlin_python.structure.graph import Vertex
+from gremlin_python.process.strategies import SubgraphStrategy, ReservedKeysVerificationStrategy, SeedStrategy
+from gremlin_python.structure.io.util import HashableDict
+from gremlin_python.driver.serializer import GraphSONSerializersV2d0
+
+__author__ = 'Marko A. Rodriguez (http://markorodriguez.com)'
+
+gremlin_server_url_http = os.environ.get('GREMLIN_SERVER_URL_HTTP', 'http://localhost:{}/')
+test_no_auth_http_url = gremlin_server_url_http.format(45940)
+
+# due to the limitation of relying on python translator for sending scripts over HTTP, certain tests are omitted
+class TestDriverRemoteConnectionHttp(object):
+ def test_traversals(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http)
+
+ assert long(6) == g.V().count().toList()[0]
+ # #
+ assert Vertex(1) == g.V(1).next()
+ assert Vertex(1) == g.V(Vertex(1)).next()
+ assert 1 == g.V(1).id_().next()
+ assert Traverser(Vertex(1)) == g.V(1).nextTraverser()
+ assert 1 == len(g.V(1).toList())
+ assert isinstance(g.V(1).toList(), list)
+ results = g.V().repeat(__.out()).times(2).name
+ results = results.toList()
+ assert 2 == len(results)
+ assert "lop" in results
+ assert "ripple" in results
+ # #
+ assert 10 == g.V().repeat(__.both()).times(5)[0:10].count().next()
+ assert 1 == g.V().repeat(__.both()).times(5)[0:1].count().next()
+ assert 0 == g.V().repeat(__.both()).times(5)[0:0].count().next()
+ assert 4 == g.V()[2:].count().next()
+ assert 2 == g.V()[:2].count().next()
+ # #
+ results = g.withSideEffect('a', ['josh', 'peter']).V(1).out('created').in_('created').values('name').where(
+ P.within('a')).toList()
+ assert 2 == len(results)
+ assert 'josh' in results
+ assert 'peter' in results
+ # #
+ results = g.V().out().profile().toList()
+ assert 1 == len(results)
+ assert 'metrics' in results[0]
+ assert 'dur' in results[0]
+ # #
+ results = g.V().has('name', 'peter').as_('a').out('created').as_('b').select('a', 'b').by(
+ __.valueMap()).toList()
+ assert 1 == len(results)
+ assert 'peter' == results[0]['a']['name'][0]
+ assert 35 == results[0]['a']['age'][0]
+ assert 'lop' == results[0]['b']['name'][0]
+ assert 'java' == results[0]['b']['lang'][0]
+ assert 2 == len(results[0]['a'])
+ assert 2 == len(results[0]['b'])
+ # #
+ results = g.V(1).inject(g.V(2).next()).values('name').toList()
+ assert 2 == len(results)
+ assert 'marko' in results
+ assert 'vadas' in results
+ # #
+ results = g.V().has('person', 'name', 'marko').map(
+ lambda: ("it.get().value('name')", "gremlin-groovy")).toList()
+ assert 1 == len(results)
+ assert 'marko' in results
+ # #
+ # this test just validates that the underscored versions of steps conflicting with Gremlin work
+ # properly and can be removed when the old steps are removed - TINKERPOP-2272
+ results = g.V().filter_(__.values('age').sum_().and_(
+ __.max_().is_(P.gt(0)), __.min_().is_(P.gt(0)))).range_(0, 1).id_().next()
+ assert 1 == results
+ # #
+ # test binding in P
+ results = g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count().next()
+ assert 2 == results
+ # #
+ # test dict keys which can only work on GraphBinary and GraphSON3 which include specific serialization
+ # types for dict
+ if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0):
+ results = g.V().has('person', 'name', 'marko').elementMap("name").groupCount().next()
+ assert {HashableDict.of({T.id: 1, T.label: 'person', 'name': 'marko'}): 1} == results
+ if not isinstance(remote_connection_http._client._message_serializer, GraphSONSerializersV2d0):
+ results = g.V().has('person', 'name', 'marko').both('knows').groupCount().by(
+ __.values('name').fold()).next()
+ assert {tuple(['vadas']): 1, tuple(['josh']): 1} == results
+
+ def test_iteration(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http)
+
+ t = g.V().count()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert 6 == t.next()
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+
+ t = g.V().has('name', P.within('marko', 'peter')).values('name').order()
+ assert "marko" == t.next()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert t.hasNext()
+ assert "peter" == t.next()
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+ assert not (t.hasNext())
+
+ try:
+ t.next()
+ assert False
+ except StopIteration:
+ assert True
+
+ def test_strategies(self, remote_connection_http):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(TraversalStrategy("SubgraphStrategy",
+ {"vertices": __.hasLabel("person"),
+ "edges": __.hasLabel("created")},
+ "org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy"))
+ assert 4 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert 1 == g.V().label().dedup().count().next()
+ assert 4 == g.V().filter_(lambda: ("x -> true", "gremlin-groovy")).count().next()
+ assert "person" == g.V().label().dedup().next()
+ #
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(SubgraphStrategy(vertices=__.hasLabel("person"), edges=__.hasLabel("created")))
+ assert 4 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert 1 == g.V().label().dedup().count().next()
+ assert "person" == g.V().label().dedup().next()
+ #
+ g = traversal().withRemote(remote_connection_http). \
+ withStrategies(SubgraphStrategy(edges=__.hasLabel("created")))
+ assert 6 == g.V().count().next()
+ assert 4 == g.E().count().next()
+ assert 1 == g.E().label().dedup().count().next()
+ assert "created" == g.E().label().dedup().next()
+ #
+ g = g.withoutStrategies(SubgraphStrategy). \
+ withComputer(vertices=__.has("name", "marko"), edges=__.limit(0))
+ assert 1 == g.V().count().next()
+ assert 0 == g.E().count().next()
+ assert "person" == g.V().label().next()
+ assert "marko" == g.V().name.next()
+ #
+ g = traversal().withRemote(remote_connection_http).withComputer()
+ assert 6 == g.V().count().next()
+ assert 6 == g.E().count().next()
+ #
+ g = traversal().withRemote(remote_connection_http).withStrategies(SeedStrategy(12345))
+ shuffledResult = g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+ assert shuffledResult == g.V().values("name").order().by(Order.shuffle).toList()
+
+ def test_clone(self, remote_connection_http):
+ g = traversal().withRemote(remote_connection_http)
+ t = g.V().both()
+ assert 12 == len(t.toList())
+ assert 5 == t.clone().limit(5).count().next()
+ assert 10 == t.clone().limit(10).count().next()
+
+ """
+ # The WsAndHttpChannelizer somehow does not distinguish the ssl handlers so authenticated https remote connection
+ # only work with HttpChannelizer that is currently not in the testing set up, thus this is commented out for now
+
+ def test_authenticated(self, remote_connection_http_authenticated):
+ statics.load_statics(globals())
+ g = traversal().withRemote(remote_connection_http_authenticated)
+
+ assert long(6) == g.V().count().toList()[0]
+ """
diff --git a/gremlin-python/src/main/python/tests/process/test_translator.py b/gremlin-python/src/main/python/tests/process/test_translator.py
index 2afa148372..8a121a5437 100644
--- a/gremlin-python/src/main/python/tests/process/test_translator.py
+++ b/gremlin-python/src/main/python/tests/process/test_translator.py
@@ -21,13 +21,14 @@ Unit tests for the Translator Class.
"""
__author__ = 'Kelvin R. Lawrence (gfxman)'
-from gremlin_python.structure.graph import Graph
+from gremlin_python.structure.graph import Graph, Vertex, Edge, VertexProperty
+from gremlin_python.process.anonymous_traversal import traversal
from gremlin_python.process.graph_traversal import __
from gremlin_python.process.translator import *
from datetime import datetime
-class TestTraversalStrategies(object):
+class TestTranslator(object):
def test_translations(self):
g = traversal().withGraph(Graph())
@@ -317,7 +318,7 @@ class TestTraversalStrategies(object):
"g.V('44').valueMap().with(WithOptions.tokens)"])
# 89
tests.append([g.withStrategies(ReadOnlyStrategy()).addV('test'),
- "g.withStrategies(new ReadOnlyStrategy()).addV('test')"])
+ "g.withStrategies(ReadOnlyStrategy).addV('test')"])
# 90
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route'))
tests.append([g.withStrategies(strategy).V().count(),
@@ -333,11 +334,11 @@ class TestTraversalStrategies(object):
# 93
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'), edges=__.hasLabel('route'))
tests.append([g.withStrategies(ReadOnlyStrategy(),strategy).V().count(),
- "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"])
+ "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'),edges:__.hasLabel('route'))).V().count()"])
# 94
strategy = SubgraphStrategy(vertices=__.has('region', 'US-TX'))
tests.append([g.withStrategies(ReadOnlyStrategy(), strategy).V().count(),
- "g.withStrategies(new ReadOnlyStrategy(),new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"])
+ "g.withStrategies(ReadOnlyStrategy,new SubgraphStrategy(vertices:__.has('region','US-TX'))).V().count()"])
# 95
tests.append([g.with_('evaluationTimeout', 500).V().count(),
"g.withStrategies(new OptionsStrategy(evaluationTimeout:500)).V().count()"])
@@ -349,7 +350,7 @@ class TestTraversalStrategies(object):
"g.withStrategies(new PartitionStrategy(partitionKey:'partition',writePartition:'a',readPartitions:['a'])).addV('test')"])
# 98
tests.append([g.withComputer().V().shortestPath().with_(ShortestPath.target, __.has('name','peter')),
- "g.withStrategies(new VertexProgramStrategy()).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"])
+ "g.withStrategies(VertexProgramStrategy).V().shortestPath().with('~tinkerpop.shortestPath.target',__.has('name','peter'))"])
# 99
tests.append([g.V().has("p1", starting_with("foo")),
@@ -373,6 +374,30 @@ class TestTraversalStrategies(object):
tests.append([g.V().has("p1", None),
"g.V().has('p1',null)"])
+ # 104
+ vertex = Vertex(0, "person")
+ tests.append([g.V(vertex),
+ "g.V(new ReferenceVertex(0,'person'))"])
+
+ # 105
+ outVertex = Vertex(0, "person")
+ inVertex = Vertex(1, "person")
+ edge = Edge(2, outVertex, "knows", inVertex)
+ tests.append([g.inject(edge),
+ "g.inject(new ReferenceEdge(2,'knows',new ReferenceVertex(1,'person'),new ReferenceVertex(0,'person')))"])
+
+ # 106
+ vp = VertexProperty(3, "time", "18:00", None)
+ tests.append([g.inject(vp),
+ "g.inject(new ReferenceVertexProperty(3,'time','18:00'))"])
+
+ # 107
+ tests.append([g.V().has('person', 'name', 'marko').map(lambda: ("it.get().value('name')", "gremlin-groovy")),
+ "g.V().has('person','name','marko').map({it.get().value('name')})"])
+
+ # 108
+ tests.append([g.V().has('person', 'age', Bindings.of('x', P.lt(30))).count(),
+ "g.V().has('person','age',Bindings.instance().of('x', lt(30))).count()"])
tlr = Translator().of('g')